Compare commits

..
Author SHA1 Message Date
“haonan” c2a225377b fix(protocol): align trace schemas with v2/v3 classify rules
RecordedTrace validation now follows classify_value — numeric version
selects v3, otherwise v2 — so mixed pages/states and v2 envelopes with
a version field fail Schema checks before Recall import. Keep standalone
trace_step as a v2|v3 union for tools that validate a single step.
2026-08-23 21:26:39 +08:00
“haonan” e64f4cac9f fix(protocol): keep generic trace schema compatible with v2 and v3
Split standalone v2/v3 schemas and drop TraceV2 deny_unknown_fields so legacy traces with extra keys still parse; mix and version checks stay in RecordedTrace.
2026-08-23 21:26:39 +08:00
Ljy-0827 368507b07d fix(protocol): compatible with dto 2026-08-23 21:26:39 +08:00
Ljy-0827 b750eed5d4 fix(protocol): integrate with vom 2026-08-23 21:26:39 +08:00
“haonan” 3d159c0254 test(protocol): align handshake fixtures with 1.0 floor and clarify v2/v3 naming 2026-08-23 21:26:39 +08:00
“haonan”andCursor 524516cb56 fix(protocol): align Trace v2/v3 classification and hover steps
Require Trace v3 version const 3, reject mixed pages/states payloads, and
align TypeScript selection optionality with the generated schema. Trace v2
on main already records hover steps, so the frozen v2 schema must accept
them instead of dropping them during reduction.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-23 21:26:39 +08:00
“haonan”andCursor f98198949a feat(protocol): negotiate Trace v3 with v2 compatibility
Define Trace v3 wire types and generated schemas while keeping protocol
1.1 peers compatible with 1.0. Default record start still produces Trace
v2, and only an explicit trace_version=3 request enables the new shape.
Refs #90

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-23 21:26:39 +08:00
Zhang GH d05279f991 Merge pull request #98 from NianJiuZst/codex/fix-daemon-idle-close-race
fix(daemon): make idle disconnect updates atomic
2026-08-23 21:25:24 +08:00
NianJiuZst 8352c3c67a fix(daemon): refresh stale idle-task comment
After folding the IPC connection count and activity timestamp into a
single mutex-protected IpcActivityState, the idle task no longer
references the connection count directly; should_exit_for_idle already
returns false while a connection is open or within the idle window.
Update the comment to describe the post-refactor control flow so it
matches the code.
2026-08-20 13:53:23 +08:00
NianJiuZst 4eedd98c1d fix(daemon): make idle disconnect updates atomic 2026-08-20 13:53:23 +08:00
39 changed files with 5540 additions and 3330 deletions
+10 -10
View File
@@ -1,4 +1,4 @@
import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types";
import type { DraftTraceStep, PageRefV2, SelectedOptionV2, StepV2 } from "@/transport/types";
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
@@ -60,7 +60,7 @@ function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
function buildPageRegistry(
steps: DraftTraceStep[],
startUrl?: string,
): { pages: PageRef[]; urlToId: Map<string, string> } {
): { pages: PageRefV2[]; urlToId: Map<string, string> } {
const urls = collectUrls(steps, startUrl);
const urlToId = new Map<string, string>();
const pages = urls.map((url, index) => {
@@ -90,19 +90,19 @@ function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | u
function effectForNavigation(
navigatedTo: string | undefined,
urlToId: Map<string, string>,
): Step["effect"] {
): StepV2["effect"] {
if (!navigatedTo) return undefined;
const pageId = urlToId.get(navigatedTo);
if (!pageId) return undefined;
return { navigated_to: pageId };
}
function withEffect(step: Step, effect: Step["effect"]): Step {
function withEffect(step: StepV2, effect: StepV2["effect"]): StepV2 {
if (!effect) return step;
return { ...step, effect };
}
function toSelection(values: string[], labels?: string[]): SelectedOption[] {
function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
return values.map((value, index) => ({
value,
...(labels?.[index] ? { label: labels[index] } : {}),
@@ -114,7 +114,7 @@ function toV2Step(
id: number,
urlToId: Map<string, string>,
fallbackUrl?: string,
): Step | null {
): StepV2 | null {
if (!shouldIncludeDraft(step)) return null;
const pageUrl = pageUrlForDraft(step, fallbackUrl);
@@ -181,8 +181,8 @@ function toV2Step(
}
export interface ReducedTrace {
pages: PageRef[];
steps: Step[];
pages: PageRefV2[];
steps: StepV2[];
}
/**
@@ -192,7 +192,7 @@ export interface ReducedTrace {
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
const collapsed = collapseNavigations(steps);
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
const out: Step[] = [];
const out: StepV2[] = [];
let id = 1;
let lastUrl = startUrl;
for (const draft of collapsed) {
@@ -210,7 +210,7 @@ export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): Re
export function resolveTraceStartUrl(
drafts: DraftTraceStep[],
startUrl?: string,
pages?: PageRef[],
pages?: PageRefV2[],
): string {
if (startUrl) return startUrl;
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
+9 -9
View File
@@ -29,7 +29,7 @@ import type {
RecordStopParams,
RecordStopResult,
RpcError,
Trace,
TraceV2,
} from "@/transport/types";
import { handleNavigate } from "./navigation";
import {
@@ -50,8 +50,8 @@ interface ActiveRecording {
steps: DraftTraceStep[];
startedAt: string;
startedAtMs: number;
finishPromise: Promise<Trace>;
resolveFinish: (trace: Trace) => void;
finishPromise: Promise<TraceV2>;
resolveFinish: (trace: TraceV2) => void;
rejectFinish: (err: Error) => void;
settled: boolean;
finishing: boolean;
@@ -154,7 +154,7 @@ async function sendRecordStartWithAck(
throw lastError ?? new Error("failed to start recording in content script");
}
function buildTrace(recording: ActiveRecording): Trace {
function buildTrace(recording: ActiveRecording): TraceV2 {
const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
return {
@@ -532,7 +532,7 @@ async function finishRecordingByRequest(
}
}
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<Trace | null> {
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<TraceV2 | null> {
const recording = recordings.get(sessionId);
if (!recording || recording.settled || recording.finishing) return null;
recording.finishing = true;
@@ -574,9 +574,9 @@ export async function handleRecordStart(
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
// instead of flashing ControlOverlay ("Agent 正在控制").
const requestId = makeRequestId(target.tabId);
let resolveFinish!: (trace: Trace) => void;
let resolveFinish!: (trace: TraceV2) => void;
let rejectFinish!: (err: Error) => void;
const finishPromise = new Promise<Trace>((resolve, reject) => {
const finishPromise = new Promise<TraceV2>((resolve, reject) => {
resolveFinish = resolve;
rejectFinish = reject;
});
@@ -785,9 +785,9 @@ export async function handleRecordAwait(
return { code: "cancelled", message: "record_await aborted" };
}
const outcome = await new Promise<{ trace: Trace } | { error: RpcError }>((resolve) => {
const outcome = await new Promise<{ trace: TraceV2 } | { error: RpcError }>((resolve) => {
let settled = false;
const finish = (result: { trace: Trace } | { error: RpcError }) => {
const finish = (result: { trace: TraceV2 } | { error: RpcError }) => {
if (settled) return;
settled = true;
if (timer) clearTimeout(timer);
+119 -28
View File
@@ -657,10 +657,60 @@ export interface EmulateResult {
}
// --------------------------------------------------------------------------
// Semantic record payloads — mirror bsk-protocol record.rs (Trace v2)
// Semantic record payloads mirror the versioned Rust protocol models.
// --------------------------------------------------------------------------
export interface TargetDescriptor {
export const TRACE_VERSION_V3 = 3;
/** Logical v2 identifier. Not a wire field — v2 envelopes omit `version`. */
export const TRACE_VERSION_V2 = 2;
export const VOM_FORMAT_VERSION = 1;
export interface TargetDescriptorV3 {
ref?: string;
role?: string;
name?: string;
ctx?: string;
unmatched?: boolean;
}
export interface RecorderInfo {
bsk: string;
vom: number;
}
export type StopReason = "user_finish" | "cli_stop";
export interface TraceStateV3 {
id: string;
url: string;
title?: string;
body: string;
truncated?: boolean;
}
export interface StepResultV3 {
state: string;
}
export interface StepCommonV3 {
id: number;
state: string;
result: StepResultV3;
}
export type NavigationCause =
| "user_typed"
| "link"
| "form_submit"
| "reload"
| "history"
| "script"
| "browser";
export type FillCommit = "enter" | "suggestion" | "blur";
/** Legacy v2 target shape retained for existing record producers. */
export interface TargetDescriptorV2 {
role?: string;
name?: string;
tag: string;
@@ -673,43 +723,43 @@ export interface TraceEntry {
start_url: string;
}
export interface PageRef {
export interface PageRefV2 {
id: string;
url: string;
title?: string;
}
export interface SelectedOption {
export interface SelectedOptionV2 {
value: string;
label?: string;
}
export interface StepEffect {
export interface StepEffectV2 {
navigated_to: string;
}
export interface StepCommon {
export interface StepCommonV2 {
id: number;
page: string;
effect?: StepEffect;
effect?: StepEffectV2;
}
/** Capture/buffer draft before v2 reduction. */
export type DraftTraceStep =
| {
op: "click";
target: TargetDescriptor;
target: TargetDescriptorV2;
navigated_to?: string;
page_url?: string;
}
| {
op: "hover";
target: TargetDescriptor;
target: TargetDescriptorV2;
page_url?: string;
}
| {
op: "fill";
target: TargetDescriptor;
target: TargetDescriptorV2;
value: string;
redacted?: boolean;
page_url?: string;
@@ -717,14 +767,14 @@ export type DraftTraceStep =
| {
op: "press";
key: string;
target?: TargetDescriptor;
target?: TargetDescriptorV2;
modifiers?: KeyModifier[];
navigated_to?: string;
page_url?: string;
}
| {
op: "select";
target: TargetDescriptor;
target: TargetDescriptorV2;
values: string[];
labels?: string[];
navigated_to?: string;
@@ -737,34 +787,75 @@ export type DraftTraceStep =
};
/** Exported record-only step (trace v2). */
export type Step =
| ({ op: "navigate" } & StepCommon & { to: string })
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
| ({ op: "hover" } & StepCommon & { target: TargetDescriptor })
| ({ op: "fill" } & StepCommon & {
target: TargetDescriptor;
export type StepV2 =
| ({ op: "navigate" } & StepCommonV2 & { to: string })
| ({ op: "click" } & StepCommonV2 & { target: TargetDescriptorV2 })
| ({ op: "hover" } & StepCommonV2 & { target: TargetDescriptorV2 })
| ({ op: "fill" } & StepCommonV2 & {
target: TargetDescriptorV2;
value: string;
redacted?: boolean;
})
| ({ op: "select" } & StepCommon & {
target: TargetDescriptor;
selection: SelectedOption[];
| ({ op: "select" } & StepCommonV2 & {
target: TargetDescriptorV2;
selection: SelectedOptionV2[];
})
| ({ op: "press" } & StepCommon & {
| ({ op: "press" } & StepCommonV2 & {
key: string;
modifiers?: KeyModifier[];
target?: TargetDescriptor;
target?: TargetDescriptorV2;
});
export interface Trace {
export interface TraceV2 {
recorded_at: string;
started_at?: string;
purpose?: string;
entry: TraceEntry;
pages: PageRef[];
steps: Step[];
pages: PageRefV2[];
steps: StepV2[];
}
export interface SelectedOptionV3 {
value: string;
label?: string;
}
export type StepV3 =
| ({ op: "navigate" } & StepCommonV3 & { to: string; cause: NavigationCause })
| ({ op: "click" } & StepCommonV3 & { target: TargetDescriptorV3 })
| ({ op: "hover" } & StepCommonV3 & { target: TargetDescriptorV3 })
| ({ op: "fill" } & StepCommonV3 & {
target: TargetDescriptorV3;
value: string;
commit: FillCommit;
redacted?: boolean;
})
| ({ op: "select" } & StepCommonV3 & {
target: TargetDescriptorV3;
selection?: SelectedOptionV3[];
})
| ({ op: "press" } & StepCommonV3 & {
key: string;
modifiers?: KeyModifier[];
target?: TargetDescriptorV3;
})
| ({ op: "scroll" } & StepCommonV3);
export interface TraceV3 {
version: typeof TRACE_VERSION_V3;
recorded_at: string;
started_at?: string;
purpose?: string;
stopped_by: StopReason;
entry: TraceEntry;
recorder: RecorderInfo;
states: TraceStateV3[];
steps: StepV3[];
}
export type RecordedTrace = TraceV2 | TraceV3;
export type RecordedStep = StepV2 | StepV3;
export interface RecordStartParams {
session_id: string;
tab_id?: number;
@@ -782,7 +873,7 @@ export interface RecordStopParams {
}
export interface RecordStopResult {
trace: Trace;
trace: RecordedTrace;
}
export interface RecordAwaitParams {
@@ -791,5 +882,5 @@ export interface RecordAwaitParams {
}
export interface RecordAwaitResult {
trace: Trace;
trace: RecordedTrace;
}
+8 -4
View File
@@ -8,7 +8,7 @@ use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{
RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams,
RecordStopResult, Trace,
RecordStopResult, RecordedTrace,
};
use clap::{Args, Subcommand};
@@ -190,7 +190,7 @@ fn record_await_ipc_timeout(timeout_ms: u32) -> Duration {
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
}
fn write_trace_file(output: &PathBuf, trace: &Trace) -> Result<(), CliError> {
fn write_trace_file(output: &PathBuf, trace: &RecordedTrace) -> Result<(), CliError> {
let json = serde_json::to_string_pretty(trace)
.context("serialize trace JSON")
.map_err(CliError::Local)?;
@@ -207,7 +207,7 @@ fn write_trace_file(output: &PathBuf, trace: &Trace) -> Result<(), CliError> {
Ok(())
}
fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(), CliError> {
fn render_finish(trace: &RecordedTrace, output: &PathBuf, format: Format) -> Result<(), CliError> {
match format {
Format::Json => {
println!(
@@ -221,7 +221,11 @@ fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(),
);
}
Format::Human => {
println!("saved {} steps to {}", trace.steps.len(), output.display());
let step_count = match trace {
RecordedTrace::V2(trace) => trace.steps.len(),
RecordedTrace::V3(trace) => trace.steps.len(),
};
println!("saved {step_count} steps to {}", output.display());
}
}
Ok(())
+95 -27
View File
@@ -15,7 +15,6 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::path::Path;
use std::sync::Arc;
use std::sync::Mutex;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::{Duration, Instant};
use anyhow::{Context, Result};
@@ -291,8 +290,7 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
});
let started_at = Instant::now();
let activity: Arc<Mutex<Instant>> = Arc::new(Mutex::new(started_at));
let active_ipc_connections = Arc::new(AtomicUsize::new(0));
let activity = Arc::new(Mutex::new(IpcActivityState::new(started_at)));
let (ipc_shutdown_tx, ipc_shutdown_rx) = tokio::sync::oneshot::channel::<()>();
let status = ipc::DaemonStatus {
started_at,
@@ -314,18 +312,14 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
let handler = handler.clone();
let ipc_open = {
let activity = activity.clone();
let active = active_ipc_connections.clone();
move || {
active.fetch_add(1, Ordering::SeqCst);
record_activity(&activity);
record_ipc_open(&activity);
}
};
let ipc_close = {
let activity = activity.clone();
let active = active_ipc_connections.clone();
move || {
active.fetch_sub(1, Ordering::SeqCst);
record_activity(&activity);
record_ipc_close(&activity);
}
};
tokio::spawn(ipc::serve(
@@ -344,7 +338,6 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
let idle_task = {
let activity = activity.clone();
let state = Arc::clone(&state);
let active_ipc_connections = active_ipc_connections.clone();
let daemon_idle = cfg.daemon_idle;
tokio::spawn(async move {
let tick = (daemon_idle / 4).max(Duration::from_millis(250));
@@ -352,22 +345,19 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
ticker.tick().await;
let last = match activity.lock() {
Ok(a) => *a,
Err(p) => *p.into_inner(),
};
if last.elapsed() < daemon_idle {
let ipc_is_idle =
lock_activity(&activity).should_exit_for_idle(Instant::now(), daemon_idle);
if !ipc_is_idle {
continue;
}
// Bridge from M2/M3 (which only knew about connection
// counts) to M4/M5 (browser + session registries):
// hold the daemon alive while any IPC client is
// connected, any browser is paired, or any session
// is live (design §3.2).
if active_ipc_connections.load(Ordering::SeqCst) > 0
|| !state.browsers.is_empty()
|| !state.sessions.is_empty()
{
// IPC liveness (open-connection count + idle window) is already
// enforced above by `should_exit_for_idle`, which reads
// `IpcActivityState` under one lock and returns true only when
// there are zero connections AND the idle interval has elapsed.
// Here we additionally hold the daemon alive while any browser
// is paired or any session is live (design §3.2, M4/M5
// registries).
if !state.browsers.is_empty() || !state.sessions.is_empty() {
continue;
}
info!(
@@ -682,10 +672,55 @@ fn restart_start_args(cfg: &DaemonConfig) -> StartArgs {
}
}
fn record_activity(activity: &Arc<Mutex<Instant>>) {
if let Ok(mut a) = activity.lock() {
*a = Instant::now();
#[derive(Debug)]
struct IpcActivityState {
last_activity: Instant,
active_connections: usize,
}
impl IpcActivityState {
fn new(now: Instant) -> Self {
Self {
last_activity: now,
active_connections: 0,
}
}
fn touch(&mut self, now: Instant) {
self.last_activity = now;
}
fn opened(&mut self, now: Instant) {
self.active_connections = self.active_connections.saturating_add(1);
self.last_activity = now;
}
fn closed(&mut self, now: Instant) {
self.active_connections = self.active_connections.saturating_sub(1);
self.last_activity = now;
}
fn should_exit_for_idle(&self, now: Instant, idle: Duration) -> bool {
self.active_connections == 0 && now.saturating_duration_since(self.last_activity) >= idle
}
}
fn lock_activity(
activity: &Arc<Mutex<IpcActivityState>>,
) -> std::sync::MutexGuard<'_, IpcActivityState> {
activity.lock().unwrap_or_else(|poison| poison.into_inner())
}
fn record_activity(activity: &Arc<Mutex<IpcActivityState>>) {
lock_activity(activity).touch(Instant::now());
}
fn record_ipc_open(activity: &Arc<Mutex<IpcActivityState>>) {
lock_activity(activity).opened(Instant::now());
}
fn record_ipc_close(activity: &Arc<Mutex<IpcActivityState>>) {
lock_activity(activity).closed(Instant::now());
}
/// Initialise tracing for the daemon: write a daily-rotated file in
@@ -1068,4 +1103,37 @@ mod tests {
assert_eq!(args.session_idle, Some(Duration::from_secs(11)));
assert_eq!(args.daemon_idle, Some(Duration::from_secs(22)));
}
#[test]
fn ipc_close_restarts_the_idle_window_atomically() {
let started = Instant::now();
let idle = Duration::from_secs(10);
let mut activity = IpcActivityState::new(started);
activity.opened(started);
let closed_at = started + Duration::from_secs(30);
assert!(
!activity.should_exit_for_idle(closed_at, idle),
"an open IPC connection must hold the daemon alive"
);
activity.closed(closed_at);
assert_eq!(activity.active_connections, 0);
assert_eq!(activity.last_activity, closed_at);
assert!(!activity.should_exit_for_idle(closed_at, idle));
assert!(!activity.should_exit_for_idle(closed_at + idle - Duration::from_millis(1), idle));
assert!(activity.should_exit_for_idle(closed_at + idle, idle));
}
#[test]
fn ipc_activity_keeps_the_daemon_alive_until_a_full_idle_interval_passes() {
let started = Instant::now();
let idle = Duration::from_secs(10);
let mut activity = IpcActivityState::new(started);
let request_at = started + Duration::from_secs(20);
activity.touch(request_at);
assert!(!activity.should_exit_for_idle(request_at + Duration::from_secs(9), idle));
assert!(activity.should_exit_for_idle(request_at + idle, idle));
}
}
@@ -7,10 +7,18 @@
],
"properties": {
"trace": {
"$ref": "#/definitions/Trace"
"$ref": "#/definitions/RecordedTrace"
}
},
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
@@ -21,7 +29,19 @@
"shift"
]
},
"PageRef": {
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"PageRefV2": {
"description": "Page context dictionary entry — referenced by steps via `page` id.",
"type": "object",
"required": [
@@ -43,7 +63,42 @@
}
}
},
"SelectedOption": {
"RecordedTrace": {
"if": {
"required": [
"version"
],
"properties": {
"version": {
"type": "integer"
}
}
},
"then": {
"$ref": "#/definitions/TraceV3"
},
"else": {
"$ref": "#/definitions/TraceV2"
}
},
"RecorderInfo": {
"type": "object",
"required": [
"bsk",
"vom"
],
"properties": {
"bsk": {
"type": "string"
},
"vom": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
}
},
"SelectedOptionV2": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
@@ -61,11 +116,53 @@
}
}
},
"Step": {
"description": "One recorded user action — discriminated union by `op`.",
"SelectedOptionV3": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepEffectV2": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
}
},
"StepResultV3": {
"type": "object",
"required": [
"state"
],
"properties": {
"state": {
"type": "string"
}
}
},
"StepV2": {
"description": "One recorded user action — discriminated union by `op` (v2).",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -77,7 +174,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -105,7 +202,7 @@
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -117,7 +214,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -140,12 +237,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -157,7 +254,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -180,12 +277,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -198,7 +295,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -227,7 +324,7 @@
]
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
},
"value": {
"type": "string"
@@ -235,7 +332,7 @@
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -248,7 +345,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -273,16 +370,16 @@
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOption"
"$ref": "#/definitions/SelectedOptionV2"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -294,7 +391,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -331,7 +428,7 @@
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
},
{
"type": "null"
@@ -342,21 +439,294 @@
}
]
},
"StepEffect": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
"StepV3": {
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"result",
"state",
"to"
],
"properties": {
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"result",
"state",
"target",
"value"
],
"properties": {
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"redacted": {
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV3"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV3"
},
{
"type": "null"
}
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
}
]
},
"TargetDescriptor": {
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
"StopReason": {
"type": "string",
"enum": [
"user_finish",
"cli_stop"
]
},
"TargetDescriptorV2": {
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
"type": "object",
"required": [
"tag"
@@ -397,8 +767,83 @@
}
}
},
"Trace": {
"description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
"TargetDescriptorV3": {
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"unmatched": {
"type": "boolean"
}
}
},
"TraceEntry": {
"type": "object",
"required": [
"start_url"
],
"properties": {
"start_url": {
"type": "string"
}
}
},
"TraceStateV3": {
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
"type": "object",
"required": [
"body",
"id",
"url"
],
"properties": {
"body": {
"description": "Full page observation (front matter + VOM body + annotations).",
"type": "string"
},
"id": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"truncated": {
"type": "boolean"
},
"url": {
"type": "string"
}
}
},
"TraceV2": {
"title": "TraceV2",
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\nUnknown extension fields are ignored so older traces remain readable. `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification.",
"type": "object",
"required": [
"entry",
@@ -413,11 +858,10 @@
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRef"
"$ref": "#/definitions/PageRefV2"
}
},
"purpose": {
"description": "Optional user-provided goal from `--purpose` (metadata only).",
"type": [
"string",
"null"
@@ -433,25 +877,76 @@
"null"
]
},
"states": false,
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/Step"
"$ref": "#/definitions/StepV2"
}
},
"version": {
"description": "Numeric version selects Trace v3. Legacy v2 envelopes omit this field.",
"not": {
"type": "integer"
}
}
}
},
"TraceEntry": {
"description": "Recording entry point — first URL the flow starts from.",
"TraceV3": {
"description": "Wire trace returned by `tool.record_stop` / `await`.",
"type": "object",
"required": [
"start_url"
"entry",
"recorded_at",
"recorder",
"states",
"steps",
"stopped_by",
"version"
],
"properties": {
"start_url": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"type": "string"
},
"recorder": {
"$ref": "#/definitions/RecorderInfo"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": {
"type": "array",
"items": {
"$ref": "#/definitions/TraceStateV3"
}
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV3"
}
},
"stopped_by": {
"$ref": "#/definitions/StopReason"
},
"version": {
"type": "integer",
"const": 3
}
}
},
"additionalProperties": false
}
}
}
@@ -7,10 +7,18 @@
],
"properties": {
"trace": {
"$ref": "#/definitions/Trace"
"$ref": "#/definitions/RecordedTrace"
}
},
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
@@ -21,7 +29,19 @@
"shift"
]
},
"PageRef": {
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"PageRefV2": {
"description": "Page context dictionary entry — referenced by steps via `page` id.",
"type": "object",
"required": [
@@ -43,7 +63,42 @@
}
}
},
"SelectedOption": {
"RecordedTrace": {
"if": {
"required": [
"version"
],
"properties": {
"version": {
"type": "integer"
}
}
},
"then": {
"$ref": "#/definitions/TraceV3"
},
"else": {
"$ref": "#/definitions/TraceV2"
}
},
"RecorderInfo": {
"type": "object",
"required": [
"bsk",
"vom"
],
"properties": {
"bsk": {
"type": "string"
},
"vom": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
}
},
"SelectedOptionV2": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
@@ -61,11 +116,53 @@
}
}
},
"Step": {
"description": "One recorded user action — discriminated union by `op`.",
"SelectedOptionV3": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepEffectV2": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
}
},
"StepResultV3": {
"type": "object",
"required": [
"state"
],
"properties": {
"state": {
"type": "string"
}
}
},
"StepV2": {
"description": "One recorded user action — discriminated union by `op` (v2).",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -77,7 +174,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -105,7 +202,7 @@
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -117,7 +214,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -140,12 +237,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -157,7 +254,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -180,12 +277,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -198,7 +295,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -227,7 +324,7 @@
]
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
},
"value": {
"type": "string"
@@ -235,7 +332,7 @@
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -248,7 +345,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -273,16 +370,16 @@
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOption"
"$ref": "#/definitions/SelectedOptionV2"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -294,7 +391,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -331,7 +428,7 @@
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
},
{
"type": "null"
@@ -342,21 +439,294 @@
}
]
},
"StepEffect": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
"StepV3": {
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"result",
"state",
"to"
],
"properties": {
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"result",
"state",
"target",
"value"
],
"properties": {
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"redacted": {
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV3"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV3"
},
{
"type": "null"
}
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
}
]
},
"TargetDescriptor": {
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
"StopReason": {
"type": "string",
"enum": [
"user_finish",
"cli_stop"
]
},
"TargetDescriptorV2": {
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
"type": "object",
"required": [
"tag"
@@ -397,8 +767,83 @@
}
}
},
"Trace": {
"description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
"TargetDescriptorV3": {
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"unmatched": {
"type": "boolean"
}
}
},
"TraceEntry": {
"type": "object",
"required": [
"start_url"
],
"properties": {
"start_url": {
"type": "string"
}
}
},
"TraceStateV3": {
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
"type": "object",
"required": [
"body",
"id",
"url"
],
"properties": {
"body": {
"description": "Full page observation (front matter + VOM body + annotations).",
"type": "string"
},
"id": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"truncated": {
"type": "boolean"
},
"url": {
"type": "string"
}
}
},
"TraceV2": {
"title": "TraceV2",
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\nUnknown extension fields are ignored so older traces remain readable. `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification.",
"type": "object",
"required": [
"entry",
@@ -413,11 +858,10 @@
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRef"
"$ref": "#/definitions/PageRefV2"
}
},
"purpose": {
"description": "Optional user-provided goal from `--purpose` (metadata only).",
"type": [
"string",
"null"
@@ -433,25 +877,76 @@
"null"
]
},
"states": false,
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/Step"
"$ref": "#/definitions/StepV2"
}
},
"version": {
"description": "Numeric version selects Trace v3. Legacy v2 envelopes omit this field.",
"not": {
"type": "integer"
}
}
}
},
"TraceEntry": {
"description": "Recording entry point — first URL the flow starts from.",
"TraceV3": {
"description": "Wire trace returned by `tool.record_stop` / `await`.",
"type": "object",
"required": [
"start_url"
"entry",
"recorded_at",
"recorder",
"states",
"steps",
"stopped_by",
"version"
],
"properties": {
"start_url": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"type": "string"
},
"recorder": {
"$ref": "#/definitions/RecorderInfo"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": {
"type": "array",
"items": {
"$ref": "#/definitions/TraceStateV3"
}
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV3"
}
},
"stopped_by": {
"$ref": "#/definitions/StopReason"
},
"version": {
"type": "integer",
"const": 3
}
}
},
"additionalProperties": false
}
}
}
+571 -76
View File
@@ -1,49 +1,31 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Trace",
"description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
"type": "object",
"required": [
"entry",
"pages",
"recorded_at",
"steps"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRef"
}
},
"purpose": {
"description": "Optional user-provided goal from `--purpose` (metadata only).",
"type": [
"string",
"null"
]
},
"recorded_at": {
"description": "RFC 3339 timestamp when recording stopped.",
"type": "string"
},
"started_at": {
"type": [
"string",
"null"
]
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/Step"
"title": "RecordedTrace",
"if": {
"required": [
"version"
],
"properties": {
"version": {
"type": "integer"
}
}
},
"then": {
"$ref": "#/definitions/TraceV3"
},
"else": {
"$ref": "#/definitions/TraceV2"
},
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
@@ -54,7 +36,19 @@
"shift"
]
},
"PageRef": {
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"PageRefV2": {
"description": "Page context dictionary entry — referenced by steps via `page` id.",
"type": "object",
"required": [
@@ -76,7 +70,24 @@
}
}
},
"SelectedOption": {
"RecorderInfo": {
"type": "object",
"required": [
"bsk",
"vom"
],
"properties": {
"bsk": {
"type": "string"
},
"vom": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
}
},
"SelectedOptionV2": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
@@ -94,11 +105,53 @@
}
}
},
"Step": {
"description": "One recorded user action — discriminated union by `op`.",
"SelectedOptionV3": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepEffectV2": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
}
},
"StepResultV3": {
"type": "object",
"required": [
"state"
],
"properties": {
"state": {
"type": "string"
}
}
},
"StepV2": {
"description": "One recorded user action — discriminated union by `op` (v2).",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -110,7 +163,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -138,7 +191,7 @@
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -150,7 +203,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -173,12 +226,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -190,7 +243,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -213,12 +266,12 @@
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -231,7 +284,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -260,7 +313,7 @@
]
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
},
"value": {
"type": "string"
@@ -268,7 +321,7 @@
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -281,7 +334,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -306,16 +359,16 @@
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOption"
"$ref": "#/definitions/SelectedOptionV2"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
@@ -327,7 +380,7 @@
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
@@ -364,7 +417,7 @@
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptor"
"$ref": "#/definitions/TargetDescriptorV2"
},
{
"type": "null"
@@ -375,21 +428,294 @@
}
]
},
"StepEffect": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
"StepV3": {
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"result",
"state",
"to"
],
"properties": {
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"result",
"state",
"target",
"value"
],
"properties": {
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"redacted": {
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV3"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV3"
},
{
"type": "null"
}
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
}
]
},
"TargetDescriptor": {
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
"StopReason": {
"type": "string",
"enum": [
"user_finish",
"cli_stop"
]
},
"TargetDescriptorV2": {
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
"type": "object",
"required": [
"tag"
@@ -430,8 +756,40 @@
}
}
},
"TargetDescriptorV3": {
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"unmatched": {
"type": "boolean"
}
}
},
"TraceEntry": {
"description": "Recording entry point — first URL the flow starts from.",
"type": "object",
"required": [
"start_url"
@@ -441,6 +799,143 @@
"type": "string"
}
}
},
"TraceStateV3": {
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
"type": "object",
"required": [
"body",
"id",
"url"
],
"properties": {
"body": {
"description": "Full page observation (front matter + VOM body + annotations).",
"type": "string"
},
"id": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"truncated": {
"type": "boolean"
},
"url": {
"type": "string"
}
}
},
"TraceV2": {
"title": "TraceV2",
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\nUnknown extension fields are ignored so older traces remain readable. `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification.",
"type": "object",
"required": [
"entry",
"pages",
"recorded_at",
"steps"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRefV2"
}
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"description": "RFC 3339 timestamp when recording stopped.",
"type": "string"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": false,
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV2"
}
},
"version": {
"description": "Numeric version selects Trace v3. Legacy v2 envelopes omit this field.",
"not": {
"type": "integer"
}
}
}
},
"TraceV3": {
"description": "Wire trace returned by `tool.record_stop` / `await`.",
"type": "object",
"required": [
"entry",
"recorded_at",
"recorder",
"states",
"steps",
"stopped_by",
"version"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"type": "string"
},
"recorder": {
"$ref": "#/definitions/RecorderInfo"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": {
"type": "array",
"items": {
"$ref": "#/definitions/TraceStateV3"
}
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV3"
}
},
"stopped_by": {
"$ref": "#/definitions/StopReason"
},
"version": {
"type": "integer",
"const": 3
}
},
"additionalProperties": false
}
}
}
+664 -284
View File
@@ -1,286 +1,32 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "Step",
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"to"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target",
"value"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"redacted": {
"type": [
"boolean",
"null"
]
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"selection",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOption"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"page"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptor"
},
{
"type": "null"
}
]
}
}
}
],
"title": "RecordedStep",
"not": {
"required": [
"page",
"state"
]
},
"if": {
"required": [
"state"
]
},
"then": {
"$ref": "#/definitions/StepV3"
},
"else": {
"$ref": "#/definitions/StepV2"
},
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
@@ -291,7 +37,19 @@
"shift"
]
},
"SelectedOption": {
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"SelectedOptionV2": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
@@ -309,7 +67,25 @@
}
}
},
"StepEffect": {
"SelectedOptionV3": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepEffectV2": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
@@ -322,8 +98,579 @@
}
}
},
"TargetDescriptor": {
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
"StepResultV3": {
"type": "object",
"required": [
"state"
],
"properties": {
"state": {
"type": "string"
}
}
},
"StepV2": {
"description": "One recorded user action — discriminated union by `op` (v2).",
"oneOf": [
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"to"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target",
"value"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"redacted": {
"type": [
"boolean",
"null"
]
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"selection",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV2"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"page"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV2"
},
{
"type": "null"
}
]
}
}
}
]
},
"StepV3": {
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"result",
"state",
"to"
],
"properties": {
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"result",
"state",
"target",
"value"
],
"properties": {
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"redacted": {
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV3"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV3"
},
{
"type": "null"
}
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
]
},
"TargetDescriptorV2": {
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
"type": "object",
"required": [
"tag"
@@ -363,6 +710,39 @@
"type": "string"
}
}
},
"TargetDescriptorV3": {
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"unmatched": {
"type": "boolean"
}
}
}
}
}
@@ -0,0 +1,368 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "StepV2",
"description": "One recorded user action — discriminated union by `op` (v2).",
"oneOf": [
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"to"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target",
"value"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"redacted": {
"type": [
"boolean",
"null"
]
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"selection",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV2"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"page"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV2"
},
{
"type": "null"
}
]
}
}
}
],
"definitions": {
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
"enum": [
"alt",
"ctrl",
"meta",
"shift"
]
},
"SelectedOptionV2": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepEffectV2": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
}
},
"TargetDescriptorV2": {
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
"type": "object",
"required": [
"tag"
],
"properties": {
"name": {
"type": [
"string",
"null"
]
},
"name_attr": {
"type": [
"string",
"null"
]
},
"nearby_label": {
"type": [
"string",
"null"
]
},
"placeholder": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"tag": {
"type": "string"
}
}
}
}
}
@@ -0,0 +1,375 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "StepV3",
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"result",
"state",
"to"
],
"properties": {
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"result",
"state",
"target",
"value"
],
"properties": {
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"redacted": {
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV3"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV3"
},
{
"type": "null"
}
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
],
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
"enum": [
"alt",
"ctrl",
"meta",
"shift"
]
},
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"SelectedOptionV3": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepResultV3": {
"type": "object",
"required": [
"state"
],
"properties": {
"state": {
"type": "string"
}
}
},
"TargetDescriptorV3": {
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"unmatched": {
"type": "boolean"
}
}
}
}
}
+451
View File
@@ -0,0 +1,451 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "TraceV2",
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\nUnknown extension fields are ignored so older traces remain readable. `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification.",
"type": "object",
"required": [
"entry",
"pages",
"recorded_at",
"steps"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRefV2"
}
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"description": "RFC 3339 timestamp when recording stopped.",
"type": "string"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": false,
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV2"
}
},
"version": {
"description": "Numeric version selects Trace v3. Legacy v2 envelopes omit this field.",
"not": {
"type": "integer"
}
}
},
"definitions": {
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
"enum": [
"alt",
"ctrl",
"meta",
"shift"
]
},
"PageRefV2": {
"description": "Page context dictionary entry — referenced by steps via `page` id.",
"type": "object",
"required": [
"id",
"url"
],
"properties": {
"id": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"url": {
"type": "string"
}
}
},
"SelectedOptionV2": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepEffectV2": {
"description": "Observed navigation after a step (objective fact only).",
"type": "object",
"required": [
"navigated_to"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"type": "string"
}
}
},
"StepV2": {
"description": "One recorded user action — discriminated union by `op` (v2).",
"oneOf": [
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"to"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target",
"value"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"redacted": {
"type": [
"boolean",
"null"
]
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"selection",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV2"
}
},
"target": {
"$ref": "#/definitions/TargetDescriptorV2"
}
}
},
{
"description": "Fields shared by every v2 step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"page"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffectV2"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV2"
},
{
"type": "null"
}
]
}
}
}
]
},
"TargetDescriptorV2": {
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
"type": "object",
"required": [
"tag"
],
"properties": {
"name": {
"type": [
"string",
"null"
]
},
"name_attr": {
"type": [
"string",
"null"
]
},
"nearby_label": {
"type": [
"string",
"null"
]
},
"placeholder": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"tag": {
"type": "string"
}
}
},
"TraceEntry": {
"type": "object",
"required": [
"start_url"
],
"properties": {
"start_url": {
"type": "string"
}
}
}
}
}
+496
View File
@@ -0,0 +1,496 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "TraceV3",
"description": "Wire trace returned by `tool.record_stop` / `await`.",
"type": "object",
"required": [
"entry",
"recorded_at",
"recorder",
"states",
"steps",
"stopped_by",
"version"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"purpose": {
"type": [
"string",
"null"
]
},
"recorded_at": {
"type": "string"
},
"recorder": {
"$ref": "#/definitions/RecorderInfo"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": {
"type": "array",
"items": {
"$ref": "#/definitions/TraceStateV3"
}
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/StepV3"
}
},
"stopped_by": {
"$ref": "#/definitions/StopReason"
},
"version": {
"type": "integer",
"const": 3
}
},
"additionalProperties": false,
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
"enum": [
"alt",
"ctrl",
"meta",
"shift"
]
},
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"RecorderInfo": {
"type": "object",
"required": [
"bsk",
"vom"
],
"properties": {
"bsk": {
"type": "string"
},
"vom": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
}
},
"SelectedOptionV3": {
"description": "One selected option (`select` op).",
"type": "object",
"required": [
"value"
],
"properties": {
"label": {
"type": [
"string",
"null"
]
},
"value": {
"type": "string"
}
}
},
"StepResultV3": {
"type": "object",
"required": [
"state"
],
"properties": {
"state": {
"type": "string"
}
}
},
"StepV3": {
"description": "One recorded user action — discriminated union by `op`.",
"oneOf": [
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"result",
"state",
"to"
],
"properties": {
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"navigate"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"click"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"result",
"state",
"target",
"value"
],
"properties": {
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"fill"
]
},
"redacted": {
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
},
"value": {
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state",
"target"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"select"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"selection": {
"type": "array",
"items": {
"$ref": "#/definitions/SelectedOptionV3"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptorV3"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"key",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"key": {
"type": "string"
},
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"op": {
"type": "string",
"enum": [
"press"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"anyOf": [
{
"$ref": "#/definitions/TargetDescriptorV3"
},
{
"type": "null"
}
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
]
},
"StopReason": {
"type": "string",
"enum": [
"user_finish",
"cli_stop"
]
},
"TargetDescriptorV3": {
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"ref": {
"type": [
"string",
"null"
]
},
"role": {
"type": [
"string",
"null"
]
},
"unmatched": {
"type": "boolean"
}
}
},
"TraceEntry": {
"type": "object",
"required": [
"start_url"
],
"properties": {
"start_url": {
"type": "string"
}
}
},
"TraceStateV3": {
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
"type": "object",
"required": [
"body",
"id",
"url"
],
"properties": {
"body": {
"description": "Full page observation (front matter + VOM body + annotations).",
"type": "string"
},
"id": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"truncated": {
"type": "boolean"
},
"url": {
"type": "string"
}
}
}
}
}
+6 -2
View File
@@ -113,8 +113,12 @@ fn main() {
dump!(RequestHelpParams, "tool_request_help_params");
dump!(RequestHelpResult, "tool_request_help_result");
dump!(Trace, "trace");
dump!(Step, "trace_step");
dump!(TraceV2, "trace_v2");
dump!(TraceV3, "trace_v3");
dump!(RecordedTrace, "trace");
dump!(StepV2, "trace_step_v2");
dump!(StepV3, "trace_step_v3");
dump!(RecordedStep, "trace_step");
dump!(RecordStartParams, "tool_record_start_params");
dump!(RecordStartResult, "tool_record_start_result");
dump!(RecordStopParams, "tool_record_stop_params");
+3
View File
@@ -9,6 +9,9 @@ pub mod navigation;
pub mod network;
pub mod observation;
pub mod record;
mod record_common;
mod record_v2;
mod record_v3;
pub mod script;
pub mod session;
pub mod tabs;
+380 -388
View File
@@ -1,147 +1,23 @@
//! Semantic user-action recording (`tool.record_start` / `stop` / `await`).
//!
//! Trace v2 is a **record-only** log of user actions: what was clicked, filled,
//! selected, and where navigation occurred. No LLM runs during recording, so
//! variable-vs-constant classification is **not** stored — executing agents
//! infer that at run time from raw values and control names.
//! Wire traces are either Trace v2 (`pages[]`) or Trace v3 (`version: 3`,
//! `states[]`). Version-specific models live in `record_v2` / `record_v3`.
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use super::interaction::KeyModifier;
pub use super::record_common::TraceEntry;
pub use super::record_v2::{
PageRefV2, SelectedOptionV2, StepCommonV2, StepEffectV2, StepV2, TargetDescriptorV2, TraceV2,
};
pub use super::record_v3::{
FillCommit, NavigationCause, RecorderInfo, SelectedOptionV3, StepCommonV3, StepResultV3,
StepV3, StopReason, TRACE_VERSION_V3, TargetDescriptorV3, TraceStateV3, TraceV3,
VOM_FORMAT_VERSION,
};
// ---------------------------------------------------------------------------
// Target
// ---------------------------------------------------------------------------
/// Stable semantic handle for an interacted element.
///
/// `name` and `nearby_label` are **untrusted page text**.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TargetDescriptor {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub role: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
pub tag: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name_attr: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub placeholder: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub nearby_label: Option<String>,
}
// ---------------------------------------------------------------------------
// Trace envelope
// ---------------------------------------------------------------------------
/// Recording entry point — first URL the flow starts from.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TraceEntry {
pub start_url: String,
}
/// Page context dictionary entry — referenced by steps via `page` id.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct PageRef {
pub id: String,
pub url: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
}
/// One selected option (`select` op).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct SelectedOption {
pub value: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub label: Option<String>,
}
/// Observed navigation after a step (objective fact only).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepEffect {
/// Reference into `pages[]` for the destination page.
pub navigated_to: String,
}
/// Fields shared by every step variant (flattened in JSON).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepCommon {
pub id: u32,
/// Reference into `pages[]`.
pub page: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub effect: Option<StepEffect>,
}
// ---------------------------------------------------------------------------
// Step op-specific payloads
// ---------------------------------------------------------------------------
/// One recorded user action — discriminated union by `op`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[serde(tag = "op", rename_all = "snake_case")]
pub enum Step {
Navigate {
#[serde(flatten)]
common: StepCommon,
to: String,
},
Click {
#[serde(flatten)]
common: StepCommon,
target: TargetDescriptor,
},
Hover {
#[serde(flatten)]
common: StepCommon,
target: TargetDescriptor,
},
Fill {
#[serde(flatten)]
common: StepCommon,
target: TargetDescriptor,
value: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
redacted: Option<bool>,
},
Select {
#[serde(flatten)]
common: StepCommon,
target: TargetDescriptor,
selection: Vec<SelectedOption>,
},
Press {
#[serde(flatten)]
common: StepCommon,
key: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
modifiers: Option<Vec<KeyModifier>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
target: Option<TargetDescriptor>,
},
}
// ---------------------------------------------------------------------------
// Trace root
// ---------------------------------------------------------------------------
/// Persisted user-action trace exported by `tool.record_stop` / `await`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct Trace {
/// RFC 3339 timestamp when recording stopped.
pub recorded_at: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub started_at: Option<String>,
/// Optional user-provided goal from `--purpose` (metadata only).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub purpose: Option<String>,
pub entry: TraceEntry,
pub pages: Vec<PageRef>,
pub steps: Vec<Step>,
}
/// Logical v2 identifier. Not a wire field — v2 envelopes omit `version`.
pub const TRACE_VERSION_V2: u32 = 2;
// ---------------------------------------------------------------------------
// RPC params / results
@@ -169,9 +45,209 @@ pub struct RecordStopParams {
pub session_id: String,
}
/// Wire trace payload — v2 (legacy `pages[]`) or v3 (`version: 3`, `states[]`).
///
/// Classification matches [`RecordedTrace::classify_value`]: a numeric `version`
/// selects v3 (and forbids `pages[]`); otherwise the envelope is v2 and must
/// not include `states[]`.
#[derive(Debug, Clone, PartialEq)]
pub enum RecordedTrace {
V2(TraceV2),
V3(TraceV3),
}
impl RecordedTrace {
pub fn classify_value(v: &serde_json::Value) -> Result<Self, String> {
if let Some(ver) = v.get("version").and_then(|x| x.as_u64()) {
if ver != u64::from(TRACE_VERSION_V3) {
return Err(format!("unsupported trace version {ver}"));
}
if v.get("pages").is_some() {
return Err("trace v3 must not include legacy pages[]".into());
}
return serde_json::from_value(v.clone())
.map(RecordedTrace::V3)
.map_err(|e| e.to_string());
}
if v.get("states").is_some() {
return Err("trace v2 must not include states[]; set version: 3 for Trace v3".into());
}
if v.get("pages").is_some() {
return serde_json::from_value(v.clone())
.map(RecordedTrace::V2)
.map_err(|e| e.to_string());
}
Err("ambiguous or unparseable trace".into())
}
}
impl Serialize for RecordedTrace {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
match self {
RecordedTrace::V2(t) => t.serialize(serializer),
RecordedTrace::V3(t) => t.serialize(serializer),
}
}
}
impl<'de> Deserialize<'de> for RecordedTrace {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let value = serde_json::Value::deserialize(deserializer)?;
Self::classify_value(&value).map_err(serde::de::Error::custom)
}
}
impl JsonSchema for RecordedTrace {
fn schema_name() -> String {
"RecordedTrace".into()
}
fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::schema::Schema {
// Mirror classify_value: a numeric `version` selects v3; otherwise v2.
// TraceV2 stays open for unknown extension fields but forbids `states[]`
// and a numeric `version`. TraceV3 already denies `pages[]`.
let mut version_props = schemars::Map::new();
version_props.insert(
"version".into(),
schemars::schema::SchemaObject {
instance_type: Some(schemars::schema::InstanceType::Integer.into()),
..Default::default()
}
.into(),
);
let mut required = schemars::Set::new();
required.insert("version".into());
schemars::schema::SchemaObject {
subschemas: Some(Box::new(schemars::schema::SubschemaValidation {
if_schema: Some(Box::new(
schemars::schema::SchemaObject {
object: Some(Box::new(schemars::schema::ObjectValidation {
properties: version_props,
required,
..Default::default()
})),
..Default::default()
}
.into(),
)),
then_schema: Some(Box::new(generator.subschema_for::<TraceV3>())),
else_schema: Some(Box::new(generator.subschema_for::<TraceV2>())),
..Default::default()
})),
..Default::default()
}
.into()
}
}
/// One recorded step — v2 (`page`) or v3 (`state` / `result.state`).
///
/// Classification matches [`RecordedStep::classify_value`]: `state` selects v3
/// (and forbids `page`); otherwise the step is v2 and must include `page`.
#[derive(Debug, Clone, PartialEq)]
pub enum RecordedStep {
V2(StepV2),
V3(StepV3),
}
impl RecordedStep {
pub fn classify_value(v: &serde_json::Value) -> Result<Self, String> {
if v.get("state").is_some() {
if v.get("page").is_some() {
return Err("step v3 must not include legacy page".into());
}
return serde_json::from_value(v.clone())
.map(RecordedStep::V3)
.map_err(|e| e.to_string());
}
if v.get("page").is_some() {
return serde_json::from_value(v.clone())
.map(RecordedStep::V2)
.map_err(|e| e.to_string());
}
Err("ambiguous or unparseable step".into())
}
}
impl Serialize for RecordedStep {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
match self {
RecordedStep::V2(s) => s.serialize(serializer),
RecordedStep::V3(s) => s.serialize(serializer),
}
}
}
impl<'de> Deserialize<'de> for RecordedStep {
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
where
D: serde::Deserializer<'de>,
{
let value = serde_json::Value::deserialize(deserializer)?;
Self::classify_value(&value).map_err(serde::de::Error::custom)
}
}
impl JsonSchema for RecordedStep {
fn schema_name() -> String {
"RecordedStep".into()
}
fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::schema::Schema {
// Mirror classify_value: `state` selects v3; otherwise v2.
// Mixed page + state is rejected at this layer because StepV2/StepV3
// still allow unknown extension fields.
let mut required = schemars::Set::new();
required.insert("state".into());
let mut mixed_keys = schemars::Set::new();
mixed_keys.insert("page".into());
mixed_keys.insert("state".into());
schemars::schema::SchemaObject {
subschemas: Some(Box::new(schemars::schema::SubschemaValidation {
if_schema: Some(Box::new(
schemars::schema::SchemaObject {
object: Some(Box::new(schemars::schema::ObjectValidation {
required,
..Default::default()
})),
..Default::default()
}
.into(),
)),
then_schema: Some(Box::new(generator.subschema_for::<StepV3>())),
else_schema: Some(Box::new(generator.subschema_for::<StepV2>())),
not: Some(Box::new(
schemars::schema::SchemaObject {
object: Some(Box::new(schemars::schema::ObjectValidation {
required: mixed_keys,
..Default::default()
})),
..Default::default()
}
.into(),
)),
..Default::default()
})),
..Default::default()
}
.into()
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct RecordStopResult {
pub trace: Trace,
pub trace: RecordedTrace,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
@@ -183,281 +259,197 @@ pub struct RecordAwaitParams {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct RecordAwaitResult {
pub trace: Trace,
pub trace: RecordedTrace,
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn sample_common(id: u32) -> StepCommon {
StepCommon {
id,
page: "p1".into(),
effect: None,
#[test]
fn recorded_trace_classifies_v2_and_v3() {
let v2 = json!({
"recorded_at": "2026-07-21T08:00:00Z",
"entry": { "start_url": "https://example.com/" },
"pages": [{ "id": "p1", "url": "https://example.com/" }],
"steps": []
});
match RecordedTrace::classify_value(&v2).unwrap() {
RecordedTrace::V2(_) => {}
other => panic!("expected v2, got {other:?}"),
}
}
fn sample_target() -> TargetDescriptor {
TargetDescriptor {
role: Some("button".into()),
name: Some("发布".into()),
tag: "button".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
}
}
fn sample_trace() -> Trace {
Trace {
recorded_at: "2026-07-17T09:01:10Z".into(),
started_at: Some("2026-07-17T09:00:00Z".into()),
purpose: Some("发布一篇文章".into()),
entry: TraceEntry {
start_url: "https://x.com/editor".into(),
},
pages: vec![
PageRef {
id: "p1".into(),
url: "https://x.com/editor".into(),
title: Some("写文章".into()),
},
PageRef {
id: "p2".into(),
url: "https://x.com/p/99".into(),
title: None,
},
],
steps: vec![
Step::Fill {
common: sample_common(1),
target: TargetDescriptor {
role: Some("textbox".into()),
name: Some("标题".into()),
tag: "input".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
},
value: "我的第一篇文章".into(),
redacted: None,
},
Step::Select {
common: sample_common(3),
target: TargetDescriptor {
role: Some("combobox".into()),
name: Some("分类".into()),
tag: "select".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
},
selection: vec![SelectedOption {
value: "tech".into(),
label: Some("技术分享".into()),
}],
},
Step::Click {
common: StepCommon {
effect: Some(StepEffect {
navigated_to: "p2".into(),
}),
..sample_common(4)
},
target: sample_target(),
},
],
let v3 = json!({
"version": 3,
"recorded_at": "2026-07-21T08:00:00Z",
"stopped_by": "user_finish",
"entry": { "start_url": "https://example.com/" },
"recorder": { "bsk": "0.1.10", "vom": 1 },
"states": [],
"steps": []
});
match RecordedTrace::classify_value(&v3).unwrap() {
RecordedTrace::V3(t) => assert_eq!(t.version, 3),
other => panic!("expected v3, got {other:?}"),
}
}
#[test]
fn step_click_round_trips() {
let step = Step::Click {
common: sample_common(1),
target: sample_target(),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v.get("op").and_then(|v| v.as_str()), Some("click"));
assert_eq!(v.get("page").and_then(|v| v.as_str()), Some("p1"));
assert!(v.get("key").is_none());
let round: Step = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
fn recorded_trace_rejects_mixed_and_unsupported_versions() {
let v2_with_states = json!({
"recorded_at": "2026-07-21T08:00:00Z",
"entry": { "start_url": "https://example.com/" },
"pages": [{ "id": "p1", "url": "https://example.com/" }],
"states": [],
"steps": []
});
assert!(RecordedTrace::classify_value(&v2_with_states).is_err());
let v3_with_pages = json!({
"version": 3,
"recorded_at": "2026-07-21T08:00:00Z",
"stopped_by": "user_finish",
"entry": { "start_url": "https://example.com/" },
"recorder": { "bsk": "0.1.10", "vom": 1 },
"pages": [{ "id": "p1", "url": "https://example.com/" }],
"states": [],
"steps": []
});
assert!(RecordedTrace::classify_value(&v3_with_pages).is_err());
let unsupported = json!({
"version": 2,
"recorded_at": "2026-07-21T08:00:00Z",
"stopped_by": "user_finish",
"entry": { "start_url": "https://example.com/" },
"recorder": { "bsk": "0.1.10", "vom": 1 },
"states": [],
"steps": []
});
assert!(RecordedTrace::classify_value(&unsupported).is_err());
assert!(serde_json::from_value::<TraceV3>(unsupported).is_err());
}
#[test]
fn step_fill_with_raw_value_round_trips() {
let step = Step::Fill {
common: sample_common(2),
target: TargetDescriptor {
role: Some("textbox".into()),
name: Some("服务名称".into()),
tag: "input".into(),
name_attr: Some("serviceName".into()),
placeholder: None,
nearby_label: None,
},
value: "my-svc".into(),
redacted: None,
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "fill");
assert_eq!(v["value"], "my-svc");
let round: Step = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_fill_password_is_redacted() {
let step = Step::Fill {
common: sample_common(1),
target: TargetDescriptor {
role: Some("textbox".into()),
name: Some("密码".into()),
tag: "input".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
},
value: "***".into(),
redacted: Some(true),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["value"], "***");
assert_eq!(v["redacted"], true);
}
#[test]
fn step_select_uses_object_array() {
let step = Step::Select {
common: sample_common(3),
target: TargetDescriptor {
role: Some("combobox".into()),
name: Some("分类".into()),
tag: "select".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
},
selection: vec![SelectedOption {
value: "tech".into(),
label: Some("技术分享".into()),
}],
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["selection"][0]["value"], "tech");
assert_eq!(v["selection"][0]["label"], "技术分享");
let round: Step = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_navigate_round_trips() {
let step = Step::Navigate {
common: sample_common(1),
to: "https://example.com".into(),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "navigate");
assert_eq!(v["to"], "https://example.com");
let round: Step = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_press_with_modifiers_round_trips() {
let step = Step::Press {
common: StepCommon {
effect: Some(StepEffect {
navigated_to: "p2".into(),
}),
..sample_common(2)
},
key: "Enter".into(),
modifiers: Some(vec![KeyModifier::Ctrl, KeyModifier::Shift]),
target: Some(sample_target()),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["key"], "Enter");
assert_eq!(v["modifiers"], json!(["ctrl", "shift"]));
assert_eq!(v["effect"]["navigated_to"], "p2");
let round: Step = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn trace_record_only_round_trips() {
let trace = sample_trace();
let v = serde_json::to_value(&trace).unwrap();
assert!(v.get("version").is_none());
fn assert_classifies_by_integer_version(schema: &serde_json::Value) {
assert_eq!(
v.get("purpose").and_then(|v| v.as_str()),
Some("发布一篇文章")
schema["if"]["required"],
json!(["version"]),
"numeric version is the RecordedTrace classification key"
);
assert!(v.get("parameters").is_none());
assert!(v.get("site").is_none());
assert!(v.get("goal").is_none());
assert_eq!(
v.get("entry")
.and_then(|e| e.get("start_url"))
.and_then(|u| u.as_str()),
Some("https://x.com/editor")
);
let round: Trace = serde_json::from_value(v).unwrap();
assert_eq!(round, trace);
assert_eq!(schema["if"]["properties"]["version"]["type"], "integer");
assert_eq!(schema["then"]["$ref"], "#/definitions/TraceV3");
assert_eq!(schema["else"]["$ref"], "#/definitions/TraceV2");
}
#[test]
fn discriminated_union_click_ignores_extra_key_field() {
let bad = json!({
fn recorded_trace_schema_includes_v2_and_v3() {
let schema = serde_json::to_value(schemars::schema_for!(RecordStopResult)).unwrap();
assert_classifies_by_integer_version(&schema["definitions"]["RecordedTrace"]);
assert_eq!(
schema["definitions"]["TraceV3"]["properties"]["version"]["const"],
3
);
}
#[test]
fn recorded_trace_schema_follows_classify_value() {
let standalone = serde_json::to_value(schemars::schema_for!(RecordedTrace)).unwrap();
assert_classifies_by_integer_version(&standalone);
assert!(standalone["definitions"].get("TraceV2").is_some());
assert!(standalone["definitions"].get("TraceV3").is_some());
let stop_result = serde_json::to_value(schemars::schema_for!(RecordStopResult)).unwrap();
assert_classifies_by_integer_version(&stop_result["definitions"]["RecordedTrace"]);
}
#[test]
fn standalone_trace_schema_is_recorded_trace_union() {
let schema = serde_json::to_value(schemars::schema_for!(RecordedTrace)).unwrap();
assert_eq!(schema["title"], "RecordedTrace");
assert_classifies_by_integer_version(&schema);
assert!(schema["definitions"].get("TraceV2").is_some());
assert!(schema["definitions"].get("TraceV3").is_some());
}
#[test]
fn recorded_trace_accepts_v2_with_unknown_extension_fields() {
let v2 = json!({
"recorded_at": "2026-07-21T08:00:00Z",
"entry": { "start_url": "https://example.com/" },
"pages": [{ "id": "p1", "url": "https://example.com/" }],
"steps": [],
"meta": { "tool": "legacy-exporter" }
});
match RecordedTrace::classify_value(&v2).unwrap() {
RecordedTrace::V2(trace) => assert_eq!(trace.pages.len(), 1),
other => panic!("expected v2, got {other:?}"),
}
}
fn v2_click() -> serde_json::Value {
json!({
"op": "click",
"id": 1,
"page": "p1",
"target": { "tag": "button", "name": "OK" },
"key": "Enter"
});
let step: Step = serde_json::from_value(bad).unwrap();
assert!(matches!(step, Step::Click { .. }));
let v = serde_json::to_value(&step).unwrap();
assert!(v.get("key").is_none());
"target": { "tag": "button", "role": "button", "name": "发布" }
})
}
fn v3_click() -> serde_json::Value {
json!({
"op": "click",
"id": 1,
"state": "s1",
"result": { "state": "s2" },
"target": { "ref": "e1", "role": "button", "name": "发布" }
})
}
fn assert_step_classifies_by_state(schema: &serde_json::Value) {
assert_eq!(
schema["if"]["required"],
json!(["state"]),
"state is the RecordedStep classification key"
);
assert_eq!(schema["then"]["$ref"], "#/definitions/StepV3");
assert_eq!(schema["else"]["$ref"], "#/definitions/StepV2");
let mut forbidden = schema["not"]["required"]
.as_array()
.expect("RecordedStep schema must reject mixed page + state steps")
.iter()
.filter_map(|v| v.as_str())
.collect::<Vec<_>>();
forbidden.sort_unstable();
assert_eq!(forbidden, ["page", "state"]);
}
#[test]
fn extension_trace_deserializes() {
let v = json!({
"recorded_at": "2026-07-21T08:00:00Z",
"started_at": "2026-07-21T07:59:00Z",
"purpose": "demo",
"entry": { "start_url": "https://example.com/editor" },
"pages": [
{ "id": "p1", "url": "https://example.com/editor" },
{ "id": "p2", "url": "https://example.com/p/99" }
],
"steps": [
{
"op": "fill",
"id": 1,
"page": "p1",
"target": { "tag": "input", "role": "textbox", "name": "标题" },
"value": "hello"
},
{
"op": "click",
"id": 2,
"page": "p1",
"target": { "tag": "button", "role": "button", "name": "发布" },
"effect": { "navigated_to": "p2" }
}
]
});
let trace: Trace = serde_json::from_value(v).unwrap();
assert_eq!(trace.entry.start_url, "https://example.com/editor");
assert_eq!(trace.pages.len(), 2);
assert_eq!(trace.steps.len(), 2);
fn recorded_step_classifies_v2_and_v3() {
match RecordedStep::classify_value(&v2_click()).unwrap() {
RecordedStep::V2(StepV2::Click { .. }) => {}
other => panic!("expected v2 click, got {other:?}"),
}
match RecordedStep::classify_value(&v3_click()).unwrap() {
RecordedStep::V3(StepV3::Click { .. }) => {}
other => panic!("expected v3 click, got {other:?}"),
}
}
#[test]
fn recorded_step_rejects_mixed_page_and_state() {
let mut mixed = v2_click();
mixed["state"] = json!("s1");
mixed["result"] = json!({ "state": "s2" });
assert!(RecordedStep::classify_value(&mixed).is_err());
}
#[test]
fn recorded_step_schema_follows_classify_value() {
let schema = serde_json::to_value(schemars::schema_for!(RecordedStep)).unwrap();
assert_eq!(schema["title"], "RecordedStep");
assert_step_classifies_by_state(&schema);
assert!(schema["definitions"].get("StepV2").is_some());
assert!(schema["definitions"].get("StepV3").is_some());
}
}
@@ -0,0 +1,7 @@
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TraceEntry {
pub start_url: String,
}
+330
View File
@@ -0,0 +1,330 @@
//! Trace v2 — record-only user-action log with `pages[]` and step `page` refs.
//!
//! Legacy wire format with no top-level `version` field.
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use super::interaction::KeyModifier;
use super::record_common::TraceEntry;
/// Stable semantic handle for an interacted element (v2).
///
/// `name` and `nearby_label` are **untrusted page text**.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TargetDescriptorV2 {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub role: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
pub tag: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name_attr: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub placeholder: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub nearby_label: Option<String>,
}
/// Page context dictionary entry — referenced by steps via `page` id.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct PageRefV2 {
pub id: String,
pub url: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
}
/// One selected option (`select` op).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct SelectedOptionV2 {
pub value: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub label: Option<String>,
}
/// Observed navigation after a step (objective fact only).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepEffectV2 {
/// Reference into `pages[]` for the destination page.
pub navigated_to: String,
}
/// Fields shared by every v2 step variant (flattened in JSON).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepCommonV2 {
pub id: u32,
/// Reference into `pages[]`.
pub page: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub effect: Option<StepEffectV2>,
}
/// One recorded user action — discriminated union by `op` (v2).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[serde(tag = "op", rename_all = "snake_case")]
pub enum StepV2 {
Navigate {
#[serde(flatten)]
common: StepCommonV2,
to: String,
},
Click {
#[serde(flatten)]
common: StepCommonV2,
target: TargetDescriptorV2,
},
Hover {
#[serde(flatten)]
common: StepCommonV2,
target: TargetDescriptorV2,
},
Fill {
#[serde(flatten)]
common: StepCommonV2,
target: TargetDescriptorV2,
value: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
redacted: Option<bool>,
},
Select {
#[serde(flatten)]
common: StepCommonV2,
target: TargetDescriptorV2,
selection: Vec<SelectedOptionV2>,
},
Press {
#[serde(flatten)]
common: StepCommonV2,
key: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
modifiers: Option<Vec<KeyModifier>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
target: Option<TargetDescriptorV2>,
},
}
/// Persisted user-action trace exported by legacy `tool.record_stop` / `await`.
///
/// Unknown extension fields are ignored so older traces remain readable.
/// `states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace`
/// classification — serde still ignores them, but the JSON Schema rejects them.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct TraceV2 {
/// RFC 3339 timestamp when recording stopped.
pub recorded_at: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub started_at: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub purpose: Option<String>,
pub entry: TraceEntry,
pub pages: Vec<PageRefV2>,
pub steps: Vec<StepV2>,
}
fn constrain_trace_v2_schema(mut schema: schemars::schema::Schema) -> schemars::schema::Schema {
let schemars::schema::Schema::Object(obj) = &mut schema else {
return schema;
};
obj.metadata().title = Some("TraceV2".into());
obj.metadata().description = Some(
"Persisted user-action trace exported by legacy `tool.record_stop` / `await`.\n\n\
Unknown extension fields are ignored so older traces remain readable. \
`states[]` and a numeric `version` are reserved for Trace v3 / `RecordedTrace` classification."
.into(),
);
let object = obj.object.get_or_insert_with(Default::default);
object
.properties
.insert("states".into(), schemars::schema::Schema::Bool(false));
object.properties.insert(
"version".into(),
schemars::schema::SchemaObject {
metadata: Some(Box::new(schemars::schema::Metadata {
description: Some(
"Numeric version selects Trace v3. Legacy v2 envelopes omit this field.".into(),
),
..Default::default()
})),
subschemas: Some(Box::new(schemars::schema::SubschemaValidation {
not: Some(Box::new(
schemars::schema::SchemaObject {
instance_type: Some(schemars::schema::InstanceType::Integer.into()),
..Default::default()
}
.into(),
)),
..Default::default()
})),
..Default::default()
}
.into(),
);
schema
}
impl JsonSchema for TraceV2 {
fn schema_name() -> String {
"TraceV2".into()
}
fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::schema::Schema {
#[derive(JsonSchema)]
#[allow(dead_code)]
struct TraceV2Shape {
/// RFC 3339 timestamp when recording stopped.
recorded_at: String,
started_at: Option<String>,
purpose: Option<String>,
entry: TraceEntry,
pages: Vec<PageRefV2>,
steps: Vec<StepV2>,
}
constrain_trace_v2_schema(TraceV2Shape::json_schema(generator))
}
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn sample_common(id: u32) -> StepCommonV2 {
StepCommonV2 {
id,
page: "p1".into(),
effect: None,
}
}
fn sample_target() -> TargetDescriptorV2 {
TargetDescriptorV2 {
role: Some("button".into()),
name: Some("发布".into()),
tag: "button".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
}
}
#[test]
fn trace_v2_has_no_version_field() {
let trace = TraceV2 {
recorded_at: "2026-07-17T09:01:10Z".into(),
started_at: None,
purpose: None,
entry: TraceEntry {
start_url: "https://example.com/".into(),
},
pages: vec![PageRefV2 {
id: "p1".into(),
url: "https://example.com/".into(),
title: None,
}],
steps: vec![StepV2::Click {
common: sample_common(1),
target: sample_target(),
}],
};
let v = serde_json::to_value(&trace).unwrap();
assert!(v.get("version").is_none());
assert!(v.get("pages").is_some());
assert!(v.get("states").is_none());
}
#[test]
fn extension_v2_trace_deserializes() {
let v = json!({
"recorded_at": "2026-07-21T08:00:00Z",
"started_at": "2026-07-21T07:59:00Z",
"purpose": "demo",
"entry": { "start_url": "https://example.com/editor" },
"pages": [
{ "id": "p1", "url": "https://example.com/editor" },
{ "id": "p2", "url": "https://example.com/p/99" }
],
"steps": [
{
"op": "fill",
"id": 1,
"page": "p1",
"target": { "tag": "input", "role": "textbox", "name": "标题" },
"value": "hello"
},
{
"op": "click",
"id": 2,
"page": "p1",
"target": { "tag": "button", "role": "button", "name": "发布" },
"effect": { "navigated_to": "p2" }
}
]
});
let trace: TraceV2 = serde_json::from_value(v).unwrap();
assert_eq!(trace.pages.len(), 2);
assert_eq!(trace.steps.len(), 2);
}
#[test]
fn v2_hover_steps_round_trip() {
let value = json!({
"recorded_at": "2026-07-21T08:00:00Z",
"entry": { "start_url": "https://example.com/" },
"pages": [{ "id": "p1", "url": "https://example.com/" }],
"steps": [{
"op": "hover",
"id": 1,
"page": "p1",
"target": { "tag": "span", "role": "button", "name": "Account" }
}]
});
let trace: TraceV2 = serde_json::from_value(value).unwrap();
assert!(matches!(trace.steps.as_slice(), [StepV2::Hover { .. }]));
assert_eq!(
serde_json::to_value(&trace).unwrap()["steps"][0]["op"],
"hover"
);
}
#[test]
fn trace_v2_ignores_unknown_extension_fields() {
let value = json!({
"recorded_at": "2026-07-21T08:00:00Z",
"entry": { "start_url": "https://example.com/" },
"pages": [{ "id": "p1", "url": "https://example.com/" }],
"steps": [],
"meta": { "tool": "legacy-exporter" }
});
let trace: TraceV2 = serde_json::from_value(value).unwrap();
assert_eq!(trace.pages.len(), 1);
assert!(trace.steps.is_empty());
}
#[test]
fn trace_v2_schema_allows_additional_properties() {
let schema = serde_json::to_value(schemars::schema_for!(TraceV2)).unwrap();
assert_ne!(
schema.get("additionalProperties"),
Some(&serde_json::Value::Bool(false)),
"Trace v2 must keep accepting traces with unknown extension fields"
);
}
#[test]
fn trace_v2_schema_forbids_classification_keys() {
let schema = serde_json::to_value(schemars::schema_for!(TraceV2)).unwrap();
assert_eq!(
schema["properties"]["states"],
json!(false),
"Trace v2 schema must reject states[] (reserved for v3)"
);
assert_eq!(
schema["properties"]["version"]["not"]["type"], "integer",
"Trace v2 schema must reject a numeric version (that selects the v3 path)"
);
}
}
+428
View File
@@ -0,0 +1,428 @@
//! Trace v3 — state-action-state log with `states[]` and step `state` refs.
//!
//! Wire format with top-level `version: 3`.
use schemars::JsonSchema;
use serde::{Deserialize, Deserializer, Serialize};
use super::interaction::KeyModifier;
use super::record_common::TraceEntry;
pub const TRACE_VERSION_V3: u32 = 3;
pub const VOM_FORMAT_VERSION: u32 = 1;
fn deserialize_trace_v3_version<'de, D>(deserializer: D) -> Result<u32, D::Error>
where
D: Deserializer<'de>,
{
let version = u32::deserialize(deserializer)?;
if version != TRACE_VERSION_V3 {
return Err(serde::de::Error::custom(format!(
"unsupported trace version {version} (expected {TRACE_VERSION_V3})"
)));
}
Ok(version)
}
fn trace_v3_version_schema(_: &mut schemars::r#gen::SchemaGenerator) -> schemars::schema::Schema {
schemars::schema::SchemaObject {
instance_type: Some(schemars::schema::InstanceType::Integer.into()),
const_value: Some(serde_json::json!(TRACE_VERSION_V3)),
..Default::default()
}
.into()
}
/// Stable semantic handle for an interacted element within a page observation.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TargetDescriptorV3 {
#[serde(default, skip_serializing_if = "Option::is_none", rename = "ref")]
pub element_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub role: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ctx: Option<String>,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub unmatched: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct RecorderInfo {
pub bsk: String,
pub vom: u32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum StopReason {
UserFinish,
CliStop,
}
/// Page observation dictionary entry — referenced by steps via `state` / `result.state`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TraceStateV3 {
pub id: String,
pub url: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
/// Full page observation (front matter + VOM body + annotations).
pub body: String,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub truncated: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepResultV3 {
pub state: String,
}
/// Fields shared by every step variant (flattened in JSON).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepCommonV3 {
pub id: u32,
/// Observation id immediately before this action.
pub state: String,
pub result: StepResultV3,
}
/// One selected option (`select` op).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct SelectedOptionV3 {
pub value: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub label: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum NavigationCause {
UserTyped,
Link,
FormSubmit,
Reload,
History,
Script,
Browser,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum FillCommit {
Enter,
Suggestion,
Blur,
}
/// One recorded user action — discriminated union by `op`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[serde(tag = "op", rename_all = "snake_case")]
pub enum StepV3 {
Navigate {
#[serde(flatten)]
common: StepCommonV3,
to: String,
cause: NavigationCause,
},
Click {
#[serde(flatten)]
common: StepCommonV3,
target: TargetDescriptorV3,
},
Hover {
#[serde(flatten)]
common: StepCommonV3,
target: TargetDescriptorV3,
},
Fill {
#[serde(flatten)]
common: StepCommonV3,
target: TargetDescriptorV3,
value: String,
commit: FillCommit,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
redacted: bool,
},
Select {
#[serde(flatten)]
common: StepCommonV3,
target: TargetDescriptorV3,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
selection: Vec<SelectedOptionV3>,
},
Press {
#[serde(flatten)]
common: StepCommonV3,
key: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
modifiers: Option<Vec<KeyModifier>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
target: Option<TargetDescriptorV3>,
},
Scroll {
#[serde(flatten)]
common: StepCommonV3,
},
}
/// Wire trace returned by `tool.record_stop` / `await`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[serde(deny_unknown_fields)]
pub struct TraceV3 {
#[serde(deserialize_with = "deserialize_trace_v3_version")]
#[schemars(schema_with = "trace_v3_version_schema")]
pub version: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub purpose: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub started_at: Option<String>,
pub recorded_at: String,
pub stopped_by: StopReason,
pub entry: TraceEntry,
pub recorder: RecorderInfo,
pub states: Vec<TraceStateV3>,
pub steps: Vec<StepV3>,
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn sample_common(id: u32, state: &str, result_state: &str) -> StepCommonV3 {
StepCommonV3 {
id,
state: state.into(),
result: StepResultV3 {
state: result_state.into(),
},
}
}
fn sample_target() -> TargetDescriptorV3 {
TargetDescriptorV3 {
element_ref: Some("e21".into()),
role: Some("button".into()),
name: Some("发布".into()),
ctx: Some("金桔柠檬 6 号".into()),
unmatched: false,
}
}
fn sample_trace() -> TraceV3 {
TraceV3 {
version: TRACE_VERSION_V3,
purpose: Some("把草稿商品发布上架".into()),
started_at: Some("2026-08-10T02:10:41.080Z".into()),
recorded_at: "2026-08-10T02:12:55.360Z".into(),
stopped_by: StopReason::UserFinish,
entry: TraceEntry {
start_url: "https://example.com/".into(),
},
recorder: RecorderInfo {
bsk: "0.1.10".into(),
vom: VOM_FORMAT_VERSION,
},
states: vec![
TraceStateV3 {
id: "s1".into(),
url: "https://example.com/".into(),
title: Some("Example Domain".into()),
body: "@vom 1\nRootWebArea \"Example Domain\"".into(),
truncated: false,
},
TraceStateV3 {
id: "s2".into(),
url: "https://shop.example.com/products?status=draft".into(),
title: Some("商品管理".into()),
body: "@vom 1\nRootWebArea \"商品管理\"".into(),
truncated: false,
},
],
steps: vec![
StepV3::Navigate {
common: sample_common(1, "s1", "s2"),
to: "https://shop.example.com/products?status=draft".into(),
cause: NavigationCause::UserTyped,
},
StepV3::Fill {
common: sample_common(2, "s2", "s2"),
target: TargetDescriptorV3 {
element_ref: Some("e12".into()),
role: Some("textbox".into()),
name: Some("搜索商品".into()),
ctx: None,
unmatched: false,
},
value: "金桔柠檬".into(),
commit: FillCommit::Enter,
redacted: false,
},
StepV3::Click {
common: sample_common(3, "s2", "s2"),
target: sample_target(),
},
],
}
}
#[test]
fn step_click_round_trips() {
let step = StepV3::Click {
common: sample_common(1, "s1", "s2"),
target: sample_target(),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v.get("op").and_then(|v| v.as_str()), Some("click"));
assert_eq!(v.get("state").and_then(|v| v.as_str()), Some("s1"));
assert_eq!(v["result"]["state"], "s2");
assert_eq!(v["target"]["ref"], "e21");
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_fill_with_commit_round_trips() {
let step = StepV3::Fill {
common: sample_common(2, "s2", "s3"),
target: TargetDescriptorV3 {
element_ref: Some("e12".into()),
role: Some("textbox".into()),
name: Some("搜索商品".into()),
ctx: None,
unmatched: false,
},
value: "browser skill".into(),
commit: FillCommit::Enter,
redacted: false,
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "fill");
assert_eq!(v["commit"], "enter");
assert!(v.get("redacted").is_none());
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_fill_password_is_redacted() {
let step = StepV3::Fill {
common: sample_common(1, "s1", "s1"),
target: TargetDescriptorV3 {
element_ref: Some("e3".into()),
role: Some("textbox".into()),
name: Some("密码".into()),
ctx: None,
unmatched: false,
},
value: "***".into(),
commit: FillCommit::Blur,
redacted: true,
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["value"], "***");
assert_eq!(v["redacted"], true);
}
#[test]
fn step_navigate_with_cause_round_trips() {
let step = StepV3::Navigate {
common: sample_common(1, "s1", "s2"),
to: "https://example.com".into(),
cause: NavigationCause::UserTyped,
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "navigate");
assert_eq!(v["cause"], "user_typed");
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn trace_v3_round_trips() {
let trace = sample_trace();
let v = serde_json::to_value(&trace).unwrap();
assert_eq!(v.get("version").and_then(|v| v.as_u64()), Some(3));
assert!(v.get("pages").is_none());
assert_eq!(v["recorder"]["vom"], 1);
assert_eq!(v["states"].as_array().unwrap().len(), 2);
let round: TraceV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, trace);
}
#[test]
fn default_fields_are_omitted() {
let step = StepV3::Click {
common: sample_common(1, "s1", "s1"),
target: TargetDescriptorV3 {
element_ref: Some("e1".into()),
role: Some("button".into()),
name: Some("OK".into()),
ctx: None,
unmatched: false,
},
};
let v = serde_json::to_value(&step).unwrap();
assert!(v.get("unmatched").is_none());
assert!(v["target"].get("ctx").is_none());
assert!(v["target"].get("unmatched").is_none());
}
#[test]
fn unmatched_target_serializes_flag() {
let step = StepV3::Click {
common: sample_common(1, "s1", "s2"),
target: TargetDescriptorV3 {
element_ref: None,
role: Some("button".into()),
name: Some("发布".into()),
ctx: None,
unmatched: true,
},
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["target"]["unmatched"], true);
assert!(v["target"].get("ref").is_none());
}
#[test]
fn extension_trace_deserializes() {
let v = json!({
"version": 3,
"recorded_at": "2026-07-21T08:00:00Z",
"started_at": "2026-07-21T07:59:00Z",
"purpose": "demo",
"stopped_by": "user_finish",
"entry": { "start_url": "https://example.com/editor" },
"recorder": { "bsk": "0.1.10", "vom": 1 },
"states": [
{ "id": "s1", "url": "https://example.com/editor", "body": "@vom 1" },
{ "id": "s2", "url": "https://example.com/p/99", "body": "@vom 1" }
],
"steps": [
{
"op": "fill",
"id": 1,
"state": "s1",
"result": { "state": "s1" },
"target": { "ref": "e1", "role": "textbox", "name": "标题" },
"value": "hello",
"commit": "blur"
},
{
"op": "click",
"id": 2,
"state": "s1",
"result": { "state": "s2" },
"target": { "ref": "e2", "role": "button", "name": "发布" }
}
]
});
let trace: TraceV3 = serde_json::from_value(v).unwrap();
assert_eq!(trace.version, 3);
assert_eq!(trace.states.len(), 2);
assert_eq!(trace.steps.len(), 2);
}
}
+10 -31
View File
@@ -20,43 +20,22 @@ extension keep owning the actual browser control — this package is a thin, wel
| `browser_snapshot` | `bsk snapshot` | Indented aria-tree snapshot with `@eN` refs for interaction tools. |
| `browser_observe` | `bsk observe` | Semantic VOM observation (read-only) with `@eN` refs. |
| `browser_click` | `bsk click` | Click a snapshot ref or CSS selector (button / click-count options). |
| `browser_hover` | `bsk hover` | Hover a ref or selector and allow hover-triggered UI to settle. |
| `browser_fill` | `bsk fill` | Fill an input / textarea / contenteditable (clears first by default). |
| `browser_select` | `bsk select` | Select one or more option values on a `<select>`. |
| `browser_press` | `bsk press` | Dispatch a key or combo, optionally focusing a target first. |
| `browser_screenshot` | `bsk screenshot` | PNG capture of the tab or a ref-cropped element; inlines the image when the deployment supports image input, otherwise returns a file path. |
| `browser_emulate` | `bsk emulate` | Apply or clear mobile device emulation on the active tab. |
| `browser_tab_list` | `bsk tab list` | List user and/or Agent Window tabs visible to an owned session. |
| `browser_tab_create` | `bsk tab create` | Create a foreground or background tab in the Agent Window. |
| `browser_tab_select` | `bsk tab select` | Focus an Agent Window tab. |
| `browser_tab_close` | `bsk tab close` | Close an Agent Window tab. |
| `browser_tab_borrow` | `bsk tab borrow` | Move a user tab into the Agent Window for controlled interaction. |
| `browser_tab_return` | `bsk tab return` | Return a borrowed tab to its original user window. |
| `browser_navigate_back` / `browser_navigate_forward` | `bsk navigate-back` / `bsk navigate-forward` | Traverse tab history and wait for a lifecycle phase. |
| `browser_reload` | `bsk reload` | Reload the active tab, optionally bypassing cache. |
| `browser_wait_for_navigation` | `bsk wait-for-navigation` | Wait explicitly for a page lifecycle event. |
| `browser_request_help` | `bsk request-help` | Pause for login, captcha, OTP, confirmation, or another human browser step. |
| `browser_get_html` | `bsk get-html` | Read raw document or ref-scoped HTML after semantic observation is insufficient. |
| `browser_console` | `bsk console` | Read buffered console/log/exception entries without evaluating JavaScript. |
| `browser_network` | `bsk network` | Read buffered network response/failure metadata. |
| `browser_window_resize` | `bsk window resize` | Resize the owned session's Agent Window. |
`bsk evaluate` and `bsk record` are intentionally not exposed by this plugin: arbitrary page
evaluation is a higher-risk capability, while recording is long-running and needs a dedicated
DeepSeek Harness lifecycle before it can be added safely.
## Agent skill (progressive disclosure)
Beyond the tools, the plugin publishes the **`browser-skill` agent skill** through the harness's
official skill seam (`ctx.skills.register`): the catalog entry (name + routing description) is
resident in `<available_skills>`, and the body is loaded only when the model invokes the `skill`
tool. Its single source is the DSH-specific `skill/SKILL.md`, which documents only structured
`browser_*` calls and their plugin semantics. The repository-root CLI skill is intentionally not
concatenated: its command examples belong to a different execution interface and would bypass
the plugin's ownership, live observation UI, cancellation, and cleanup path if followed directly.
The build rejects internal CLI-name leakage, command-line code blocks, unknown browser tools, and
missing supported browser tools before embedding the Markdown. Registration and every pre-step
catalog snapshot are pure in-memory reads (no disk/process/daemon); compositions without the skill
tool. The body is assembled at build time from two parts, so there is exactly one source of
truth: a dsh-specific prelude (`skill/prelude.md` — tool↔CLI map, owned-session semantics,
plugin-only overrides) followed verbatim by the canonical CLI skill (`skill/SKILL.md` at the
repo root — the same file `crates/bsk-cli/build.rs` mirrors for the CLI package; workflows,
stop-when-done rules, refs usage, sandbox rules). Registration and every pre-step catalog
snapshot are pure in-memory reads (no disk/process/daemon); compositions without the skill
seam degrade silently.
## Multi-session model
@@ -109,7 +88,7 @@ All fields are optional and validated through the plugin's Schemastery `Config`:
# lazyTools: true # reveal browser_* tools only after the skill is invoked
```
- **`lazyTools` (default `true`)** — the final progressive-disclosure stage: the 28
- **`lazyTools` (default `true`)** — the final progressive-disclosure stage: the eleven
`browser_*` tool schemas stay OUT of the system prompt (zero schema tokens) until the
`browser-skill` skill is actually invoked — the skill catalog entry is the only
advertisement. One successful invocation (model tool call, or a `/browser-skill` user
@@ -173,7 +152,7 @@ shell's theme cannot bleed back in.
card). Screenshots additionally attach the image itself when the host mounts an attachment store
and the active model route declares image input; otherwise the PNG path is returned.
- **Web UI toolview (browser half)**: the package is dual-face. `dsh.client` (platform `web`) ships
`lib/client.cjs`, which registers a keyed `tool.call.toolview` view for `browser_screenshot`. The
`lib/client.js`, which registers a keyed `tool.call.toolview` view for `browser_screenshot`. The
custom view keeps the terminal block (command + output) and, when the settled result carries an
image block, resolves the durable attachment through the client session's authorized
`readAttachment` RPC and renders it with the shared `MessageImage` thumbnail/lightbox atoms.
@@ -204,8 +183,8 @@ as `@wxg-prc-cpg/browser-skill-dsh-plugin`. The Cordis plugin id stays
Trigger it by pushing a tag that matches `package.json`'s `version`:
```sh
git tag dsh-plugin-v0.1.2
git push origin dsh-plugin-v0.1.2
git tag dsh-plugin-v0.1.0
git push origin dsh-plugin-v0.1.0
```
Or run the workflow from the Actions tab (`workflow_dispatch`). The job reads the
@@ -1,7 +1,7 @@
{
"name": "@wxg-prc-cpg/browser-skill-dsh-plugin",
"description": "DeepSeek Harness tool plugin that exposes BrowserSkill browser automation (browser_* tools) to the model",
"version": "0.1.2",
"description": "DeepSeek Harness tool plugin that exposes BrowserSkill (bsk) browser automation to the model",
"version": "0.1.1",
"license": "MIT",
"publishConfig": {
"access": "public",
@@ -1,15 +1,20 @@
// Embed the DeepSeek Harness-specific browser skill at build time. The model-facing
// skill deliberately documents only injected browser_* tools; the repository's CLI
// skill is a separate interface and must never be concatenated here.
// Assemble the plugin's skill content: a dsh-specific routing prelude, the
// canonical CLI skill body verbatim (repo-root skill/SKILL.md — the same source
// crates/bsk-cli's build.rs mirrors), then a dsh routing reminder postlude. The
// original's YAML frontmatter is stripped: name/description are supplied by the
// registration.
// Output: src/skill-content.generated.ts (gitignored, built before tsdown and
// before vitest), so registration is a pure in-memory read.
// before vitest) so the markdown ships inlined in the bundle and registration
// is a pure in-memory read — never a per-step disk access.
import { readFileSync, writeFileSync } from "node:fs";
import { dirname, join } from "node:path";
import { fileURLToPath } from "node:url";
const here = dirname(fileURLToPath(import.meta.url));
const pkg = join(here, "..");
const source = readFileSync(join(pkg, "skill", "SKILL.md"), "utf8");
const prelude = readFileSync(join(pkg, "skill", "prelude.md"), "utf8");
const upstream = readFileSync(join(pkg, "..", "..", "skill", "SKILL.md"), "utf8");
const postlude = readFileSync(join(pkg, "skill", "postlude.md"), "utf8");
function stripFrontmatter(markdown) {
if (!markdown.startsWith("---\n")) return markdown;
@@ -17,69 +22,18 @@ function stripFrontmatter(markdown) {
return end === -1 ? markdown : markdown.slice(end + 5).replace(/^\s*\n/, "");
}
const SUPPORTED_BROWSER_TOOLS = [
"browser_session_start",
"browser_session_stop",
"browser_session_list",
"browser_navigate",
"browser_navigate_back",
"browser_navigate_forward",
"browser_reload",
"browser_wait_for_navigation",
"browser_snapshot",
"browser_observe",
"browser_get_html",
"browser_screenshot",
"browser_console",
"browser_network",
"browser_click",
"browser_hover",
"browser_fill",
"browser_select",
"browser_press",
"browser_tab_list",
"browser_tab_create",
"browser_tab_select",
"browser_tab_close",
"browser_tab_borrow",
"browser_tab_return",
"browser_request_help",
"browser_emulate",
"browser_window_resize",
];
function validateModelFacingSkill(markdown) {
if (/\bbsk\b/i.test(markdown)) {
throw new Error("DSH skill must not expose the internal CLI name");
}
if (/```(?:bash|sh|shell)\b/i.test(markdown)) {
throw new Error("DSH skill must not contain command-line code blocks");
}
const mentioned = new Set(markdown.match(/\bbrowser_[a-z][a-z_]*\b/g) ?? []);
const supported = new Set(SUPPORTED_BROWSER_TOOLS);
const unknown = [...mentioned].filter((name) => !supported.has(name));
const missing = SUPPORTED_BROWSER_TOOLS.filter((name) => !mentioned.has(name));
if (unknown.length > 0) {
throw new Error(`DSH skill mentions unsupported browser tools: ${unknown.join(", ")}`);
}
if (missing.length > 0) {
throw new Error(`DSH skill omits supported browser tools: ${missing.join(", ")}`);
}
}
const NAME = "browser-skill";
const DESCRIPTION =
"Browser automation against the user's logged-in Chromium through this plugin's injected " +
"browser_* tools. Use to visit and read pages, fill forms, click through flows, inspect tabs, " +
"debug page activity, or smoke-test UI changes in a managed Agent Window.";
"Browser automation against the user's logged-in Chromium through this plugin's injected browser_* tools; never invoke bsk through bash or shell. " +
"tools (bsk engine): visit and read pages, fill forms, click through flows, scrape data, " +
"smoke-test UI changes. Use when a task needs a real browser; requires the bsk CLI and the " +
"browser-skill extension connected.";
const content = `${stripFrontmatter(source).trim()}\n`;
validateModelFacingSkill(content);
const content = `${prelude.trim()}\n\n${stripFrontmatter(upstream).trim()}\n\n${postlude.trim()}\n`;
const banner =
"// GENERATED by scripts/build-skill-content.mjs — do not edit.\n" +
"// Source: skill/SKILL.md (DeepSeek Harness browser_* tool instructions).\n";
"// Sources: skill/prelude.md + ../../../skill/SKILL.md + skill/postlude.md.\n";
writeFileSync(
join(pkg, "src", "skill-content.generated.ts"),
@@ -1,271 +0,0 @@
---
name: browser-skill
description: |
Use the injected browser_* tools to automate the user's logged-in Chromium
browser in a managed Agent Window.
---
# browser-skill for DeepSeek Harness
Drive the user's real Chromium browser through the structured `browser_*` tools provided by this
plugin. The browser keeps the user's existing logins and cookies while isolating automation in an
Agent Window. Use the tool schemas exactly as exposed by DeepSeek Harness; arguments are structured
fields, not command-line flags.
## Tool availability
Loading this skill reveals the browser tools for the rest of the conversation. If the tools were
hidden before this skill was loaded, start browser work only after the skill invocation succeeds.
All browser operations must use the injected tools directly. Do not translate the examples in this
document into terminal commands or invoke another process to control the browser. Direct tool calls
preserve session ownership, the live observation window, tool cards, cancellation, and cleanup.
## When to use
- Visit and read pages that require the user's existing browser session.
- Fill forms, select options, click through bounded flows, or smoke-test a web UI.
- Inspect tabs, page structure, screenshots, console messages, or network metadata.
- Work with a user tab after explicitly borrowing it into the Agent Window.
Do not use browser automation for tasks that need only local files, a purpose-built API, or an
explanation. Never extract credentials, cookies, tokens, or other secrets. Do not keep control of a
personal tab longer than the immediate task requires.
## Mandatory workflow
Every browser task follows this lifecycle:
1. Call `browser_session_start` and retain its returned `sessionId`.
2. Use that session for every browser operation. Pass `session` explicitly when more than one
session exists.
3. Call `browser_session_stop` when the bounded goal is reached or an error ends the task. Treat
cleanup like a `finally` path unless the user explicitly asks to keep the session open.
Conceptual tool-call sequence:
```text
browser_session_start({ url: "https://example.com" })
browser_observe({ session: sessionId })
browser_click({ session: sessionId, target: "@e4" })
browser_observe({ session: sessionId })
browser_session_stop({ session: sessionId })
```
The plugin owns only sessions created by `browser_session_start`. `browser_session_list` reports
only those owned sessions. Never guess or reuse a session id from another program. When `session`
is omitted, most tools use the most recently started or used owned session; explicit ids are safer
for multi-session work.
## Stop when the goal is met
Browser work is a bounded task, not open-ended browsing.
1. Define one observable success condition from the user's request.
2. Take the shortest path: observe, act, then perform at most one relevant check.
3. Stop immediately when success is visible. Do not refresh, navigate, click, or perform extra
verification after success.
4. If a human-only step appears, use `browser_request_help`. If an action fails twice without
progress, pause instead of brute-forcing.
5. When uncertain, make at most one fresh observation. If the state is still ambiguous, ask the
user rather than continuing to click.
## Core interaction loop
Use `browser_navigate` to open a destination in the active Agent Window tab, then inspect the page
before interacting:
1. `browser_observe` is the primary semantic view. It returns roles, states, text, and `@eN` refs,
including hints about conditional hover or focus surfaces.
2. `browser_snapshot` is the static accessibility-tree fallback when the semantic view is
insufficient or a stricter tree is useful.
3. Prefer a fresh `@eN` ref over a raw CSS selector for `browser_click`, `browser_hover`,
`browser_fill`, `browser_select`, and targeted `browser_press` calls.
4. Observe again after navigation or a meaningful DOM change before using another ref.
Refs invalidate after navigation and may also become stale after large DOM changes. Never carry a
ref to a new page. If an observation marks a control as hover-triggered, call `browser_hover` on the
trigger, observe again, and then act on the newly visible ref.
Use interaction tools according to their schemas:
- `browser_click`: click a ref or selector; optionally choose mouse button and click count.
- `browser_hover`: move to a ref or selector and allow hover UI to settle; then observe again.
- `browser_fill`: clear and fill a text-capable element; set `noClear` only when appending is
intentional.
- `browser_select`: pass `values` as an array, including for a single option.
- `browser_press`: send a key or combo, optionally focusing `target` first.
Actions may trigger a navigation after the action itself returns. Use
`browser_wait_for_navigation` only when that lifecycle wait is actually needed, then observe the
new page.
## Observation and page reading priority
Escalate page reading only as needed:
1. `browser_observe` for normal semantic understanding and interaction refs.
2. `browser_snapshot` for a stricter static accessibility tree.
3. `browser_get_html` for hidden DOM, metadata, or exact markup that the semantic views cannot
provide. A `ref` must come from a fresh observation; keep `maxBytes` bounded.
4. `browser_screenshot` for layout, styling, canvas, image content, or another genuinely visual
question. Pass a fresh `ref` to crop to one element, or omit it for the visible tab.
Do not start with raw HTML or a screenshot merely to discover ordinary controls. Screenshots may
return an inline image or a saved path depending on the deployment.
## Tabs and the Agent Window boundary
Write actions normally affect only Agent Window tabs.
- `browser_tab_list` lists `agent`, `user`, or `all` tabs visible to the owned session. Use
`scope: "user"` to locate a user tab before borrowing it.
- `browser_tab_create` creates an Agent Window tab. It is active by default; set `active: false`
for a background tab.
- `browser_tab_select` focuses an Agent Window tab.
- `browser_tab_close` closes an Agent Window tab.
- `browser_tab_borrow` moves a chosen user tab into the Agent Window before any write action.
- `browser_tab_return` returns a borrowed tab to its original window and position. Return it as
soon as the immediate task is finished; stopping the session is only the fallback cleanup.
Use only a `tabId` returned by `browser_tab_list` or `browser_tab_create`. Do not assume a tab id
from page position or title.
## Navigation and lifecycle
- `browser_navigate` opens a URL and can wait for a requested lifecycle phase.
- `browser_navigate_back` and `browser_navigate_forward` move one history entry.
- `browser_reload` reloads the current tab; use `hard: true` only when bypassing cache is necessary.
- `browser_wait_for_navigation` waits for a lifecycle event after an action that did not already
wait. Avoid speculative waits when no navigation is expected.
After navigation, history traversal, or reload, discard old refs and call `browser_observe` or
`browser_snapshot` again.
## Ask the user for browser help
Call `browser_request_help` for login, captcha, OTP, payment confirmation, consent, or another step
that requires the user. Give a clear `prompt`; when the prompt refers to a concrete control, pass
its fresh ref or selector in `targets` so the overlay can highlight it. Use `title` only when it
adds clarity.
Optional `completionCriteria` can finish the pause automatically when the page has an explicit,
stable success signal. It supports `any` or `all` conditions using `urlContains`, `urlMatches`,
`selectorExists`, `selectorMissing`, `textExists`, or `textMissing`, plus `stableForMs`.
```text
browser_request_help({
session: sessionId,
title: "Sign in required",
prompt: "Complete sign-in, then return control after the account page appears.",
targets: ["@e7"],
timeoutMs: 300000,
completionCriteria: {
any: [{ urlContains: "/account" }, { selectorExists: "[data-testid='account-menu']" }],
stableForMs: 1000
}
})
```
Interpret outcomes deliberately:
- `continued` or `completed`: the user returned control or the explicit detector matched. Observe
again before reasoning about the new state or using refs.
- `cancelled`: stop the requested flow.
- `timed_out`: report the timeout; do not repeatedly reopen the prompt.
- `disabled`: unattended mode does not allow help; proceed only if safe or stop gracefully.
- `navigated`: legacy outcome; do not treat navigation alone as proof of success.
## Debugging without page scripting
Use the read-only debugging tools only when relevant to the user's task:
- `browser_console` reads buffered console messages, browser logs, and JavaScript exceptions.
Set `includeStack: true` when stack frames are useful.
- `browser_network` reads buffered response and failure metadata. It does not return request or
response headers or bodies and does not modify traffic.
Both tools accept `tabId`, `since`, `limit`, and `maxTextChars`. Their result returns `nextSince`;
pass that value as the next `since` cursor instead of repeatedly requesting the same entries.
Arbitrary page script evaluation and interaction recording are intentionally unsupported in this
DeepSeek Harness plugin. Do not invent tool names or route around this limitation. If the task
strictly requires either capability, explain that it is not available in this phase.
## Window and device controls
- `browser_window_resize` changes the Agent Window's outer `width` and `height`; each dimension
must be from 100 through 7680 CSS pixels.
- `browser_emulate` applies a built-in device preset or explicit viewport settings to the active
tab. Explicit `width` and `height` must be supplied together. `mobile: true` also requires both
dimensions. Use `off: true` alone to clear emulation. Emulation is per-tab and new tabs do not
inherit it.
## Tool reference
### Sessions
| Tool | Purpose |
| --- | --- |
| `browser_session_start` | Start an owned Agent Window session, optionally with URL, size, browser instance, focus behavior, or device preset. |
| `browser_session_stop` | Stop an owned session and close its Agent Window. |
| `browser_session_list` | List owned sessions and identify the current one. |
### Navigation and reading
| Tool | Purpose |
| --- | --- |
| `browser_navigate` | Navigate the active tab to a URL. |
| `browser_navigate_back` | Go back one history entry. |
| `browser_navigate_forward` | Go forward one history entry. |
| `browser_reload` | Reload the active tab, optionally bypassing cache. |
| `browser_wait_for_navigation` | Wait for an expected page lifecycle event. |
| `browser_observe` | Read the primary semantic page representation with refs. |
| `browser_snapshot` | Read the static accessibility-tree fallback with refs. |
| `browser_get_html` | Read raw document or ref-scoped HTML as a last resort. |
| `browser_screenshot` | Capture the visible tab or a ref-cropped element. |
### Interaction
| Tool | Purpose |
| --- | --- |
| `browser_click` | Click a ref or CSS selector. |
| `browser_hover` | Hover a ref or selector to reveal conditional UI. |
| `browser_fill` | Fill a text-capable element. |
| `browser_select` | Select one or more option values. |
| `browser_press` | Send a keyboard key or combo. |
### Tabs
| Tool | Purpose |
| --- | --- |
| `browser_tab_list` | List visible user and Agent Window tabs. |
| `browser_tab_create` | Create an Agent Window tab. |
| `browser_tab_select` | Focus an Agent Window tab. |
| `browser_tab_close` | Close an Agent Window tab. |
| `browser_tab_borrow` | Move a user tab into the Agent Window. |
| `browser_tab_return` | Return a borrowed tab to its original window. |
### Assistance, diagnostics, and display
| Tool | Purpose |
| --- | --- |
| `browser_request_help` | Pause for an explicit user browser step. |
| `browser_console` | Read buffered console and exception entries. |
| `browser_network` | Read buffered response and failure metadata. |
| `browser_window_resize` | Resize the Agent Window. |
| `browser_emulate` | Apply or clear per-tab device emulation. |
## Error handling and final checklist
On a stale ref, observe again and retry once with a fresh ref. On a closed or unknown tab, list
tabs instead of guessing. On an unknown session, list owned sessions or start a new one; never try
foreign ids. On timeout, decide whether one longer, purposeful wait is justified. Avoid repeated
actions with unchanged state.
Before finishing a browser task, verify that:
- The user's observable goal is met, or the blocker is clearly reported.
- Any borrowed tab has been returned when practical.
- The owned session has been stopped unless the user explicitly asked to keep it open.
- No further page action is taken after success.
@@ -0,0 +1,10 @@
---
## DSH routing reminder
The CLI-oriented reference above describes behavior, not the execution mechanism in
DeepSeek Harness. For every browser operation, call the injected `browser_*` model
tool. Do not translate these examples into `bash` commands and do not execute `bsk`
directly; the plugin invokes `bsk` internally. In particular, always begin with
`browser_session_start`, pass its returned session id to subsequent `browser_*`
tools, and finish with `browser_session_stop`.
@@ -0,0 +1,71 @@
# browser-skill (dsh edition)
## DSH tool-routing requirement (highest priority)
This skill is running inside DeepSeek Harness, where the plugin has injected
structured `browser_*` model tools. **Always call those injected `browser_*` tools
for browser work. Never use `bash`, `shell`, `exec`, or another tool to invoke the
`bsk` CLI directly.** The CLI command names shown below are only conceptual
mappings for understanding the tools; they are not instructions to run commands.
If a `browser_*` tool is available, using the CLI directly is incorrect because it
bypasses plugin session ownership, observation UI, tool cards, and cleanup.
Use the injected tool whose name matches the operation, for example:
`browser_session_start`, `browser_navigate`, `browser_observe`, `browser_snapshot`,
`browser_click`, `browser_fill`, `browser_press`, `browser_screenshot`, and
`browser_session_stop`. Pass arguments as the tool schema specifies (not CLI flags).
Drive the user's **real Chromium browser** (logins, cookies, Agent Window isolation)
through this plugin's `browser_*` tools. Each injected tool is implemented by the
plugin using the `bsk` engine internally — the model must not invoke that engine
itself. The workflow, judgment rules, and safety constraints remain identical to
the canonical CLI skill.
> **Tool availability**: invoking this skill (which you just did) makes the plugin's
> `browser_*` tools available for the rest of the session — they are injected on
> successful skill load. Until then they are deliberately hidden, so do not attempt
> a browser action before loading this skill.
## Tool ↔ CLI map
The tools below are thin, structured wrappers over the bsk CLI verbs of the same
name; `session` parameters take the 4-letter session id (no `--session` flag
spelling here):
| dsh tool | bsk equivalent |
|---|---|
| `browser_session_start` / `browser_session_stop` / `browser_session_list` | `bsk session start/stop/list` |
| `browser_navigate` | `bsk navigate` |
| `browser_snapshot` / `browser_observe` | `bsk snapshot` / `bsk observe` |
| `browser_click` / `browser_fill` / `browser_press` | `bsk click` / `bsk fill` / `bsk press` |
| `browser_screenshot` | `bsk screenshot` |
| `browser_emulate` | `bsk emulate` |
(`bsk console` / `bsk network` have no tool counterpart in this plugin yet.)
Plugin-specific semantics that override or narrow the CLI rules below:
- **Owned sessions only**: this plugin sees and acts on just the sessions IT
started via `browser_session_start`. `browser_session_list` shows only those
(with the current marker); passing a session id created by another program
(or person) sharing the bsk daemon is refused outright — never retry those
with the same id.
- **Current session**: omitted `session` parameters resolve to the most
recently started or used OWNED session. With several sessions in flight,
pass `session` explicitly to avoid surprises.
- **Screenshots**: when the model route accepts images, `browser_screenshot`
returns the PNG inline as an image attachment; otherwise it returns the
saved file path.
- **`browser_emulate mobile`**: requires `width`+`height` — the bsk daemon
refuses `--mobile` without viewport dimensions.
- **Live observation**: in the dsh Web UI, every owned session is watched by
the floating overlay (thumbnails, action, interrupt) — observational side
traffic is free and never steals your command turn; just keep acting.
The canonical CLI skill body follows verbatim (source of truth: `skill/SKILL.md`
in the BrowserSkill repo, mirrored by build into `crates/bsk-cli/skill/SKILL.md`).
Read `bsk <verb> --session <id>` as "the `browser_<verb>` tool with
`session: <id>`"; everything else — when to use, the stop-when-done rules, the
interaction loop, refs, sandbox rules — applies unchanged.
---
@@ -1,6 +1,6 @@
/**
* Lazy tool-schema injection ("progressive disclosure", final stage): with
* `lazyTools` on (the default), the 28 browser_* tool schemas stay OUT of the
* `lazyTools` on (the default), the 11 browser_* tool schemas stay OUT of the
* system prompt until the `browser-skill` skill has actually been invoked —
* the skill catalog entry is the only advertisement. One successful
* invocation (model tool call, or a user's `/browser-skill` gesture) reveals
@@ -96,7 +96,7 @@ export function hasSuccessfulSkillInvocation(events: readonly SessionEventLike[]
/**
* Arm the lazy reveal. Returns a disposer tearing down listeners and — when
* the reveal already happened — the tool suite itself.
* @param registerSuite - registers the 28 browser_* tools, returns their disposer.
* @param registerSuite - registers the 11 browser_* tools, returns their disposer.
*/
export function armLazyTools(ctx: Context, registerSuite: () => () => void): () => void {
let suiteDisposer: (() => void) | undefined;
@@ -400,48 +400,14 @@ export function actionForLabel(label: string): string {
return "observing";
case "click":
return "clicking";
case "hover":
return "hovering";
case "fill":
return "filling";
case "select":
return "selecting";
case "press":
return "pressing";
case "screenshot":
return "capturing";
case "emulate":
return "emulating";
case "tab list":
return "listing tabs";
case "tab create":
return "creating tab";
case "tab close":
return "closing tab";
case "tab select":
return "selecting tab";
case "tab borrow":
return "borrowing tab";
case "tab return":
return "returning tab";
case "navigate-back":
return "navigating back";
case "navigate-forward":
return "navigating forward";
case "reload":
return "reloading";
case "wait-for-navigation":
return "waiting for navigation";
case "request-help":
return "waiting for user";
case "get-html":
return "reading HTML";
case "console":
return "reading console";
case "network":
return "reading network";
case "window resize":
return "resizing window";
default:
return label;
}
@@ -1,58 +0,0 @@
import type { ToolDefinition, ToolResult, ToolRunContext } from "@deepseek-ai/dsh-tools";
import type { ToolDeps } from "./tools";
export type ToolRegistrar = (definition: ToolDefinition) => () => void;
type TerminalPresentation = { card: "terminal"; output: string; exitCode: number } | undefined;
/** Existing tool runtime seams reused by the phase-one capability modules. */
export interface PhaseOneRuntime {
run(
exec: ToolRunContext,
args: string[],
label: string,
observeSession?: string,
runnerTimeoutMs?: number,
): Promise<unknown>;
commandLine(args: string[]): string;
presentTerminalResult: (_args: never, result: ToolResult) => TerminalPresentation;
}
export function requireNonEmpty(value: string, name: string): void {
if (value.trim().length === 0) throw new Error(`${name} must be a non-empty string`);
}
export function requirePositive(value: number | undefined, name: string): void {
if (value !== undefined && value <= 0) throw new Error(`${name} must be greater than zero`);
}
/** CLI positional target detection is shared by hover/select/request-help. */
export function isSnapshotRef(target: string): boolean {
return /^@?e\d+$/.test(target);
}
/**
* Give the child enough time to honour a command-level timeout plus IPC
* settlement slack, without shortening the plugin's configured default.
*/
export function runnerTimeout(deps: ToolDeps, commandTimeoutMs: number | undefined): number {
if (commandTimeoutMs === undefined) return deps.config.defaultTimeoutMs;
return Math.max(deps.config.defaultTimeoutMs, commandTimeoutMs + 15_000);
}
export function appendTarget(args: string[], target: string): void {
args.push(target);
}
export function appendTabId(args: string[], tabId: number | undefined): void {
if (tabId !== undefined) args.push("--tab-id", String(tabId));
}
export function appendWaitOptions(
args: string[],
waitUntil: string | undefined,
timeoutMs: number | undefined,
): void {
if (waitUntil !== undefined) args.push("--wait-until", waitUntil);
if (timeoutMs !== undefined) args.push("--timeout", `${timeoutMs}ms`);
}
@@ -1,197 +0,0 @@
import { defineTool } from "@deepseek-ai/dsh-tools";
import {
appendTabId,
appendTarget,
type PhaseOneRuntime,
requireNonEmpty,
requirePositive,
runnerTimeout,
type ToolRegistrar,
} from "./phase-one-runtime";
import { SESSION_PARAM, TAB_ID_PARAM, TIMEOUT_MS_PARAM } from "./tool-params";
import type { ToolDeps } from "./tools";
const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const;
/** Add hover and select without bypassing session ownership or observation. */
export function registerPhaseOneInteractionTools(
deps: ToolDeps,
register: ToolRegistrar,
runtime: PhaseOneRuntime,
): void {
const { registry } = deps;
register(
defineTool({
name: "browser_hover",
description:
"Move the mouse over a snapshot ref or CSS selector to reveal hover-triggered UI. " +
"Run browser_observe or browser_snapshot again afterwards to discover newly visible refs.",
parameters: {
target: {
type: "string",
required: true,
description: "Snapshot ref (@e3 / e3) or CSS selector of the element to hover.",
},
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
modifiers: {
type: "array",
items: { type: "string", enum: MODIFIERS },
description: "Keyboard modifiers held during the mouse move.",
},
settleMs: {
type: "integer",
description: "Milliseconds to wait for hover-triggered UI to settle (default: 200).",
},
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
x: { type: "number", required: true },
y: { type: "number", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text: `[session ${value.session}] hovered at (${value.x}, ${value.y}) on tab ${value.tabId}`,
},
],
},
async execute(args, exec) {
requireNonEmpty(args.target, "target");
requirePositive(args.timeoutMs, "timeoutMs");
requirePositive(args.settleMs, "settleMs");
const sessionId = registry.resolve(args.session, "browser_hover");
const cmdArgs = ["hover", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.modifiers !== undefined && args.modifiers.length > 0) {
cmdArgs.push("--modifiers", args.modifiers.join(","));
}
if (args.settleMs !== undefined) cmdArgs.push("--settle", `${args.settleMs}ms`);
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
appendTarget(cmdArgs, args.target);
const reply = (await runtime.run(
exec,
cmdArgs,
"hover",
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as { tab_id: number; x: number; y: number };
return { session: sessionId, tabId: reply.tab_id, x: reply.x, y: reply.y };
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"hover",
args.target,
"--session",
args.session ?? "(current)",
]),
description: "Hover an element",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_select",
description:
"Set a select element's option values. Pass one value for a normal select or multiple " +
"values for a multi-select; values replace the current selection.",
parameters: {
target: {
type: "string",
required: true,
description: "Snapshot ref (@e3 / e3) or CSS selector of the select element.",
},
values: {
type: "array",
required: true,
items: { type: "string" },
description: "Option value attributes to select; at least one value is required.",
},
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
multiple: { type: "boolean", required: true },
selectedValues: {
type: "array",
required: true,
items: { type: "string" },
},
selectedLabels: {
type: "array",
required: true,
items: { type: "string" },
},
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] selected ` +
`${value.selectedValues.join(", ") || "(none)"} on tab ${value.tabId}`,
},
],
},
async execute(args, exec) {
requireNonEmpty(args.target, "target");
requirePositive(args.timeoutMs, "timeoutMs");
if (args.values.length === 0) throw new Error("values must contain at least one option");
const sessionId = registry.resolve(args.session, "browser_select");
const cmdArgs = ["select", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
for (const value of args.values) cmdArgs.push("--value", value);
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
appendTarget(cmdArgs, args.target);
const reply = (await runtime.run(
exec,
cmdArgs,
"select",
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as {
tab_id: number;
multiple: boolean;
selected_values: string[];
selected_labels: string[];
};
return {
session: sessionId,
tabId: reply.tab_id,
multiple: reply.multiple,
selectedValues: reply.selected_values,
selectedLabels: reply.selected_labels,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"select",
args.target,
"--session",
args.session ?? "(current)",
]),
description: `Select ${args.values.length} option${args.values.length === 1 ? "" : "s"}`,
}),
presentResult: runtime.presentTerminalResult,
}),
);
}
@@ -1,230 +0,0 @@
import { defineTool } from "@deepseek-ai/dsh-tools";
import {
appendTabId,
appendWaitOptions,
type PhaseOneRuntime,
requirePositive,
runnerTimeout,
type ToolRegistrar,
} from "./phase-one-runtime";
import { SESSION_PARAM, TAB_ID_PARAM, TIMEOUT_MS_PARAM, WAIT_UNTIL_PARAM } from "./tool-params";
import type { ToolDeps } from "./tools";
interface RawHistoryResult {
tab_id: number;
previous_url?: string;
final_url?: string;
reached: string;
error_text?: string;
}
const HISTORY_OUTPUT = {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
previousUrl: { type: "string" },
finalUrl: { type: "string" },
reached: { type: "string", required: true },
errorText: { type: "string" },
},
} as const;
function mapHistory(sessionId: string, reply: RawHistoryResult) {
return {
session: sessionId,
tabId: reply.tab_id,
...(reply.previous_url !== undefined ? { previousUrl: reply.previous_url } : {}),
...(reply.final_url !== undefined ? { finalUrl: reply.final_url } : {}),
reached: reply.reached,
...(reply.error_text !== undefined ? { errorText: reply.error_text } : {}),
};
}
/** Add browser history, reload, and explicit lifecycle waiting. */
export function registerPhaseOneNavigationTools(
deps: ToolDeps,
register: ToolRegistrar,
runtime: PhaseOneRuntime,
): void {
const { registry } = deps;
const registerHistory = (direction: "back" | "forward") => {
const toolName = `browser_navigate_${direction}` as const;
const command = `navigate-${direction}`;
register(
defineTool({
name: toolName,
description: `Navigate the active Agent Window tab ${direction} by one history entry.`,
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
waitUntil: WAIT_UNTIL_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: HISTORY_OUTPUT,
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] navigated ${direction} on tab ${value.tabId}` +
(value.finalUrl !== undefined ? ` to ${value.finalUrl}` : "") +
` (reached: ${value.reached})` +
(value.errorText !== undefined ? ` — ${value.errorText}` : ""),
},
],
},
async execute(args, exec) {
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, toolName);
const cmdArgs = [command, "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
appendWaitOptions(cmdArgs, args.waitUntil, args.timeoutMs);
const reply = (await runtime.run(
exec,
cmdArgs,
command,
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as RawHistoryResult;
if (reply.final_url !== undefined) deps.observation.setUrl(sessionId, reply.final_url);
return mapHistory(sessionId, reply);
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([command, "--session", args.session ?? "(current)"]),
description: `Navigate ${direction}`,
}),
presentResult: runtime.presentTerminalResult,
}),
);
};
registerHistory("back");
registerHistory("forward");
register(
defineTool({
name: "browser_reload",
description:
"Reload the active Agent Window tab and wait for a lifecycle phase. Set hard=true to " +
"bypass the HTTP cache.",
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
waitUntil: WAIT_UNTIL_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
hard: { type: "boolean", description: "Bypass the HTTP cache while reloading." },
},
output: {
schema: HISTORY_OUTPUT,
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] reloaded tab ${value.tabId}` +
(value.finalUrl !== undefined ? ` at ${value.finalUrl}` : "") +
` (reached: ${value.reached})` +
(value.errorText !== undefined ? ` — ${value.errorText}` : ""),
},
],
},
async execute(args, exec) {
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, "browser_reload");
const cmdArgs = ["reload", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
appendWaitOptions(cmdArgs, args.waitUntil, args.timeoutMs);
if (args.hard === true) cmdArgs.push("--hard");
const reply = (await runtime.run(
exec,
cmdArgs,
"reload",
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as RawHistoryResult;
if (reply.final_url !== undefined) deps.observation.setUrl(sessionId, reply.final_url);
return mapHistory(sessionId, reply);
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"reload",
"--session",
args.session ?? "(current)",
...(args.hard === true ? ["--hard"] : []),
]),
description: "Reload the active tab",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_wait_for_navigation",
description:
"Wait for a page lifecycle event on the active Agent Window tab. Use after an action " +
"that may navigate when the action result itself does not wait for navigation.",
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
waitUntil: WAIT_UNTIL_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
reached: { type: "string", required: true },
errorText: { type: "string" },
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] navigation wait on tab ${value.tabId}: ${value.reached}` +
(value.errorText !== undefined ? ` — ${value.errorText}` : ""),
},
],
},
async execute(args, exec) {
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, "browser_wait_for_navigation");
const timeoutMs = args.timeoutMs ?? 30_000;
const cmdArgs = ["wait-for-navigation", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.waitUntil !== undefined) cmdArgs.push("--wait-until", args.waitUntil);
cmdArgs.push("--timeout", `${timeoutMs}ms`);
const reply = (await runtime.run(
exec,
cmdArgs,
"wait-for-navigation",
sessionId,
runnerTimeout(deps, timeoutMs),
)) as { tab_id: number; reached: string; error_text?: string };
return {
session: sessionId,
tabId: reply.tab_id,
reached: reply.reached,
...(reply.error_text !== undefined ? { errorText: reply.error_text } : {}),
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"wait-for-navigation",
"--session",
args.session ?? "(current)",
]),
description: "Wait for page navigation",
}),
presentResult: runtime.presentTerminalResult,
}),
);
}
@@ -1,672 +0,0 @@
import { defineTool } from "@deepseek-ai/dsh-tools";
import {
appendTabId,
isSnapshotRef,
type PhaseOneRuntime,
requireNonEmpty,
requirePositive,
runnerTimeout,
type ToolRegistrar,
} from "./phase-one-runtime";
import { SESSION_PARAM, TAB_ID_PARAM, TIMEOUT_MS_PARAM } from "./tool-params";
import type { ToolDeps } from "./tools";
interface HelpConditionInput {
urlContains?: string;
urlMatches?: string;
selectorExists?: string;
selectorMissing?: string;
textExists?: string;
textMissing?: string;
}
interface CompletionCriteriaInput {
any?: HelpConditionInput[];
all?: HelpConditionInput[];
stableForMs?: number;
}
interface RawConsoleFrame {
function_name?: string;
url?: string;
line?: number;
column?: number;
}
interface RawConsoleEntry {
sequence: number;
kind: string;
level: string;
text: string;
url?: string;
line?: number;
column?: number;
timestamp?: number;
stack_trace?: RawConsoleFrame[];
truncated?: boolean;
}
interface RawNetworkEntry {
sequence: number;
kind: "response" | "failure";
method?: string;
url?: string;
status?: number;
status_text?: string;
mime_type?: string;
resource_type?: string;
error_text?: string;
timestamp?: number;
truncated?: boolean;
}
const HELP_CONDITION_SCHEMA = {
type: "object",
additionalProperties: false,
properties: {
urlContains: { type: "string" },
urlMatches: { type: "string" },
selectorExists: { type: "string" },
selectorMissing: { type: "string" },
textExists: { type: "string" },
textMissing: { type: "string" },
},
} as const;
const DEBUG_PARAMETERS = {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
since: {
type: "integer",
description: "Return entries with a sequence strictly greater than this cursor.",
},
limit: {
type: "integer",
description: "Maximum entries to return (default: 50, daemon cap: 200).",
},
maxTextChars: {
type: "integer",
description: "Maximum characters per entry text/URL (default: 1000, daemon cap: 4096).",
},
} as const;
function mapCondition(condition: HelpConditionInput) {
return {
...(condition.urlContains !== undefined ? { url_contains: condition.urlContains } : {}),
...(condition.urlMatches !== undefined ? { url_matches: condition.urlMatches } : {}),
...(condition.selectorExists !== undefined
? { selector_exists: condition.selectorExists }
: {}),
...(condition.selectorMissing !== undefined
? { selector_missing: condition.selectorMissing }
: {}),
...(condition.textExists !== undefined ? { text_exists: condition.textExists } : {}),
...(condition.textMissing !== undefined ? { text_missing: condition.textMissing } : {}),
};
}
function mapCompletionCriteria(criteria: CompletionCriteriaInput) {
return {
...(criteria.any !== undefined ? { any: criteria.any.map(mapCondition) } : {}),
...(criteria.all !== undefined ? { all: criteria.all.map(mapCondition) } : {}),
...(criteria.stableForMs !== undefined ? { stable_for_ms: criteria.stableForMs } : {}),
};
}
function validateDebugArgs(args: { since?: number; limit?: number; maxTextChars?: number }): void {
if (args.since !== undefined && args.since < 0) {
throw new Error("since must be zero or greater");
}
requirePositive(args.limit, "limit");
requirePositive(args.maxTextChars, "maxTextChars");
}
function appendDebugOptions(
args: string[],
options: { tabId?: number; since?: number; limit?: number; maxTextChars?: number },
): void {
appendTabId(args, options.tabId);
if (options.since !== undefined) args.push("--since", String(options.since));
if (options.limit !== undefined) args.push("--limit", String(options.limit));
if (options.maxTextChars !== undefined) {
args.push("--max-text-chars", String(options.maxTextChars));
}
}
/** Human help, raw HTML, diagnostics, and Agent Window sizing. */
export function registerPhaseOneSupportTools(
deps: ToolDeps,
register: ToolRegistrar,
runtime: PhaseOneRuntime,
): void {
const { registry } = deps;
register(
defineTool({
name: "browser_request_help",
description:
"Pause browser automation and ask the user to complete an in-page step such as login, " +
"captcha, OTP, or confirmation. After continued/completed, observe again before using refs.",
parameters: {
prompt: {
type: "string",
required: true,
description: "Clear instructions shown to the user in the browser help overlay.",
},
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
title: { type: "string", description: "Optional title for the help overlay." },
targets: {
type: "array",
items: { type: "string" },
description: "Snapshot refs or CSS selectors to scroll to and highlight for the user.",
},
timeoutMs: {
...TIMEOUT_MS_PARAM,
description: "Maximum wait for the user in milliseconds (default: 300000).",
},
completionCriteria: {
type: "object",
additionalProperties: false,
description: "Optional explicit success detector for automatic completion.",
properties: {
any: { type: "array", items: HELP_CONDITION_SCHEMA },
all: { type: "array", items: HELP_CONDITION_SCHEMA },
stableForMs: {
type: "integer",
description: "How long criteria must remain true before completing.",
},
},
},
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
outcome: {
type: "string",
required: true,
enum: ["continued", "cancelled", "timed_out", "completed", "navigated", "disabled"],
},
completedBy: { type: "string" },
note: { type: "string" },
resolvedTargets: {
type: "array",
items: {
type: "object",
additionalProperties: false,
properties: {
matched: { type: "boolean", required: true },
ref: { type: "string" },
selector: { type: "string" },
},
},
},
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] user help outcome on tab ${value.tabId}: ${value.outcome}` +
(value.note !== undefined ? ` — ${value.note}` : "") +
(value.resolvedTargets !== undefined
? `\ntargets: ${value.resolvedTargets
.map(
(target) =>
`${target.ref ?? target.selector ?? "(unknown)"}=${target.matched ? "matched" : "not found"}`,
)
.join(", ")}`
: ""),
},
],
},
async execute(args, exec) {
requireNonEmpty(args.prompt, "prompt");
if (args.title !== undefined) requireNonEmpty(args.title, "title");
requirePositive(args.timeoutMs, "timeoutMs");
for (const target of args.targets ?? []) requireNonEmpty(target, "target");
const criteria = args.completionCriteria as CompletionCriteriaInput | undefined;
if (criteria?.stableForMs !== undefined && criteria.stableForMs < 0) {
throw new Error("completionCriteria.stableForMs must be zero or greater");
}
const sessionId = registry.resolve(args.session, "browser_request_help");
const timeoutMs = args.timeoutMs ?? 300_000;
const cmdArgs = [
"request-help",
"--session",
sessionId,
"--prompt",
args.prompt,
"--timeout",
`${timeoutMs}ms`,
];
appendTabId(cmdArgs, args.tabId);
if (args.title !== undefined) cmdArgs.push("--title", args.title);
for (const target of args.targets ?? []) cmdArgs.push("--target", target);
if (criteria !== undefined) {
cmdArgs.push("--completion-criteria", JSON.stringify(mapCompletionCriteria(criteria)));
}
const reply = (await runtime.run(
exec,
cmdArgs,
"request-help",
sessionId,
runnerTimeout(deps, timeoutMs),
)) as {
outcome: "continued" | "cancelled" | "timed_out" | "completed" | "navigated" | "disabled";
completed_by?: string;
note?: string;
tab_id: number;
resolved_targets?: { matched: boolean; ref?: string; selector?: string }[];
};
return {
session: sessionId,
tabId: reply.tab_id,
outcome: reply.outcome,
...(reply.completed_by !== undefined ? { completedBy: reply.completed_by } : {}),
...(reply.note !== undefined ? { note: reply.note } : {}),
...(reply.resolved_targets !== undefined
? { resolvedTargets: reply.resolved_targets }
: {}),
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine(["request-help", "--session", args.session ?? "(current)"]),
description: "Ask the user for browser help",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_get_html",
description:
"Read raw DOM HTML from the active Agent Window tab, optionally scoped to a fresh " +
"snapshot ref. Use only when browser_observe/browser_snapshot cannot answer the question.",
parameters: {
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
ref: { type: "string", description: "Fresh snapshot ref that scopes the HTML subtree." },
maxBytes: {
type: "integer",
description: "Maximum returned HTML bytes before truncation (default: 524288).",
},
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
html: { type: "string", required: true },
truncated: { type: "boolean", required: true },
byteSize: { type: "integer", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text:
value.html +
(value.truncated ? `\n(HTML truncated; original size ${value.byteSize} bytes)` : ""),
},
],
},
isConcurrencySafe: () => true,
async execute(args, exec) {
if (args.ref !== undefined) {
requireNonEmpty(args.ref, "ref");
if (!isSnapshotRef(args.ref)) throw new Error("ref must be a snapshot ref such as @e3");
}
requirePositive(args.maxBytes, "maxBytes");
const sessionId = registry.resolve(args.session, "browser_get_html");
const cmdArgs = ["get-html", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.ref !== undefined) cmdArgs.push("--ref", args.ref);
if (args.maxBytes !== undefined) cmdArgs.push("--max-bytes", String(args.maxBytes));
const reply = (await runtime.run(exec, cmdArgs, "get-html", sessionId)) as {
tab_id: number;
html: string;
truncated?: boolean;
byte_size: number;
};
return {
session: sessionId,
tabId: reply.tab_id,
html: reply.html,
truncated: reply.truncated ?? false,
byteSize: reply.byte_size,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"get-html",
"--session",
args.session ?? "(current)",
...(args.ref !== undefined ? ["--ref", args.ref] : []),
]),
description: "Read raw page HTML",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_console",
description:
"Read buffered console messages, browser log entries, and JavaScript exceptions from a tab. " +
"This is read-only and does not evaluate JavaScript.",
parameters: {
...DEBUG_PARAMETERS,
includeStack: {
type: "boolean",
description: "Include structured stack frames in returned console entries.",
},
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
entries: {
type: "array",
required: true,
items: {
type: "object",
additionalProperties: false,
properties: {
sequence: { type: "integer", required: true },
kind: { type: "string", required: true },
level: { type: "string", required: true },
text: { type: "string", required: true },
url: { type: "string" },
line: { type: "integer" },
column: { type: "integer" },
timestamp: { type: "number" },
stackTrace: {
type: "array",
required: true,
items: {
type: "object",
additionalProperties: false,
properties: {
functionName: { type: "string" },
url: { type: "string" },
line: { type: "integer" },
column: { type: "integer" },
},
},
},
truncated: { type: "boolean", required: true },
},
},
},
nextSince: { type: "integer", required: true },
truncated: { type: "boolean", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text:
value.entries.length === 0
? "(no console messages captured)"
: value.entries
.map((entry) => {
const location =
entry.url === undefined
? ""
: ` ${entry.url}${entry.line !== undefined ? `:${entry.line}` : ""}${entry.column !== undefined ? `:${entry.column}` : ""}`;
const stack = entry.stackTrace
.map(
(frame) =>
`\n at ${frame.functionName ?? "<anonymous>"} ${frame.url ?? ""}` +
`${frame.line !== undefined ? `:${frame.line}` : ""}` +
`${frame.column !== undefined ? `:${frame.column}` : ""}`,
)
.join("");
return `#${entry.sequence} ${entry.level} ${entry.kind}${location} ${entry.text}${stack}`;
})
.join("\n") +
(value.truncated
? `\n(output truncated; continue with since=${value.nextSince})`
: ""),
},
],
},
isConcurrencySafe: () => true,
async execute(args, exec) {
validateDebugArgs(args);
const sessionId = registry.resolve(args.session, "browser_console");
const cmdArgs = ["console", "--session", sessionId];
appendDebugOptions(cmdArgs, args);
if (args.includeStack === true) cmdArgs.push("--include-stack");
const reply = (await runtime.run(exec, cmdArgs, "console", sessionId)) as {
tab_id: number;
entries?: RawConsoleEntry[];
next_since: number;
truncated?: boolean;
};
return {
session: sessionId,
tabId: reply.tab_id,
entries: (reply.entries ?? []).map((entry) => ({
sequence: entry.sequence,
kind: entry.kind,
level: entry.level,
text: entry.text,
...(entry.url !== undefined ? { url: entry.url } : {}),
...(entry.line !== undefined ? { line: entry.line } : {}),
...(entry.column !== undefined ? { column: entry.column } : {}),
...(entry.timestamp !== undefined ? { timestamp: entry.timestamp } : {}),
stackTrace: (entry.stack_trace ?? []).map((frame) => ({
...(frame.function_name !== undefined ? { functionName: frame.function_name } : {}),
...(frame.url !== undefined ? { url: frame.url } : {}),
...(frame.line !== undefined ? { line: frame.line } : {}),
...(frame.column !== undefined ? { column: frame.column } : {}),
})),
truncated: entry.truncated ?? false,
})),
nextSince: reply.next_since,
truncated: reply.truncated ?? false,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine(["console", "--session", args.session ?? "(current)"]),
description: "Read browser console messages",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_network",
description:
"Read buffered network responses and failures from a tab. Returns metadata only; request " +
"and response headers and bodies are not captured.",
parameters: DEBUG_PARAMETERS,
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
entries: {
type: "array",
required: true,
items: {
type: "object",
additionalProperties: false,
properties: {
sequence: { type: "integer", required: true },
kind: { type: "string", required: true, enum: ["response", "failure"] },
method: { type: "string" },
url: { type: "string" },
status: { type: "integer" },
statusText: { type: "string" },
mimeType: { type: "string" },
resourceType: { type: "string" },
errorText: { type: "string" },
timestamp: { type: "number" },
truncated: { type: "boolean", required: true },
},
},
},
nextSince: { type: "integer", required: true },
truncated: { type: "boolean", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text:
value.entries.length === 0
? "(no network activity captured)"
: value.entries
.map((entry) =>
entry.kind === "failure"
? `#${entry.sequence} FAILED ${entry.method ?? "?"} ${entry.url ?? "(unknown)"} — ${entry.errorText ?? "failed"}`
: `#${entry.sequence} ${entry.status ?? "?"} ${entry.method ?? "?"} ${entry.url ?? "(unknown)"}`,
)
.join("\n") +
(value.truncated
? `\n(output truncated; continue with since=${value.nextSince})`
: ""),
},
],
},
isConcurrencySafe: () => true,
async execute(args, exec) {
validateDebugArgs(args);
const sessionId = registry.resolve(args.session, "browser_network");
const cmdArgs = ["network", "--session", sessionId];
appendDebugOptions(cmdArgs, args);
const reply = (await runtime.run(exec, cmdArgs, "network", sessionId)) as {
tab_id: number;
entries?: RawNetworkEntry[];
next_since: number;
truncated?: boolean;
};
return {
session: sessionId,
tabId: reply.tab_id,
entries: (reply.entries ?? []).map((entry) => ({
sequence: entry.sequence,
kind: entry.kind,
...(entry.method !== undefined ? { method: entry.method } : {}),
...(entry.url !== undefined ? { url: entry.url } : {}),
...(entry.status !== undefined ? { status: entry.status } : {}),
...(entry.status_text !== undefined ? { statusText: entry.status_text } : {}),
...(entry.mime_type !== undefined ? { mimeType: entry.mime_type } : {}),
...(entry.resource_type !== undefined ? { resourceType: entry.resource_type } : {}),
...(entry.error_text !== undefined ? { errorText: entry.error_text } : {}),
...(entry.timestamp !== undefined ? { timestamp: entry.timestamp } : {}),
truncated: entry.truncated ?? false,
})),
nextSince: reply.next_since,
truncated: reply.truncated ?? false,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine(["network", "--session", args.session ?? "(current)"]),
description: "Read browser network activity",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_window_resize",
description: "Resize the owned session's Agent Window using outer CSS-pixel dimensions.",
parameters: {
width: {
type: "integer",
required: true,
description: "Agent Window outer width in CSS pixels (100..=7680).",
},
height: {
type: "integer",
required: true,
description: "Agent Window outer height in CSS pixels (100..=7680).",
},
session: SESSION_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
windowId: { type: "integer", required: true },
width: { type: "integer", required: true },
height: { type: "integer", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text: `[session ${value.session}] resized window ${value.windowId} to ${value.width}x${value.height}`,
},
],
},
async execute(args, exec) {
if (args.width < 100 || args.width > 7680 || args.height < 100 || args.height > 7680) {
throw new Error("width and height must each be in the range 100..=7680");
}
const sessionId = registry.resolve(args.session, "browser_window_resize");
const reply = (await runtime.run(
exec,
[
"window",
"resize",
"--session",
sessionId,
"--width",
String(args.width),
"--height",
String(args.height),
],
"window resize",
sessionId,
)) as { window_id: number; width: number; height: number };
return {
session: sessionId,
windowId: reply.window_id,
width: reply.width,
height: reply.height,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"window",
"resize",
"--session",
args.session ?? "(current)",
"--width",
String(args.width),
"--height",
String(args.height),
]),
description: "Resize the Agent Window",
}),
presentResult: runtime.presentTerminalResult,
}),
);
}
@@ -1,402 +0,0 @@
import { defineTool } from "@deepseek-ai/dsh-tools";
import type { PhaseOneRuntime, ToolRegistrar } from "./phase-one-runtime";
import { requireNonEmpty } from "./phase-one-runtime";
import { SESSION_PARAM } from "./tool-params";
import type { ToolDeps } from "./tools";
const TAB_ID_REQUIRED = {
type: "integer",
required: true,
description: "Chrome tab id returned by browser_tab_list or browser_tab_create.",
} as const;
interface RawTabInfo {
tab_id: number;
title?: string;
url?: string;
window_id?: number;
active?: boolean;
scope?: "user" | "agent";
}
/** Add Agent Window tab management while preserving the owned-session boundary. */
export function registerPhaseOneTabTools(
deps: ToolDeps,
register: ToolRegistrar,
runtime: PhaseOneRuntime,
): void {
const { registry } = deps;
register(
defineTool({
name: "browser_tab_list",
description:
"List tabs visible to an owned browser session. Other sessions' Agent Windows remain " +
"hidden; use scope=user to find a user tab before borrowing it.",
parameters: {
session: SESSION_PARAM,
scope: {
type: "string",
enum: ["user", "agent", "all"],
description: "Which tabs to list (default: all).",
},
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabs: {
type: "array",
required: true,
items: {
type: "object",
additionalProperties: false,
properties: {
tabId: { type: "integer", required: true },
title: { type: "string" },
url: { type: "string" },
windowId: { type: "integer" },
active: { type: "boolean" },
scope: { type: "string", enum: ["user", "agent"] },
},
},
},
},
},
render: (_args, value) => [
{
type: "text",
text:
value.tabs.length === 0
? `[session ${value.session}] no matching tabs`
: value.tabs
.map(
(tab) =>
`${tab.tabId} [${tab.scope ?? "unknown"}]${tab.active === true ? " [active]" : ""} ` +
`${tab.title ?? "(untitled)"} — ${tab.url ?? "(unknown URL)"}`,
)
.join("\n"),
},
],
},
isConcurrencySafe: () => true,
async execute(args, exec) {
const sessionId = registry.resolve(args.session, "browser_tab_list");
const cmdArgs = ["tab", "list", "--session", sessionId];
if (args.scope !== undefined) cmdArgs.push("--scope", args.scope);
const reply = (await runtime.run(exec, cmdArgs, "tab list", sessionId)) as {
tabs: RawTabInfo[];
};
return {
session: sessionId,
tabs: reply.tabs.map((tab) => ({
tabId: tab.tab_id,
...(tab.title !== undefined ? { title: tab.title } : {}),
...(tab.url !== undefined ? { url: tab.url } : {}),
...(tab.window_id !== undefined ? { windowId: tab.window_id } : {}),
...(tab.active !== undefined ? { active: tab.active } : {}),
...(tab.scope !== undefined ? { scope: tab.scope } : {}),
})),
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"tab",
"list",
"--session",
args.session ?? "(current)",
...(args.scope !== undefined ? ["--scope", args.scope] : []),
]),
description: "List browser tabs",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_tab_create",
description:
"Create a tab inside the owned session's Agent Window. The tab is focused by default; " +
"set active=false to open it in the background.",
parameters: {
session: SESSION_PARAM,
url: { type: "string", description: "Initial URL (default: chrome://newtab/)." },
active: { type: "boolean", description: "Whether to focus the new tab (default: true)." },
index: { type: "integer", description: "Insertion index in the Agent Window tab strip." },
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
windowId: { type: "integer", required: true },
url: { type: "string", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text: `[session ${value.session}] created tab ${value.tabId} in window ${value.windowId} (${value.url || "pending URL"})`,
},
],
},
async execute(args, exec) {
if (args.url !== undefined) requireNonEmpty(args.url, "url");
const sessionId = registry.resolve(args.session, "browser_tab_create");
const cmdArgs = ["tab", "create", "--session", sessionId];
if (args.url !== undefined) cmdArgs.push("--url", args.url);
if (args.active === false) cmdArgs.push("--no-active");
if (args.index !== undefined) cmdArgs.push("--index", String(args.index));
const reply = (await runtime.run(exec, cmdArgs, "tab create", sessionId)) as {
tab_id: number;
window_id: number;
url: string;
};
if (args.active !== false && reply.url.length > 0) {
deps.observation.setUrl(sessionId, reply.url);
}
return {
session: sessionId,
tabId: reply.tab_id,
windowId: reply.window_id,
url: reply.url,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"tab",
"create",
"--session",
args.session ?? "(current)",
...(args.url !== undefined ? ["--url", args.url] : []),
]),
description: "Create an Agent Window tab",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_tab_close",
description: "Close a tab in the owned session's Agent Window.",
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
},
},
render: (_args, value) => [
{ type: "text", text: `[session ${value.session}] closed tab ${value.tabId}` },
],
},
async execute(args, exec) {
const sessionId = registry.resolve(args.session, "browser_tab_close");
const reply = (await runtime.run(
exec,
["tab", "close", String(args.tabId), "--session", sessionId],
"tab close",
sessionId,
)) as { tab_id: number };
return { session: sessionId, tabId: reply.tab_id };
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"tab",
"close",
String(args.tabId),
"--session",
args.session ?? "(current)",
]),
description: "Close an Agent Window tab",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_tab_select",
description: "Focus a tab in the owned session's Agent Window.",
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
windowId: { type: "integer", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text: `[session ${value.session}] selected tab ${value.tabId} in window ${value.windowId}`,
},
],
},
async execute(args, exec) {
const sessionId = registry.resolve(args.session, "browser_tab_select");
const reply = (await runtime.run(
exec,
["tab", "select", String(args.tabId), "--session", sessionId],
"tab select",
sessionId,
)) as { tab_id: number; window_id: number };
return { session: sessionId, tabId: reply.tab_id, windowId: reply.window_id };
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"tab",
"select",
String(args.tabId),
"--session",
args.session ?? "(current)",
]),
description: "Select an Agent Window tab",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_tab_borrow",
description:
"Move a user-window tab into the owned session's Agent Window for controlled interaction. " +
"Return it with browser_tab_return when finished; stopping the session also auto-returns it.",
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
originalWindowId: { type: "integer", required: true },
originalIndex: { type: "integer", required: true },
agentWindowId: { type: "integer", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] borrowed tab ${value.tabId} from window ` +
`${value.originalWindowId} into Agent Window ${value.agentWindowId}`,
},
],
},
async execute(args, exec) {
const sessionId = registry.resolve(args.session, "browser_tab_borrow");
const reply = (await runtime.run(
exec,
["tab", "borrow", String(args.tabId), "--session", sessionId],
"tab borrow",
sessionId,
)) as {
tab_id: number;
original_window_id: number;
original_index: number;
agent_window_id: number;
};
return {
session: sessionId,
tabId: reply.tab_id,
originalWindowId: reply.original_window_id,
originalIndex: reply.original_index,
agentWindowId: reply.agent_window_id,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"tab",
"borrow",
String(args.tabId),
"--session",
args.session ?? "(current)",
]),
description: "Borrow a user tab",
}),
presentResult: runtime.presentTerminalResult,
}),
);
register(
defineTool({
name: "browser_tab_return",
description: "Return a borrowed tab to its original user window and tab-strip position.",
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
returnedToWindowId: { type: "integer", required: true },
returnedToIndex: { type: "integer", required: true },
fallback: { type: "boolean", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] returned tab ${value.tabId} to window ` +
`${value.returnedToWindowId} at index ${value.returnedToIndex}` +
(value.fallback ? " (original window unavailable; used fallback)" : ""),
},
],
},
async execute(args, exec) {
const sessionId = registry.resolve(args.session, "browser_tab_return");
const reply = (await runtime.run(
exec,
["tab", "return", String(args.tabId), "--session", sessionId],
"tab return",
sessionId,
)) as {
tab_id: number;
returned_to_window_id: number;
returned_to_index: number;
fallback?: boolean;
};
return {
session: sessionId,
tabId: reply.tab_id,
returnedToWindowId: reply.returned_to_window_id,
returnedToIndex: reply.returned_to_index,
fallback: reply.fallback ?? false,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"tab",
"return",
String(args.tabId),
"--session",
args.session ?? "(current)",
]),
description: "Return a borrowed tab",
}),
presentResult: runtime.presentTerminalResult,
}),
);
}
@@ -1,18 +0,0 @@
import type { PhaseOneRuntime, ToolRegistrar } from "./phase-one-runtime";
import { registerPhaseOneInteractionTools } from "./phase-one-tools-interaction";
import { registerPhaseOneNavigationTools } from "./phase-one-tools-navigation";
import { registerPhaseOneSupportTools } from "./phase-one-tools-support";
import { registerPhaseOneTabTools } from "./phase-one-tools-tabs";
import type { ToolDeps } from "./tools";
/** Register the first DSH capability-parity tranche on the existing runtime. */
export function registerPhaseOneTools(
deps: ToolDeps,
register: ToolRegistrar,
runtime: PhaseOneRuntime,
): void {
registerPhaseOneInteractionTools(deps, register, runtime);
registerPhaseOneTabTools(deps, register, runtime);
registerPhaseOneNavigationTools(deps, register, runtime);
registerPhaseOneSupportTools(deps, register, runtime);
}
@@ -2,10 +2,10 @@
* Progressive disclosure of the BrowserSkill agent skill through the harness's
* official skill seam (`ctx.skills`). The catalog entry (name + description)
* is resident; the body is loaded only when the model invokes the `skill`
* tool. The dedicated DeepSeek Harness markdown is embedded into a static
* module at build time, so registration and every pre-step catalog snapshot
* are pure in-memory reads — no disk, no process, no daemon. The CLI-oriented
* repository skill is intentionally a separate interface.
* tool. The markdown is build-time assembled into a static module (dsh
* prelude + the canonical CLI skill body verbatim), so registration and every
* pre-step catalog snapshot are pure in-memory reads — no disk, no process,
* no daemon.
*/
import type { Context } from "@deepseek-ai/cordis";
@@ -26,7 +26,7 @@ interface SkillsLike {
}
/**
* Publish the browser skill as an embedded runtime skill. Returns the unregistration
* Publish the BSK skill as an embedded runtime skill. Returns the unregistration
* disposer. A composition without the `skills` service (or with dsh-tool-skill
* retired) leaves the rest of the plugin unaffected — the no-op is silent.
*/
@@ -1,24 +0,0 @@
/** Shared model-facing parameter schemas for browser tools. */
export const SESSION_PARAM = {
type: "string",
description:
"bsk session id to act on; must be one created by this plugin's browser_session_start. " +
"Omit to use the current session (the one most recently started or used).",
} as const;
export const TAB_ID_PARAM = {
type: "integer",
description: "Target tab id. Omit to use the Agent Window's active tab.",
} as const;
export const WAIT_UNTIL_PARAM = {
type: "string",
enum: ["load", "domcontentloaded", "networkidle", "commit"],
description: "Page lifecycle phase to wait for (default: load).",
} as const;
export const TIMEOUT_MS_PARAM = {
type: "integer",
description: "Command timeout in milliseconds; must be greater than zero.",
} as const;
+8 -10
View File
@@ -19,11 +19,9 @@ import {
} from "@deepseek-ai/dsh-tools";
import { trySaveScreenshot } from "./image";
import { actionForLabel, type ObservationService } from "./observation";
import { registerPhaseOneTools } from "./phase-one-tools";
import type { KeyedExecutor } from "./queue";
import { type BskRunner, bskInstallMessage, isCommandNotFound, parseBskJson } from "./runner";
import type { SessionRegistry } from "./sessions";
import { SESSION_PARAM } from "./tool-params";
/** Plugin configuration resolved from the Schemastery schema in index.ts. */
export interface PluginConfig {
@@ -63,6 +61,13 @@ const DEVICE_PRESETS = [
"galaxy-tab-s8",
] as const;
const SESSION_PARAM = {
type: "string",
description:
"bsk session id to act on; must be one created by this plugin's browser_session_start. " +
"Omit to use the current session (the one most recently started or used).",
} as const;
/**
* Run one bsk command and return its parsed JSON payload, or throw.
* `observeSession` marks the run as model-facing work on that session: it is
@@ -76,7 +81,6 @@ async function runBsk(
args: string[],
label: string,
observeSession?: string,
runnerTimeoutMs?: number,
): Promise<unknown> {
// beginAction must fire only when the task actually starts inside the
// per-session queue — marking it while still queued would overwrite the
@@ -93,7 +97,7 @@ async function runBsk(
}
return deps.runner.run(args, {
signal: exec.signal,
timeoutMs: runnerTimeoutMs ?? deps.config.defaultTimeoutMs,
timeoutMs: deps.config.defaultTimeoutMs,
...(observeSession !== undefined ? { tag: observeSession } : {}),
});
};
@@ -951,12 +955,6 @@ export function registerTools(deps: ToolDeps): () => void {
presentResult: presentTerminalResult,
}),
);
registerPhaseOneTools(deps, register, {
run: (exec, args, label, observeSession, runnerTimeoutMs) =>
runBsk(deps, exec, args, label, observeSession, runnerTimeoutMs),
commandLine: (args) => cmdline(deps, args),
presentTerminalResult,
});
return () => {
for (const dispose of disposers.splice(0)) dispose();
};
@@ -530,9 +530,6 @@ describe("actionForLabel", () => {
expect(actionForLabel("navigate")).toBe("navigating");
expect(actionForLabel("screenshot")).toBe("capturing");
expect(actionForLabel("session start")).toBe("starting");
expect(actionForLabel("tab borrow")).toBe("borrowing tab");
expect(actionForLabel("request-help")).toBe("waiting for user");
expect(actionForLabel("wait-for-navigation")).toBe("waiting for navigation");
});
});
@@ -1,4 +1,4 @@
// Browser skill injection: registration wiring, catalog content, progressive-load
// BSK skill injection: registration wiring, catalog content, progressive-load
// weight, and silent degradation without the skill seam.
import { describe, expect, it } from "vitest";
@@ -9,7 +9,7 @@ function fakeCtx(skills?: unknown) {
}
describe("registerBskSkill", () => {
it("registers the catalog entry with the DSH browser-tool body", () => {
it("registers the catalog entry with the assembled body", () => {
const registrations: { skill: Record<string, unknown>; disposer: () => void }[] = [];
const skills = {
register(skill: Record<string, unknown>) {
@@ -26,51 +26,23 @@ describe("registerBskSkill", () => {
expect(skill.name).toBe("browser-skill");
expect(typeof skill.description).toBe("string");
expect(String(skill.description)).toContain("browser_*");
// body = dsh prelude + canonical CLI skill body + dsh postlude
const content = String(skill.content);
const mentionedTools = [...new Set(content.match(/\bbrowser_[a-z][a-z_]*\b/g) ?? [])].sort();
expect(mentionedTools).toEqual([
"browser_click",
"browser_console",
"browser_emulate",
"browser_fill",
"browser_get_html",
"browser_hover",
"browser_navigate",
"browser_navigate_back",
"browser_navigate_forward",
"browser_network",
"browser_observe",
"browser_press",
"browser_reload",
"browser_request_help",
"browser_screenshot",
"browser_select",
"browser_session_list",
"browser_session_start",
"browser_session_stop",
"browser_snapshot",
"browser_tab_borrow",
"browser_tab_close",
"browser_tab_create",
"browser_tab_list",
"browser_tab_return",
"browser_tab_select",
"browser_wait_for_navigation",
"browser_window_resize",
]);
expect(String(skill.description)).not.toMatch(/\bbsk\b/i);
expect(content).not.toMatch(/\bbsk\b/i);
expect(content).not.toMatch(/```(?:bash|sh|shell)\b/i);
expect(content).not.toMatch(/--[a-z]/);
expect(content).toContain("All browser operations must use the injected tools directly");
expect(content).toContain("browser_observe");
expect(String(skill.description)).toContain("never invoke bsk through bash or shell");
expect(content).toContain("Always call those injected `browser_*` tools");
expect(content).toContain("Never use `bash`, `shell`, `exec`");
expect(content).toContain("## DSH routing reminder");
expect(content.trimEnd()).toMatch(
/always begin with[\s\S]*`browser_session_start`[\s\S]*`browser_session_stop`\.$/,
);
expect(content).toContain("Owned sessions only");
expect(content).toContain("Mandatory workflow");
expect(content).toContain("Refs invalidate after navigation");
expect(content).toContain("evaluation and interaction recording are intentionally unsupported");
expect(content.length).toBeGreaterThan(10_000);
expect(skill.source).toBe("bundled");
// Source frontmatter is registration metadata and must not leak into the body.
expect(content.startsWith("---")).toBe(false);
expect(content).not.toContain("name: browser-skill\ndescription:");
// the original YAML frontmatter must not leak into the body
expect(content.includes("name: browser-skill\ndescription:")).toBe(false);
// disposer passthrough
unregister();
expect(skill.__disposed).toBe(true);
@@ -142,33 +142,16 @@ describe("tool registration", () => {
const { tools } = setup({});
expect([...tools.keys()].sort()).toEqual([
"browser_click",
"browser_console",
"browser_emulate",
"browser_fill",
"browser_get_html",
"browser_hover",
"browser_navigate",
"browser_navigate_back",
"browser_navigate_forward",
"browser_network",
"browser_observe",
"browser_press",
"browser_reload",
"browser_request_help",
"browser_screenshot",
"browser_select",
"browser_session_list",
"browser_session_start",
"browser_session_stop",
"browser_snapshot",
"browser_tab_borrow",
"browser_tab_close",
"browser_tab_create",
"browser_tab_list",
"browser_tab_return",
"browser_tab_select",
"browser_wait_for_navigation",
"browser_window_resize",
]);
});
@@ -453,336 +436,6 @@ describe("interaction tools", () => {
/mutually exclusive/,
);
});
it("browser_hover and browser_select map their structured arguments", async () => {
const { tools, calls } = setup({
"session start": START_REPLY("s1"),
hover: { tab_id: 7, used_ref: "@e3", x: 12, y: 24 },
select: {
tab_id: 7,
used_ref: "@e4",
multiple: true,
selected_values: ["a", "b"],
selected_labels: ["Alpha", "Beta"],
},
});
await startSession(tools);
const hovered = (await tools
.get("browser_hover")
?.execute(
{ target: "@e3", modifiers: ["shift"], settleMs: 250, timeoutMs: 2000 },
makeExec(),
)) as { x: number; y: number };
expect(hovered).toMatchObject({ x: 12, y: 24 });
expect(calls[1].args).toEqual([
"hover",
"--session",
"s1",
"--modifiers",
"shift",
"--settle",
"250ms",
"--timeout",
"2000ms",
"@e3",
]);
const selected = (await tools
.get("browser_select")
?.execute({ target: "@e4", values: ["a", "b"], tabId: 7 }, makeExec())) as {
selectedValues: string[];
selectedLabels: string[];
};
expect(selected).toMatchObject({
selectedValues: ["a", "b"],
selectedLabels: ["Alpha", "Beta"],
});
expect(calls[2].args).toEqual([
"select",
"--session",
"s1",
"--tab-id",
"7",
"--value",
"a",
"--value",
"b",
"@e4",
]);
});
});
describe("phase-one tab tools", () => {
it("maps list/create/close/select/borrow/return through the owned session", async () => {
const { tools, calls } = setup({
"session start": START_REPLY("s1"),
"tab list": {
tabs: [
{
tab_id: 7,
title: "Example",
url: "https://example.test",
window_id: 2,
active: true,
scope: "agent",
},
],
},
"tab create": { tab_id: 8, window_id: 2, url: "https://new.test" },
"tab close": { tab_id: 8 },
"tab select": { tab_id: 7, window_id: 2 },
"tab borrow": {
tab_id: 20,
original_window_id: 9,
original_index: 1,
agent_window_id: 2,
},
"tab return": {
tab_id: 20,
returned_to_window_id: 9,
returned_to_index: 1,
fallback: false,
},
});
await startSession(tools);
const listed = (await tools
.get("browser_tab_list")
?.execute({ scope: "agent" }, makeExec())) as { tabs: { tabId: number }[] };
expect(listed.tabs[0].tabId).toBe(7);
const created = (await tools
.get("browser_tab_create")
?.execute({ url: "https://new.test", active: false, index: 1 }, makeExec())) as {
tabId: number;
};
expect(created.tabId).toBe(8);
await tools.get("browser_tab_close")?.execute({ tabId: 8 }, makeExec());
await tools.get("browser_tab_select")?.execute({ tabId: 7 }, makeExec());
const borrowed = (await tools
.get("browser_tab_borrow")
?.execute({ tabId: 20 }, makeExec())) as { originalWindowId: number };
expect(borrowed.originalWindowId).toBe(9);
const returned = (await tools
.get("browser_tab_return")
?.execute({ tabId: 20 }, makeExec())) as { fallback: boolean };
expect(returned.fallback).toBe(false);
expect(calls.slice(1).map((call) => call.args)).toEqual([
["tab", "list", "--session", "s1", "--scope", "agent"],
[
"tab",
"create",
"--session",
"s1",
"--url",
"https://new.test",
"--no-active",
"--index",
"1",
],
["tab", "close", "8", "--session", "s1"],
["tab", "select", "7", "--session", "s1"],
["tab", "borrow", "20", "--session", "s1"],
["tab", "return", "20", "--session", "s1"],
]);
});
});
describe("phase-one navigation tools", () => {
it("maps back/forward/reload/wait and preserves timeout budgets", async () => {
const { tools, calls } = setup({
"session start": START_REPLY("s1"),
"navigate-back": {
tab_id: 7,
previous_url: "https://b.test",
final_url: "https://a.test",
reached: "load",
},
"navigate-forward": {
tab_id: 7,
previous_url: "https://a.test",
final_url: "https://b.test",
reached: "load",
},
reload: {
tab_id: 7,
previous_url: "https://b.test",
final_url: "https://b.test",
reached: "networkidle",
},
"wait-for-navigation": { tab_id: 7, reached: "timeout", error_text: "no navigation" },
});
await startSession(tools);
await tools
.get("browser_navigate_back")
?.execute({ tabId: 7, waitUntil: "commit", timeoutMs: 1000 }, makeExec());
await tools.get("browser_navigate_forward")?.execute({}, makeExec());
await tools
.get("browser_reload")
?.execute({ hard: true, waitUntil: "networkidle" }, makeExec());
const waited = (await tools
.get("browser_wait_for_navigation")
?.execute({ timeoutMs: 2500 }, makeExec())) as { reached: string; errorText: string };
expect(waited).toMatchObject({ reached: "timeout", errorText: "no navigation" });
expect(calls[1].args).toEqual([
"navigate-back",
"--session",
"s1",
"--tab-id",
"7",
"--wait-until",
"commit",
"--timeout",
"1000ms",
]);
expect(calls[2].args).toEqual(["navigate-forward", "--session", "s1"]);
expect(calls[3].args).toEqual([
"reload",
"--session",
"s1",
"--wait-until",
"networkidle",
"--hard",
]);
expect(calls[4].args).toEqual([
"wait-for-navigation",
"--session",
"s1",
"--timeout",
"2500ms",
]);
expect(calls[4].options.timeoutMs).toBe(120_000);
});
});
describe("phase-one support tools", () => {
it("maps request-help targets and structured completion criteria", async () => {
const { tools, calls } = setup({
"session start": START_REPLY("s1"),
"request-help": {
outcome: "completed",
completed_by: "system",
note: "signed in",
tab_id: 7,
resolved_targets: [{ matched: true, ref: "@e3" }],
},
});
await startSession(tools);
const value = (await tools.get("browser_request_help")?.execute(
{
prompt: "Please sign in",
title: "Login required",
targets: ["@e3", "#submit"],
timeoutMs: 300_000,
completionCriteria: {
any: [{ urlContains: "/dashboard" }, { selectorExists: "[data-user]" }],
stableForMs: 1000,
},
},
makeExec(),
)) as { outcome: string; note: string };
expect(value).toMatchObject({ outcome: "completed", note: "signed in" });
expect(calls[1].options.timeoutMs).toBe(315_000);
expect(calls[1].args.slice(0, 8)).toEqual([
"request-help",
"--session",
"s1",
"--prompt",
"Please sign in",
"--timeout",
"300000ms",
"--title",
]);
const criteriaIndex = calls[1].args.indexOf("--completion-criteria");
expect(JSON.parse(calls[1].args[criteriaIndex + 1])).toEqual({
any: [{ url_contains: "/dashboard" }, { selector_exists: "[data-user]" }],
stable_for_ms: 1000,
});
});
it("maps get-html, console, and network results without raw shell access", async () => {
const { tools, calls } = setup({
"session start": START_REPLY("s1"),
"get-html": { tab_id: 7, html: "<main>Hi</main>", truncated: false, byte_size: 15 },
console: {
tab_id: 7,
entries: [
{
sequence: 3,
kind: "exception",
level: "error",
text: "boom",
stack_trace: [{ function_name: "run", url: "https://a.test/app.js", line: 4 }],
},
],
next_since: 3,
truncated: false,
},
network: {
tab_id: 7,
entries: [
{
sequence: 5,
kind: "response",
method: "GET",
url: "https://a.test/api",
status: 200,
mime_type: "application/json",
},
],
next_since: 5,
truncated: false,
},
});
await startSession(tools);
const html = (await tools
.get("browser_get_html")
?.execute({ ref: "@e2", maxBytes: 2048 }, makeExec())) as { html: string };
expect(html.html).toBe("<main>Hi</main>");
const consoleValue = (await tools
.get("browser_console")
?.execute({ since: 1, limit: 10, includeStack: true }, makeExec())) as {
entries: { stackTrace: { functionName: string }[] }[];
};
expect(consoleValue.entries[0].stackTrace[0].functionName).toBe("run");
const networkValue = (await tools
.get("browser_network")
?.execute({ tabId: 7, maxTextChars: 512 }, makeExec())) as {
entries: { mimeType: string }[];
};
expect(networkValue.entries[0].mimeType).toBe("application/json");
expect(calls.slice(1).map((call) => call.args)).toEqual([
["get-html", "--session", "s1", "--ref", "@e2", "--max-bytes", "2048"],
["console", "--session", "s1", "--since", "1", "--limit", "10", "--include-stack"],
["network", "--session", "s1", "--tab-id", "7", "--max-text-chars", "512"],
]);
});
it("resizes only within the CLI-supported range", async () => {
const { tools, calls } = setup({
"session start": START_REPLY("s1"),
"window resize": { window_id: 2, width: 1280, height: 800 },
});
await startSession(tools);
const resize = tools.get("browser_window_resize");
const value = (await resize?.execute({ width: 1280, height: 800 }, makeExec())) as {
windowId: number;
};
expect(value.windowId).toBe(2);
expect(calls[1].args).toEqual([
"window",
"resize",
"--session",
"s1",
"--width",
"1280",
"--height",
"800",
]);
await expect(resize?.execute({ width: 99, height: 800 }, makeExec())).rejects.toThrow(
/100\.\.=7680/,
);
});
});
describe("browser_screenshot", () => {