feat(vom): update vom algorithm, fix overlay problem with background state machine

This commit is contained in:
Ljy-0827
2026-07-27 13:46:33 +08:00
parent e14ae9cea0
commit 4fb2e331a6
36 changed files with 2276 additions and 452 deletions
@@ -0,0 +1,40 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "ObserveParams",
"description": "Parameters for `tool.observe` — produce a semantic VOM observation for one tab plus a fresh `@e<N>` ref-store on the extension side.\n\nUnlike `tool.snapshot`, this path may run bounded perception probes such as hover-surface discovery. It must not submit input, click, navigate, or otherwise commit page state.",
"type": "object",
"required": [
"session_id"
],
"properties": {
"max_depth": {
"description": "Cap the depth of the rendered VOM tree.",
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 0.0
},
"max_tokens": {
"description": "Soft cap on rendered tokens (approximate; extension uses a best-effort heuristic based on character count).",
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 0.0
},
"session_id": {
"type": "string"
},
"tab_id": {
"description": "Optional target tab. Defaults to the Agent Window's currently active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
}
}
}
@@ -0,0 +1,108 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "ObserveResult",
"type": "object",
"required": [
"ref_count",
"tab_id",
"text"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"ref_count": {
"description": "Number of `@e<N>` refs registered for this session by this observation. Equivalent to the size of the new ref-store map.",
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"tab_id": {
"description": "Tab the observation was computed for.",
"type": "integer",
"format": "int64"
},
"text": {
"description": "Semantic VOM observation text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent interaction tools.",
"type": "string"
},
"truncated": {
"description": "Whether the rendered tree was truncated because of `max_depth` / `max_tokens`. Agents may re-run with looser caps.",
"default": false,
"type": "boolean"
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}