源码快照 @ 2026-09-01 (source-only, 剔除非源码素材)
Creator App Tests / PR Spam Gate (push) Skipped
Frontend Tests / PR Spam Gate (push) Skipped
NPM Format (website & console) / PR Spam Gate (push) Skipped
Pre-commit Checks / PR Spam Gate (push) Skipped
QwenPaw Creator Plugin Release / check-version (push) Canceled after 0s
QwenPaw Creator Plugin Release / upload-oss (push) Canceled after 0s
Tests / PR Spam Gate (push) Skipped
CodeQL / CodeQL (python) (push) Canceled after 0s
CodeQL / CodeQL (typescript) (push) Canceled after 0s
Creator App Tests / Creator Backend Tests (push) Canceled after 0s
Creator App Tests / Creator UI Build & Tests (push) Canceled after 0s
E2E Smoke Tests / Playwright UI Smoke (push) Canceled after 0s
Frontend Tests / Detect console changes (push) Canceled after 0s
NPM Format (website & console) / website format check (push) Canceled after 0s
NPM Format (website & console) / console format check (push) Canceled after 0s
Pre-commit Checks / Pre-commit (push) Canceled after 0s
QwenPaw Data Plugin Release / check-version (push) Canceled after 0s
Tests / Detect code changes (push) Canceled after 0s
Frontend Tests / Vitest Unit Tests (push) Canceled after 0s
Frontend Tests / Frontend Summary (push) Canceled after 0s
QwenPaw Data Plugin Release / upload-oss (push) Canceled after 0s
Tests / Maintainer Approval (push) Canceled after 0s
Tests / Unit Tests - py3.11 - macos-latest (push) Canceled after 0s
Tests / Unit Tests - py3.11 - ubuntu-latest (push) Canceled after 0s
Tests / Unit Tests - py3.13 - ubuntu-latest (push) Canceled after 0s
Tests / Unit Tests - py3.11 - windows-latest (push) Canceled after 0s
Tests / Contract Tests - py3.11 - macos-latest (push) Canceled after 0s
Tests / Contract Tests - py3.11 - ubuntu-latest (push) Canceled after 0s
Tests / Contract Tests - py3.13 - ubuntu-latest (push) Canceled after 0s
Tests / Contract Tests - py3.11 - windows-latest (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - macos-latest - p0 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - macos-latest - p1 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - macos-latest - p2 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - ubuntu-latest - fallback (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - ubuntu-latest - p0 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - ubuntu-latest - p1 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - ubuntu-latest - p2 (push) Canceled after 0s
Tests / Integrated Tests - py3.13 - ubuntu-latest - p0 (push) Canceled after 0s
Tests / Integrated Tests - py3.13 - ubuntu-latest - p1 (push) Canceled after 0s
Tests / Integrated Tests - py3.13 - ubuntu-latest - p2 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - windows-latest - p0 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - windows-latest - p1 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - windows-latest - p2 (push) Canceled after 0s
Tests / Coverage Report (push) Canceled after 0s
Tests / Test Summary (push) Canceled after 0s

This commit is contained in:
agent-radar
2026-09-01 20:01:50 +08:00
commit 1a3a4ccde1
4652 changed files with 1891610 additions and 0 deletions
@@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
"""computer_use tool package (plugin-provided)."""
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from .dispatch import computer_use
__all__ = ["computer_use"]
def __getattr__(name: str):
"""Load the host-bound tool entry point only when it is requested."""
if name != "computer_use":
raise AttributeError(name)
from .dispatch import computer_use
return computer_use
@@ -0,0 +1,197 @@
# -*- coding: utf-8 -*-
"""Application access decisions owned by the Computer Use plugin."""
from __future__ import annotations
import json
import os
from dataclasses import asdict, dataclass
from pathlib import Path
from threading import RLock
from typing import Mapping
@dataclass(frozen=True)
class AppApprovalRequest:
"""A native-resolved application identity awaiting a user decision."""
request_id: str
session_id: str
canonical_app_id: str
display_name: str
identity_evidence: Mapping[str, str]
risk: str = "unknown"
warning: str = ""
@dataclass(frozen=True)
class AppAccessDecision:
"""A resolved Computer Use application access decision."""
allowed: bool
source: str
@dataclass(frozen=True)
class PersistentAppAccess:
"""One explicit, installation-scoped allow decision."""
canonical_app_id: str
display_name: str
def _normalize_app_id(app_id: str) -> str:
"""Collapse equivalent spellings of one application identifier.
Executable identifiers can arrive as either an extended-length
(``\\\\?\\``) path or a plain drive path, and with mixed casing across
the discovery sources. Stripping the prefix and folding case yields a
single key so an application is approved once per session regardless of
the spelling that reached the store.
"""
text = app_id.strip()
scheme, separator, remainder = text.partition(":")
if separator and scheme == "process":
verbatim_prefix = "\\\\?\\"
if remainder.startswith(verbatim_prefix):
remainder = remainder[len(verbatim_prefix) :]
return f"{scheme}:{remainder.lower()}"
# Only the Windows ``process:`` scheme is case-folded, because NTFS is
# case-insensitive. The macOS ``app:`` path is left verbatim on purpose: a
# case-sensitive APFS volume treats two casings as two files, so folding
# here would merge identifiers the filesystem keeps distinct -- the same
# mistake that once left canonical bundle paths unlaunchable. The path is
# already canonicalized at its source, so the casing is consistent without
# folding.
return text
def _default_persistent_path() -> Path:
from qwenpaw.constant import WORKING_DIR
return (
Path(WORKING_DIR)
/ "plugin_runtime"
/ "computer-use"
/ "app_access.json"
)
class ComputerUseAccessStore:
"""Keep session decisions in memory and explicit grants on this host."""
def __init__(self, persistent_path: Path | None = None) -> None:
self._lock = RLock()
self._persistent_path = persistent_path or _default_persistent_path()
self._session_decisions: dict[tuple[str, str], bool] = {}
self._persistent_decisions = self._load_persistent()
def resolve(self, request: AppApprovalRequest) -> AppAccessDecision | None:
"""Return an existing decision, or ``None`` when input is needed."""
app_id = _normalize_app_id(request.canonical_app_id)
session_key = (request.session_id, app_id)
with self._lock:
session_decision = self._session_decisions.get(session_key)
if session_decision is not None:
return AppAccessDecision(session_decision, "session")
if app_id in self._persistent_decisions:
return AppAccessDecision(True, "persistent")
return None
def record_session(
self,
request: AppApprovalRequest,
*,
allowed: bool,
) -> None:
"""Keep a decision for the current in-memory session only."""
app_id = _normalize_app_id(request.canonical_app_id)
with self._lock:
self._session_decisions[(request.session_id, app_id)] = allowed
def record_persistent(
self,
canonical_app_id: str,
display_name: str,
) -> None:
"""Persist an explicit allow for this QwenPaw installation."""
app_id = _normalize_app_id(canonical_app_id)
decision = PersistentAppAccess(app_id, display_name)
with self._lock:
self._persistent_decisions[app_id] = decision
self._save_persistent_locked()
def list_persistent(self) -> list[PersistentAppAccess]:
"""List application grants shared by this installation's agents."""
with self._lock:
decisions = list(self._persistent_decisions.values())
return sorted(
decisions,
key=lambda decision: decision.display_name.casefold(),
)
def revoke_persistent(self, canonical_app_id: str) -> bool:
"""Remove one persistent allow so future access asks again."""
app_id = _normalize_app_id(canonical_app_id)
with self._lock:
removed = self._persistent_decisions.pop(app_id, None)
if removed is not None:
self._save_persistent_locked()
return removed is not None
def _load_persistent(self) -> dict[str, PersistentAppAccess]:
try:
with self._persistent_path.open(encoding="utf-8") as file:
payload = json.load(file)
except (OSError, json.JSONDecodeError):
return {}
records = payload.get("allowed_apps", [])
if not isinstance(records, list):
return {}
return {
app_id: PersistentAppAccess(app_id, display_name)
for record in records
if isinstance(record, dict)
and (
app_id := _normalize_app_id(
str(record.get("canonical_app_id") or "").strip(),
)
)
and (
display_name := str(
record.get("display_name") or app_id,
).strip()
)
}
def _save_persistent_locked(self) -> None:
self._persistent_path.parent.mkdir(parents=True, exist_ok=True)
temporary_path = self._persistent_path.with_suffix(".tmp")
payload = {
"allowed_apps": [
asdict(decision) for decision in self.list_persistent()
],
}
temporary_path.write_text(
json.dumps(payload, indent=2, sort_keys=True),
encoding="utf-8",
)
os.replace(temporary_path, self._persistent_path)
_access_store: ComputerUseAccessStore | None = None
_access_store_lock = RLock()
def get_computer_use_access_store() -> ComputerUseAccessStore:
"""Return the process-wide Computer Use plugin access authority."""
global _access_store
# Double-checked: the fast path skips the lock once built, and the lock
# keeps two callers on different event-loop threads from each building a
# store and racing to install it. Under the GIL this is belt-and-braces,
# but the guarantee it leans on is the interpreter's, not the code's.
if _access_store is None:
with _access_store_lock:
if _access_store is None:
_access_store = ComputerUseAccessStore()
return _access_store
@@ -0,0 +1,164 @@
# -*- coding: utf-8 -*-
"""Translate native App approval requests into the Core approval service."""
from __future__ import annotations
import asyncio
from collections.abc import Mapping
from typing import Any
from qwenpaw.app.approvals import ApprovalRequestSummary, get_approval_service
from qwenpaw.config.context import (
get_current_session_id as get_tool_session_id,
)
from qwenpaw.constant import TOOL_GUARD_APPROVAL_TIMEOUT_SECONDS
from qwenpaw.security.tool_guard.approval import ApprovalDecision
from .access import (
AppApprovalRequest,
get_computer_use_access_store,
)
_POST_APPROVAL_SETTLE_SECONDS = 0.8
def _agent_context():
"""Load request-scoped context only while resolving an approval."""
from qwenpaw.app import agent_context
return agent_context
class ComputerUseApprovalCoordinator:
"""The plugin-side adapter for native-originated App approval events.
Holds no exemption state. A newly approved request waits out the native
recency window before returning, so the approval click does not cause a
spurious refusal while later user input remains guarded normally.
"""
async def decide(self, message: Mapping[str, Any]) -> dict[str, Any]:
"""Resolve one reverse approval request without widening its scope."""
request = self._app_request(message)
if request is None:
return {"allowed": False, "source": "invalid"}
if not self._matches_active_session(request):
return {"allowed": False, "source": "session_mismatch"}
store = get_computer_use_access_store()
existing = store.resolve(request)
if existing is not None:
return {"allowed": existing.allowed, "source": existing.source}
pending = await self._create_pending(request)
try:
decision = await get_approval_service().wait_for_approval(
pending.request_id,
TOOL_GUARD_APPROVAL_TIMEOUT_SECONDS,
)
except Exception: # noqa: BLE001 - approval failures deny by default
decision = ApprovalDecision.DENIED
allowed = decision == ApprovalDecision.APPROVED
store.record_session(request, allowed=allowed)
if allowed:
# The approving click is real user input. Let the native recency
# guard age it out instead of weakening that guard with a bypass.
await asyncio.sleep(_POST_APPROVAL_SETTLE_SECONDS)
return {"allowed": allowed, "source": "session"}
@staticmethod
def _matches_active_session(request: AppApprovalRequest) -> bool:
agent_context = _agent_context()
active_session = (
agent_context.get_current_session_id()
or get_tool_session_id()
or ""
)
return bool(active_session and active_session == request.session_id)
@staticmethod
def _app_request(message: Mapping[str, Any]) -> AppApprovalRequest | None:
params = message.get("params")
meta = message.get("meta")
if not isinstance(params, Mapping) or not isinstance(meta, Mapping):
return None
app_id = str(params.get("canonical_app_id") or "").strip()
session_id = str(meta.get("session_id") or "").strip()
request_id = str(message.get("request_id") or "").strip()
evidence = params.get("identity_evidence")
if (
not app_id
or not session_id
or not request_id
or not isinstance(evidence, Mapping)
):
return None
return AppApprovalRequest(
request_id=request_id,
session_id=session_id,
canonical_app_id=app_id,
display_name=str(params.get("display_name") or app_id),
identity_evidence={
str(key): str(value) for key, value in evidence.items()
},
risk=str(params.get("risk") or "unknown"),
warning=str(params.get("warning") or ""),
)
@staticmethod
async def _create_pending(request: AppApprovalRequest):
agent_context = _agent_context()
current_session = (
agent_context.get_current_session_id() or request.session_id
)
root_session = (
agent_context.get_current_root_session_id() or current_session
)
agent_id = agent_context.get_current_agent_id() or "unknown"
summary = (
f"Computer Use requests access to {request.display_name} "
"for this session."
)
return await get_approval_service().create_pending_summary(
session_id=current_session,
root_session_id=root_session,
owner_agent_id=agent_id,
user_id=agent_context.get_current_user_id() or "",
channel=agent_context.get_current_channel() or "",
agent_id=agent_id,
summary=ApprovalRequestSummary(
source_type="computer_use_app_access",
name="Computer Use",
severity="medium",
findings_count=1,
result_summary=summary,
payload={
"canonical_app_id": request.canonical_app_id,
"display_name": request.display_name,
"risk": request.risk,
},
),
timeout_seconds=TOOL_GUARD_APPROVAL_TIMEOUT_SECONDS,
extra={
"display": {
"tool_name": "Computer Use",
"tool_source": "app access",
"exact_target": f"{request.display_name} for this session",
"is_generalized": False,
},
"tool_call": {
"id": f"computer_use:{request.request_id}",
"name": "Computer Use",
"input": {
"canonical_app_id": request.canonical_app_id,
"display_name": request.display_name,
"risk": request.risk,
"warning": request.warning,
},
},
"computer_use_app": {
"canonical_app_id": request.canonical_app_id,
"display_name": request.display_name,
},
},
)
@@ -0,0 +1,662 @@
# -*- coding: utf-8 -*-
"""Controlled client for the host-managed Computer Use native runtime."""
from __future__ import annotations
import asyncio
import sys
import threading
import uuid
from collections.abc import Awaitable, Callable, Mapping
from typing import Any
from qwenpaw.app.computer_use import (
HostRuntimeProvider,
RuntimeCapability,
get_current_computer_use_turn_id,
)
from qwenpaw.config.context import (
get_current_session_id as get_tool_session_id,
)
from .approval import ComputerUseApprovalCoordinator
from .protocol import (
PROTOCOL_VERSION,
ComputerUseProtocolError,
NativeRequest,
parse_response,
)
from .transport import (
ComputerUseTransport,
UnixSocketTransport,
WindowsPipeTransport,
)
_DEFAULT_DEADLINE_MS = 10000
# The desktop host spawns the helper process while answering acquire; the
# first spawn after an update can be slowed by antivirus scanning, and
# frozen backends still use a short per-attempt socket timeout, so retry
# the idempotent acquire a few times to cover that cold-start window.
_ACQUIRE_ATTEMPTS = 5
# A capability can name a helper that exited after announcing readiness but
# before the client connected. Starting a native session is still idempotent
# at that point: only the hello handshake has been sent, so discard the dead
# endpoint and ask the host to start a fresh helper within the same tool call.
_START_ATTEMPTS = 3
# A broken connection can be replaced transparently only when repeating the
# request cannot change the desktop. Mutating requests may have reached the
# helper before the connection failed, so replaying them could act twice.
_READ_ONLY_METHODS = frozenset(
{"list_apps", "list_windows", "observe_window"},
)
_OBSERVED_METHODS = frozenset(
{
"click",
"close_window",
"drag",
"invoke_element",
"press_key",
"scroll",
"sequence",
"set_value",
"type_text",
},
)
_REQUEST_ATTEMPTS = 2
_BROKEN_TRANSPORT_ERRORS = frozenset(
{
"invalid_frame",
"request_timeout",
"runtime_disconnected",
"runtime_unavailable",
},
)
_RETRYABLE_READ_ONLY_ERRORS = _BROKEN_TRANSPORT_ERRORS | {
# ScreenCaptureKit can miss its first callback while the capture service
# wakes up. Observation is side-effect free, so one retry is safe; input
# methods still never replay after an ambiguous failure.
"capture_failed",
}
_DEAD_ENDPOINT_ERRORS = frozenset(
{"invalid_frame", "runtime_disconnected", "runtime_unavailable"},
)
# The helper refuses rather than queues when another session holds the desktop,
# so the waiting is done here. Five attempts with doubling delays give a little
# over two seconds, which covers one action's worth of contention; beyond that
# the model is better told the desktop is busy than left waiting.
_DESKTOP_BUSY_ATTEMPTS = 5
_DESKTOP_BUSY_DELAY_SECONDS = 0.15
_ACQUIRE_RETRY_DELAY_SECONDS = 0.5
TransportFactory = Callable[[], ComputerUseTransport]
class ComputerUseClient:
"""Own one authenticated native connection for one QwenPaw session."""
def __init__(
self,
session_id: str,
transport_factory: TransportFactory | None = None,
) -> None:
self._session_id = session_id
self._transport_factory = transport_factory
self._transport: ComputerUseTransport | None = None
# The capability this client's transport was built from, kept so a dead
# endpoint can be reported back rather than reconnected to forever.
self._capability: RuntimeCapability | None = None
self._turn_id: str | None = None
# Native observations are concurrency tokens, not model input. Keeping
# the current token beside the connection prevents an agent from
# copying, guessing, or crossing it with another action while native
# still rejects stale state at the trust boundary.
self._observation_id: str | None = None
# The turn a stop applied to. Kept client-side so requests already
# queued on this lock cannot cross the turn boundary after Native has
# released its per-connection state.
self._stopped_turn: str | None = None
self._lock = asyncio.Lock()
# The loop that created the transport, its reader task and this lock.
# They may only be touched from there, and control routes arrive on the
# HTTP server's loop instead.
self._loop: asyncio.AbstractEventLoop | None = None
self._approvals = ComputerUseApprovalCoordinator()
async def execute(
self,
method: str,
params: Mapping[str, Any],
*,
deadline_ms: int = _DEFAULT_DEADLINE_MS,
) -> dict[str, Any]:
"""Execute one native operation through the authenticated transport."""
turn_id = get_current_computer_use_turn_id()
if not turn_id:
raise ComputerUseProtocolError(
"turn_unavailable",
"Computer Use is unavailable outside an active agent turn.",
)
self._check_turn_active(turn_id)
async with self._lock:
# A stop can arrive while this request is queued behind another
# operation. Recheck after acquiring the lock so queued work cannot
# cross the turn boundary and recreate a transport afterwards.
self._check_turn_active(turn_id, clear_previous=True)
transport = await self._ensure_transport()
if self._turn_id and self._turn_id != turn_id:
await self._end_turn(transport, self._turn_id)
self._observation_id = None
self._turn_id = turn_id
native_params = self._native_params(method, params)
request_attempts = (
_REQUEST_ATTEMPTS if method in _READ_ONLY_METHODS else 1
)
for request_attempt in range(request_attempts):
try:
result = await self._request_with_contention(
transport,
method,
native_params,
turn_id,
deadline_ms,
)
self._check_turn_active(turn_id)
return self._accept_result(method, result)
except asyncio.CancelledError:
# The native operation may already have changed the
# desktop. Stop this turn and abandon only its connection;
# the shared helper lets the bounded operation finish and
# remains available to other sessions.
self._stopped_turn = turn_id
await self._discard_transport()
raise
except ComputerUseProtocolError as error:
# A failed observation never replaces the old snapshot, and
# a failed action may have changed the desktop before it
# reported the error. Require fresh state in both cases.
if (
method == "observe_window"
or method in _OBSERVED_METHODS
):
self._observation_id = None
if error.code == "request_timeout":
if method not in _READ_ONLY_METHODS:
self._stopped_turn = turn_id
await self._discard_transport()
elif error.code in _BROKEN_TRANSPORT_ERRORS:
await self._discard_transport()
if error.code in _DEAD_ENDPOINT_ERRORS:
# The endpoint is unusable, so the next acquire must
# ask the host to verify or restart the helper.
self._forget_capability()
if (
error.code not in _RETRYABLE_READ_ONLY_ERRORS
or request_attempt + 1 >= request_attempts
or turn_id == self._stopped_turn
):
raise
transport = await self._ensure_transport()
self._turn_id = turn_id
raise ComputerUseProtocolError(
"runtime_unavailable",
"Computer Use native runtime is unavailable.",
)
def _check_turn_active(
self,
turn_id: str,
*,
clear_previous: bool = False,
) -> None:
"""Reject the stopped turn and optionally retire an older marker."""
if turn_id == self._stopped_turn:
self._observation_id = None
raise ComputerUseProtocolError(
"turn_stopped",
"Computer Use was stopped for this turn.",
)
if clear_previous:
self._stopped_turn = None
def _native_params(
self,
method: str,
params: Mapping[str, Any],
) -> dict[str, Any]:
"""Attach the current native observation without exposing its ID."""
native_params = dict(params)
if method not in _OBSERVED_METHODS:
return native_params
if not self._observation_id:
raise ComputerUseProtocolError(
"observation_required",
"Observe a window before performing this action.",
)
native_params["observation_id"] = self._observation_id
return native_params
def _accept_result(
self,
method: str,
result: Mapping[str, Any],
) -> dict[str, Any]:
"""Advance the observation and return only model-facing data."""
public_result = dict(result)
observation_id = public_result.pop("observation_id", None)
if method == "observe_window" or method in _OBSERVED_METHODS:
self._observation_id = (
observation_id
if isinstance(observation_id, str) and observation_id
else None
)
elif method == "launch_app":
self._observation_id = None
return public_result
async def _request_with_contention(
self,
transport: ComputerUseTransport,
method: str,
params: Mapping[str, Any],
turn_id: str,
deadline_ms: int,
) -> dict[str, Any]:
"""Send once, except for refusals known to happen before any action."""
for attempt in range(_DESKTOP_BUSY_ATTEMPTS):
request = NativeRequest(
request_id=uuid.uuid4().hex,
method=method,
params=params,
session_id=self._session_id,
turn_id=turn_id,
deadline_ms=max(100, deadline_ms),
)
try:
return parse_response(
await transport.request(request.to_message()),
)
except ComputerUseProtocolError as error:
if error.code != "desktop_busy":
raise
if attempt + 1 >= _DESKTOP_BUSY_ATTEMPTS:
raise
# Another session holds the desktop. The helper refuses before
# touching it, so retrying cannot duplicate an action. Waiting
# here also lets a stop interrupt the retry promptly.
await asyncio.sleep(
_DESKTOP_BUSY_DELAY_SECONDS * (2**attempt),
)
if turn_id == self._stopped_turn:
raise ComputerUseProtocolError(
"turn_stopped",
"Computer Use was stopped for this turn.",
) from error
raise ComputerUseProtocolError(
"desktop_busy",
"Another Computer Use session is using the desktop.",
)
@property
def has_active_turn(self) -> bool:
"""Whether this session currently owns a native Computer Use turn."""
return self._transport is not None and self._turn_id is not None
async def stop_turn(self) -> bool:
"""Stop this session's active turn without affecting other clients."""
return await self._on_owner_loop(self._stop_turn_here)
async def _stop_turn_here(self) -> bool:
"""Stop this turn after any already-dispatched operation settles."""
turn_id = self._turn_id
if self._transport is None or not turn_id:
return False
# Mark before waiting: requests already queued on the lock will see the
# marker when they acquire it and must not reach the native connection.
self._stopped_turn = turn_id
self._observation_id = None
async with self._lock:
transport = self._transport
if transport is not None and self._turn_id == turn_id:
await self._end_turn(transport, turn_id)
self._turn_id = None
return True
async def close(self) -> None:
"""End the active turn and close the client transport."""
await self._on_owner_loop(self._close_here)
async def end_turn(self) -> bool:
"""Release the native turn this session has finished with.
Keeps the connection, since the next turn will want it: the helper
drops
the turn's screenshots and accessibility handles and carries on
serving.
"""
return await self._on_owner_loop(self._end_turn_here)
async def _end_turn_here(self) -> bool:
async with self._lock:
transport = self._transport
turn_id = self._turn_id
if transport is None or not turn_id:
return False
self._turn_id = None
self._observation_id = None
await self._end_turn(transport, turn_id)
return True
@property
def owner_loop(self) -> asyncio.AbstractEventLoop | None:
"""The loop this client's transport and lock belong to, if
connected."""
return self._loop
async def _close_here(self) -> None:
transport = self._transport
if transport is None:
return
try:
if self._turn_id:
await self._end_turn(transport, self._turn_id)
finally:
self._turn_id = None
self._observation_id = None
self._transport = None
await transport.close()
async def _on_owner_loop(
self,
operation: Callable[[], Awaitable[Any]],
) -> Any:
"""Run a client operation on the loop that owns its asyncio state.
The host runs one event loop per workspace, each on its own thread, and
the control routes run on the HTTP server's loop. The transport's
streams, its reader task and this client's lock all belong to whichever
loop built them, so a coroutine touching them is handed back there
rather than awaited here.
"""
loop = self._loop
if loop is None or loop is asyncio.get_running_loop():
return await operation()
try:
handle = asyncio.run_coroutine_threadsafe(operation(), loop)
except RuntimeError:
# The owning loop is gone, so its transport is unusable anyway.
self._transport = None
self._turn_id = None
return None
return await asyncio.wrap_future(handle)
async def _ensure_transport(self) -> ComputerUseTransport:
if self._transport is not None:
return self._transport
# Everything created below belongs to this loop, so record it before
# anything else can be asked to touch it from elsewhere.
self._loop = asyncio.get_running_loop()
if self._transport_factory is not None:
transport = self._transport_factory()
transport.set_reverse_request_handler(self._approvals.decide)
await transport.connect()
self._transport = transport
return transport
for attempt in range(_START_ATTEMPTS):
capability = await self._acquire_capability()
if capability is None:
raise ComputerUseProtocolError(
"runtime_unavailable",
"Computer Use native runtime is unavailable.",
)
transport = (
WindowsPipeTransport(capability)
if sys.platform == "win32"
else UnixSocketTransport(capability)
)
# Remembered so a dead endpoint can be reported back to the
# provider; the next acquire then asks the host for a live one.
self._capability = capability
transport.set_reverse_request_handler(self._approvals.decide)
try:
await transport.connect()
except ComputerUseProtocolError as error:
try:
await transport.close()
except Exception: # noqa: BLE001 - preserve connect failure
pass
# The endpoint named by this capability did not answer, which
# is what a helper that has gone away looks like from here. A
# later acquire must ask the host again.
self._forget_capability()
if (
error.code
not in {
"invalid_frame",
"request_timeout",
"runtime_disconnected",
"runtime_unavailable",
}
or attempt + 1 >= _START_ATTEMPTS
):
raise
await asyncio.sleep(_ACQUIRE_RETRY_DELAY_SECONDS)
continue
self._transport = transport
return transport
raise ComputerUseProtocolError(
"runtime_unavailable",
"Computer Use native runtime is unavailable.",
)
def _forget_capability(self) -> None:
"""Report this client's endpoint as dead, so a fresh one is issued."""
capability, self._capability = self._capability, None
if capability is not None:
HostRuntimeProvider.invalidate_capability(capability)
@staticmethod
async def _acquire_capability():
"""Acquire the host capability, retrying cold-start misses."""
for attempt in range(_ACQUIRE_ATTEMPTS):
# The provider call blocks on a control socket; keep it off the
# event loop so other sessions stay responsive.
capability = await asyncio.to_thread(
HostRuntimeProvider.acquire_capability,
)
if capability is not None:
if capability.protocol_version != PROTOCOL_VERSION:
raise ComputerUseProtocolError(
"protocol_mismatch",
"Computer Use plugin and desktop runtime "
"versions are incompatible.",
)
return capability
if attempt + 1 < _ACQUIRE_ATTEMPTS:
await asyncio.sleep(_ACQUIRE_RETRY_DELAY_SECONDS)
return None
async def _end_turn(
self,
transport: ComputerUseTransport,
turn_id: str,
) -> None:
request = NativeRequest(
request_id=uuid.uuid4().hex,
method="end_turn",
params={},
session_id=self._session_id,
turn_id=turn_id,
deadline_ms=2000,
)
try:
parse_response(await transport.request(request.to_message()))
except ComputerUseProtocolError:
pass
async def _discard_transport(self) -> None:
"""Detach and close the current transport, ignoring shutdown errors."""
transport = self._transport
self._transport = None
self._turn_id = None
self._observation_id = None
if transport is None:
return
try:
await transport.close()
except Exception:
# Closing a broken pipe can raise transport errors; ignore them so
# the caller can re-raise its own original failure.
pass
_clients: dict[str, ComputerUseClient] = {}
# The cache is read and written from more than one event loop -- the host runs
# one per workspace on its own thread -- so a plain dict could be mutated while
# another thread iterates it during eviction. The lock covers the get-or-create
# and eviction paths; per-client work happens outside it, guarded by the
# client's own async lock.
_clients_lock = threading.Lock()
# A client caches the per-session native turn that outlives a single tool call.
# Nothing tells the plugin when a session is gone, so the cache is bounded
# instead: on insert, idle sessions are dropped oldest-first. The backend is a
# long-lived desktop process, so an unbounded dict would keep every session
# ever seen.
_MAX_CACHED_CLIENTS = 64
def _retire(client: ComputerUseClient) -> None:
"""Close an evicted client's connection, best effort.
Dropping the reference alone would leave the pipe or socket open until the
object happened to be collected. Closing needs to await, and this runs from
synchronous code holding a threading lock, so the coroutine is handed to
the
loop that owns the transport and not waited on.
"""
loop = client.owner_loop
if loop is None:
return
try:
asyncio.run_coroutine_threadsafe(client.close(), loop)
except RuntimeError:
# That loop has stopped, so its transport is already unusable.
pass
def _evict_idle_clients() -> list[ComputerUseClient]:
"""Drop cached clients for sessions with no turn in flight.
The caller holds ``_clients_lock``. Returns the clients removed so the
caller can close them outside the lock.
"""
if len(_clients) < _MAX_CACHED_CLIENTS:
return []
evicted = []
for session_id, client in list(_clients.items()):
if len(_clients) < _MAX_CACHED_CLIENTS:
break
if not client.has_active_turn:
del _clients[session_id]
evicted.append(client)
return evicted
def get_computer_use_client() -> ComputerUseClient:
"""Return the controlled client for the active QwenPaw session."""
# The request-context module initializes the web workspace stack. Defer it
# until session lookup so transport and protocol code remain lightweight.
from qwenpaw.app.agent_context import get_current_session_id
session_id = get_current_session_id() or get_tool_session_id() or ""
if not session_id:
raise ComputerUseProtocolError(
"session_unavailable",
"Computer Use requires an active session.",
)
with _clients_lock:
client = _clients.get(session_id)
evicted: list[ComputerUseClient] = []
if client is None:
evicted = _evict_idle_clients()
if len(_clients) >= _MAX_CACHED_CLIENTS:
# Every cached session still claims a turn. Refusing keeps the
# bound real and makes the situation visible, where growing the
# cache would quietly hold a connection per session forever.
raise ComputerUseProtocolError(
"too_many_sessions",
"Too many Computer Use sessions are active; "
"finish or stop one before starting another.",
)
client = ComputerUseClient(session_id)
_clients[session_id] = client
for retired in evicted:
_retire(retired)
return client
def _cached_client(session_id: str) -> ComputerUseClient | None:
"""Look up a session's client under the cache lock.
Control routes reach the cache from the HTTP server's thread while a
workspace thread may be inserting or evicting, so every read takes the lock
the rest of this module already uses.
"""
with _clients_lock:
return _clients.get(session_id)
def is_computer_use_active(session_id: str) -> bool:
"""Return whether a session owns an active native Computer Use turn."""
client = _cached_client(session_id)
return client.has_active_turn if client is not None else False
async def stop_computer_use_session(session_id: str) -> bool:
"""Stop the native Computer Use turn currently owned by one session."""
client = _cached_client(session_id)
return await client.stop_turn() if client is not None else False
async def end_computer_use_turn(session_id: str) -> bool:
"""Release the native turn a finished request was holding.
The turn id is minted per request by the host, and nothing used to retire
it: a session that used the tool once kept its turn -- and the helper's
screenshots and accessibility handles -- until the next call happened to
supply a new id. That also made the cache bound unreachable, since a client
holding a turn is never evicted.
"""
client = _cached_client(session_id)
return await client.end_turn() if client is not None else False
def known_computer_use_sessions() -> list[str]:
"""Every session this process holds a Computer Use client for.
A pending approval can only exist for one of these: the helper asks through
the connection a client owns, so the request carries that client's session.
Turning the feature off therefore has to reach all of them, not only the
session whoever flipped the switch happened to be looking at.
"""
with _clients_lock:
return list(_clients)
async def stop_all_computer_use_turns() -> int:
"""Stop every active native turn across all known sessions.
Used when the feature is switched off so no automation keeps running.
Returns the number of turns that were actually stopped.
"""
stopped = 0
# Snapshot under the lock, then stop turns without holding it: stop_turn
# awaits native I/O, and the lock is a sync primitive that must not be held
# across an await.
with _clients_lock:
clients = list(_clients.values())
for client in clients:
if await client.stop_turn():
stopped += 1
return stopped
@@ -0,0 +1,553 @@
# -*- coding: utf-8 -*-
"""Thin tool adapter for the host-managed Computer Use runtime."""
# NOTE: no `from __future__ import annotations` here, deliberately. The tool
# entry point below is handed to the runtime's JSON-schema builder, which
# resolves annotations in a namespace without our typing imports; stringized
# annotations would abort the toolkit build. Under Python 3.11 every
# annotation in this module evaluates fine at definition time.
import asyncio
import json
import logging
import threading
import time
from typing import Any, Literal, Mapping
from agentscope.message import (
Base64Source,
DataBlock,
TextBlock,
ToolResultState,
URLSource,
)
from agentscope.tool import ToolChunk
from qwenpaw.runtime.tool_registry import tool_descriptor
from .client import get_computer_use_client
from .feature_state import get_computer_use_feature_state
from .protocol import ComputerUseProtocolError
_LOGGER = logging.getLogger(__name__)
_MAX_ACTIONS_PER_MINUTE = 60
_action_times: list[float] = []
_rate_limit_lock = threading.Lock()
_SCREENSHOT_URL_PLACEHOLDER = "<image delivered as a separate attachment>"
_MAX_ACCESSIBILITY_DEPTH = 40
ComputerUseAction = Literal[
"list_apps",
"list_windows",
"observe_window",
"launch_app",
"close_window",
"click",
"double_click",
"right_click",
"scroll",
"drag",
"type",
"press_key",
"sequence",
"invoke",
"begin_text_edit",
"set_value",
"wait",
"stop",
]
def _check_rate_limit(cost: int = 1) -> None:
# The tool can be entered from more than one event loop -- the host runs
# per-workspace loops on their own threads -- so the guard is a threading
# lock rather than an asyncio one, which serialises only within a single
# loop. Under the GIL the unguarded check-then-append is narrow enough that
# overshooting the cap could not be provoked, but that is a property of the
# interpreter rather than of this code, and a free-threaded build removes
# it. The body has no await, so the lock is held briefly.
with _rate_limit_lock:
now = time.monotonic()
_action_times[:] = [
value for value in _action_times if now - value < 60
]
if len(_action_times) + cost > _MAX_ACTIONS_PER_MINUTE:
raise ComputerUseProtocolError(
"rate_limited",
"Computer Use rate limit exceeded; wait before continuing.",
)
_action_times.extend([now] * cost)
def _sequence_steps(steps: Any) -> list[dict[str, str]]:
"""Validate the bounded keyboard-only sequence contract."""
if isinstance(steps, str):
try:
steps = json.loads(steps)
except json.JSONDecodeError as error:
raise ValueError(
"sequence steps must be a JSON array.",
) from error
if not isinstance(steps, list) or not 1 <= len(steps) <= 20:
raise ValueError("sequence requires 1 to 20 steps.")
normalized = []
text_length = 0
for index, step in enumerate(steps):
if not isinstance(step, Mapping):
raise ValueError(f"sequence step {index} must be an object.")
action = str(step.get("action") or "").strip().lower()
if action not in {"type", "press_key"}:
raise ValueError(
f"sequence step {index} must use type or press_key.",
)
field = "text" if action == "type" else "key"
value = step.get(field)
if (
not isinstance(value, str)
or not value
or (action == "press_key" and not value.strip())
):
raise ValueError(
f"sequence step {index} requires non-empty {field}.",
)
if set(step) != {"action", field}:
raise ValueError(
f"sequence step {index} accepts only action and {field}.",
)
if action == "type":
text_length += len(value)
if text_length > 512:
raise ValueError("sequence text is limited to 512 characters.")
normalized.append({"action": action, field: value})
return normalized
def _without_screenshot_urls(
payload: Mapping[str, Any],
*,
attached: bool,
) -> Mapping[str, Any]:
"""Remove image data from text output, retaining metadata when attached.
Screenshots are attached as image blocks; repeating the base64 data
URL inside the JSON text block would double a multi-megabyte payload
and pollute the model's text context. Post-action refreshes are semantic
and do not capture images; a visual target starts with a fresh observation.
"""
screenshots = payload.get("screenshots")
if not isinstance(screenshots, list):
return payload
if not attached:
return {
key: value
for key, value in payload.items()
if key != "screenshots"
}
sanitized: list[Any] = []
for screenshot in screenshots:
if isinstance(screenshot, Mapping) and "url" in screenshot:
sanitized.append(
{**screenshot, "url": _SCREENSHOT_URL_PLACEHOLDER},
)
else:
sanitized.append(screenshot)
return {**payload, "screenshots": sanitized}
def _element_line(element: Mapping[str, Any]) -> str:
"""Render one accessibility element as a single compact line.
Only the model reads these elements, so the JSON scaffolding around
them is pure overhead. Coordinates come from the current screenshot;
accessibility lines expose only semantic element metadata.
"""
parts = [
str(element.get("id") or "?"),
str(element.get("control_type_name") or element.get("role") or "?"),
f'"{element.get("name") or ""}"',
]
value = element.get("value")
if isinstance(value, str) and value:
parts.append(f"={value}")
identifier = element.get("identifier") or element.get("automation_id")
if isinstance(identifier, str) and identifier:
parts.append(f"[identifier={identifier}]")
# Both states stay visible: an offscreen entry may become reachable
# after scrolling, and a disabled control tells the model not to try.
if element.get("enabled") is False:
parts.append("[disabled]")
if element.get("offscreen") is True:
parts.append("[offscreen]")
if element.get("selected") is True:
parts.append("[selected]")
if element.get("settable") is True:
parts.append("[settable]")
if element.get("resource_backed") is True:
parts.append("[resource-backed]")
actions = element.get("actions")
if isinstance(actions, list):
names = [str(action) for action in actions if str(action)]
if names:
parts.append(f"[actions={','.join(names)}]")
depth = element.get("depth")
indent = (
" " * min(depth, _MAX_ACCESSIBILITY_DEPTH)
if isinstance(depth, int) and not isinstance(depth, bool) and depth > 0
else ""
)
return indent + " ".join(parts)
def _with_compact_elements(payload: Mapping[str, Any]) -> Mapping[str, Any]:
"""Replace the accessibility element objects with one line each."""
accessibility = payload.get("accessibility")
if not isinstance(accessibility, Mapping):
return payload
elements = accessibility.get("elements")
if not isinstance(elements, list):
return payload
lines = [
_element_line(element)
for element in elements
if isinstance(element, Mapping)
]
compact = {
key: value for key, value in accessibility.items() if key != "elements"
}
compact["elements"] = "\n".join(lines)
return {**payload, "accessibility": compact}
def _screenshot_source(url: str) -> Base64Source | URLSource:
header, separator, data = url.partition(",")
if (
separator
and header.casefold().startswith("data:")
and ";base64" in header.casefold()
):
media_type = header[5:].split(";", 1)[0] or "image/*"
return Base64Source(data=data, media_type=media_type)
return URLSource(url=url, media_type="image/*")
def _response(
payload: Mapping[str, Any],
*,
include_images: bool = False,
state: ToolResultState = ToolResultState.SUCCESS,
) -> ToolChunk:
content: list[Any] = []
if include_images:
for screenshot in payload.get("screenshots", []):
if isinstance(screenshot, Mapping) and isinstance(
screenshot.get("url"),
str,
):
content.append(
DataBlock(
source=_screenshot_source(screenshot["url"]),
),
)
content.append(
TextBlock(
type="text",
text=json.dumps(
_with_compact_elements(
_without_screenshot_urls(payload, attached=include_images),
),
ensure_ascii=False,
separators=(",", ":"),
),
),
)
return ToolChunk(content=content, state=state, is_last=True)
def _error(code: str, message: str) -> ToolChunk:
payload = {
"ok": False,
"error": {"code": code, "message": message},
}
if code in {
"desktop_busy",
"focus_failed",
"input_failed",
"observation_required",
"stale_observation",
"target_not_at_point",
"unknown_screenshot",
"user_intervention",
}:
payload["requires_observe"] = True
payload["next_action"] = "observe_window"
elif code in {"stale_window", "window_not_found"}:
payload["requires_observe"] = True
payload["next_action"] = "list_windows"
return _response(
payload,
state=ToolResultState.ERROR,
)
@tool_descriptor(
name="computer_use",
enabled_by_default=True,
async_execution=True,
description=(
"Control approved desktop applications through the native "
"Computer Use runtime. Observe a window before acting; the runtime "
"keeps the current observation synchronized between actions."
),
requires_skills=("computer_use",),
)
async def computer_use(
action: ComputerUseAction,
app: str = "",
window_id: str = "",
screenshot_id: str = "",
element_id: str = "",
x: int = 0,
y: int = 0,
start_x: int = 0,
start_y: int = 0,
end_x: int = 0,
end_y: int = 0,
source_element_id: str = "",
target_element_id: str = "",
button: str = "left",
count: int = 1,
delta_y: int = 0,
text: str = "",
value: str = "",
key: str = "",
steps: list[dict[str, Any]] | str | None = None,
wait_ms: int = 500,
timeout_ms: int = 10000,
) -> ToolChunk:
"""Control one observed window at a time.
Use ``list_apps`` or ``list_windows`` first. Observe a target with
``observe_window`` before acting. The client advances the native
observation after every successful action; native rejects stale state.
``launch_app`` accepts an App ID returned by ``list_apps`` or an absolute
platform-native application path.
``observe_window`` returns screenshots and accessibility text.
Coordinate actions require the ``id`` of an attached screenshot as
``screenshot_id``; coordinates are local to that image.
Inspect the replacement observation after an action changes selection,
focus, menus, editors, dialogs, or windows. Confirm editable focus before
typing, and observe again after committing an edit.
"""
# Each early return maps to one refusal reason the model must be able to
# tell apart, so they are reported individually rather than merged.
# pylint: disable=too-many-return-statements
try:
action = str(action or "").strip().lower()
if not action:
raise ValueError("action is required.")
if not get_computer_use_feature_state().is_enabled():
return _error(
"feature_disabled",
"Computer Use is turned off. Enable it in the Computer Use "
"panel to allow desktop automation.",
)
if action == "wait":
_check_rate_limit()
await asyncio.sleep(max(0, min(wait_ms, 30_000)) / 1000)
return _response(
{"ok": True, "action": action, "waited_ms": wait_ms},
)
client = get_computer_use_client()
if action == "stop":
_check_rate_limit()
await client.stop_turn()
return _response({"ok": True, "action": action})
method, params, include_images = _native_request(
action,
app=app,
window_id=window_id,
screenshot_id=screenshot_id,
element_id=element_id,
x=x,
y=y,
start_x=start_x,
start_y=start_y,
end_x=end_x,
end_y=end_y,
source_element_id=source_element_id,
target_element_id=target_element_id,
button=button,
count=count,
delta_y=delta_y,
text=text,
value=value,
key=key,
steps=steps,
)
if method == "sequence":
_check_rate_limit(len(params["steps"]))
else:
_check_rate_limit()
result = await client.execute(
method,
params,
deadline_ms=max(100, min(timeout_ms, 30_000)),
)
failed = action == "sequence" and isinstance(
result.get("error"),
Mapping,
)
payload = {"ok": not failed, "action": action, **result}
return _response(
payload,
include_images=include_images,
state=ToolResultState.ERROR if failed else ToolResultState.SUCCESS,
)
except ComputerUseProtocolError as error:
return _error(error.code, str(error))
except ValueError as error:
return _error("invalid_request", str(error))
except (
Exception
) as error: # noqa: BLE001 - tool calls must not escape errors
# A tool entry point must not raise, but the errors that reach here are
# the unexpected ones -- an attribute error, a bad type, a broken
# import -- not the protocol failures handled above. Collapsing them to
# one message keeps the turn alive; logging the traceback first keeps
# them diagnosable rather than lost behind "Computer Use failed".
_LOGGER.exception("Computer Use tool call failed unexpectedly")
return _error("tool_failed", f"Computer Use failed: {error}")
def _native_request(
action: str,
**values: Any,
) -> tuple[str, dict[str, Any], bool]:
# One branch per action keeps the whole request contract readable in a
# single place; splitting it per action would scatter the protocol.
# pylint: disable=too-many-return-statements
# pylint: disable=too-many-branches, too-many-statements
if action == "list_apps":
return action, {}, False
if action == "list_windows":
app = str(values["app"] or "").strip()
return action, ({"app": app} if app else {}), False
if action == "launch_app":
app = str(values["app"] or "").strip()
if not app:
raise ValueError(
"launch_app requires an App ID or an absolute .exe path.",
)
return action, {"app": app}, False
if action == "observe_window":
window_id = str(values["window_id"] or "").strip()
if not window_id:
raise ValueError(
"observe_window requires window_id from list_windows.",
)
return action, {"window_id": window_id}, True
if action == "close_window":
return action, {}, False
if action in {"click", "double_click", "right_click"}:
params = {}
element_id = str(values.get("element_id") or "").strip()
if element_id:
params["element_id"] = element_id
else:
params["screenshot_id"] = _screenshot_id(values)
params["x"] = values["x"]
params["y"] = values["y"]
params["button"] = (
"right" if action == "right_click" else values["button"]
)
params["count"] = 2 if action == "double_click" else values["count"]
return "click", params, False
if action == "scroll":
params = {
"screenshot_id": _screenshot_id(values),
"x": values["x"],
"y": values["y"],
}
params["delta_y"] = values["delta_y"]
return action, params, False
if action == "drag":
source_element_id = str(
values.get("source_element_id") or "",
).strip()
target_element_id = str(
values.get("target_element_id") or "",
).strip()
if bool(source_element_id) != bool(target_element_id):
raise ValueError(
"drag requires both source_element_id and "
"target_element_id, or neither.",
)
params = {}
if source_element_id:
params.update(
source_element_id=source_element_id,
target_element_id=target_element_id,
)
else:
params.update(
screenshot_id=_screenshot_id(values),
start_x=values["start_x"],
start_y=values["start_y"],
end_x=values["end_x"],
end_y=values["end_y"],
)
return action, params, False
if action == "type":
text = str(values["text"] or "")
if not text:
raise ValueError("type requires non-empty text.")
return (
"type_text",
{"text": text},
False,
)
if action in {"invoke", "begin_text_edit", "set_value"}:
element_id = str(values["element_id"] or "").strip()
if not element_id:
raise ValueError(
f"{action} requires element_id from observe_window.",
)
params = {"element_id": element_id}
if action == "begin_text_edit":
params["expects_text_input"] = True
if action == "set_value":
params["value"] = str(values["value"] or "")
return (
(
"invoke_element"
if action in {"invoke", "begin_text_edit"}
else action
),
params,
False,
)
if action == "press_key":
key = str(values["key"] or "").strip()
if not key:
raise ValueError("press_key requires key.")
return action, {"key": key}, False
if action == "sequence":
return action, {"steps": _sequence_steps(values.get("steps"))}, False
raise ValueError(
"Unknown action. Valid actions: list_apps, list_windows, "
"observe_window, launch_app, close_window, click, "
"double_click, right_click, scroll, drag, type, press_key, invoke, "
"begin_text_edit, set_value, sequence, wait, stop.",
)
def _screenshot_id(values: Mapping[str, Any]) -> str:
screenshot_id = str(values.get("screenshot_id") or "").strip()
if not screenshot_id:
raise ValueError(
"Coordinate input requires screenshot_id from observe_window.",
)
return screenshot_id
@@ -0,0 +1,106 @@
# -*- coding: utf-8 -*-
"""Installation-scoped on/off switch for the Computer Use feature."""
from __future__ import annotations
import json
import logging
import os
from pathlib import Path
from threading import RLock
_LOGGER = logging.getLogger(__name__)
def _default_state_path() -> Path:
from qwenpaw.constant import WORKING_DIR
return (
Path(WORKING_DIR)
/ "plugin_runtime"
/ "computer-use"
/ "feature_state.json"
)
class ComputerUseFeatureState:
"""Persist whether the Computer Use feature is allowed on this host.
The feature is enabled by default so existing installations keep their
current behaviour; the user can turn it off from the plugin page. The
flag is installation-scoped and survives restarts.
That default applies only when no decision has been recorded. A state file
that exists but cannot be read is an unknown, not a default, and an unknown
resolves to off -- the alternative is quietly restoring desktop access that
the user may have switched off.
"""
def __init__(self, persistent_path: Path | None = None) -> None:
self._lock = RLock()
self._persistent_path = persistent_path or _default_state_path()
self._enabled = self._load()
def is_enabled(self) -> bool:
"""Return whether desktop automation is currently allowed."""
with self._lock:
return self._enabled
def set_enabled(self, value: bool) -> None:
"""Persist a new enabled/disabled decision for this installation."""
with self._lock:
self._enabled = bool(value)
self._save_locked()
def _load(self) -> bool:
try:
with self._persistent_path.open(encoding="utf-8") as file:
payload = json.load(file)
except FileNotFoundError:
# No decision has been recorded yet, so the installation default
# applies. Nothing is lost by starting from it: the feature being
# on means an agent may ask, and every action on an application
# still waits for the user to approve that application.
return True
except (OSError, json.JSONDecodeError) as error:
# A file that exists but cannot be read is different: the user may
# have turned the feature off, and reading that as on would put a
# decision about desktop access back the way they did not choose.
_LOGGER.warning(
"Computer Use feature state at %s is unreadable (%s); "
"treating the feature as off until it is set again.",
self._persistent_path,
error,
)
return False
enabled = payload.get("enabled")
if isinstance(enabled, bool):
return enabled
# Parsed, but says nothing this code understands -- the same unknown as
# an unreadable file.
_LOGGER.warning(
"Computer Use feature state at %s has no usable 'enabled' flag; "
"treating the feature as off until it is set again.",
self._persistent_path,
)
return False
def _save_locked(self) -> None:
self._persistent_path.parent.mkdir(parents=True, exist_ok=True)
temporary_path = self._persistent_path.with_suffix(".tmp")
temporary_path.write_text(
json.dumps({"enabled": self._enabled}, indent=2, sort_keys=True),
encoding="utf-8",
)
os.replace(temporary_path, self._persistent_path)
_feature_state: ComputerUseFeatureState | None = None
def get_computer_use_feature_state() -> ComputerUseFeatureState:
"""Return the process-wide Computer Use feature switch."""
global _feature_state
if _feature_state is None:
_feature_state = ComputerUseFeatureState()
return _feature_state
@@ -0,0 +1,53 @@
# -*- coding: utf-8 -*-
"""Retiring a Computer Use turn when the request that opened it finishes.
The host mints a fresh turn id for every request, in a request-setup hook. This
is the other half of that: a hook in the FINALLY phase, which the runtime runs
inside a `finally` block, so the turn is released whether the request
succeeded,
failed or was cancelled.
Without it nothing ever told the plugin a turn was over. A session that used
the
tool once went on claiming its turn -- holding the connection, and holding the
helper's screenshots and accessibility handles for that turn -- until some
later
request happened to arrive with a different id.
"""
from __future__ import annotations
import logging
from qwenpaw.hooks.base import LifecycleHook
from qwenpaw.runtime.hooks import HookContext, HookResult
from qwenpaw.runtime.phases import Phase
from .client import end_computer_use_turn
logger = logging.getLogger(__name__)
class ComputerUseTurnEndHook(LifecycleHook):
"""Release the native turn a finished request was holding."""
phase = Phase.FINALLY
name = "computer_use_turn_end"
priority = 50
async def run(self, ctx: HookContext) -> HookResult:
session_id = getattr(ctx, "session_id", "") or ""
if not session_id:
return HookResult()
try:
await end_computer_use_turn(session_id)
except Exception: # noqa: BLE001 - cleanup must not fail a request
logger.debug(
"computer_use_turn_end: release failed session=%s",
session_id,
exc_info=True,
)
return HookResult()
__all__ = ["ComputerUseTurnEndHook"]
@@ -0,0 +1,120 @@
# -*- coding: utf-8 -*-
"""Protocol models shared by the Computer Use client and transports."""
from __future__ import annotations
import uuid
from dataclasses import dataclass
from typing import Any, Mapping
# The plugin can be updated independently of the desktop host. Keep its
# expected version local so mismatched installations fail at the handshake.
PROTOCOL_VERSION = 2
# Every method name this adapter may put on the wire. The helper matches on
# these, so the two sides have to agree exactly -- a name only one of them
# knows
# fails as an unsupported operation at run time, with nothing to catch it
# earlier. A contract test compares this set against the helper's dispatch.
#
# Tool-level action names are a separate vocabulary: several of them map onto
# one
# method here (a double click and a right click are both ``click``), and some
# never reach the helper at all.
NATIVE_METHODS = frozenset(
{
"click",
"close_window",
"drag",
"end_turn",
"hello",
"invoke_element",
"launch_app",
"list_apps",
"list_windows",
"observe_window",
"press_key",
"scroll",
"sequence",
"set_value",
"type_text",
},
)
class ComputerUseProtocolError(RuntimeError):
"""A stable native protocol failure with an actionable error code."""
def __init__(self, code: str, message: str) -> None:
super().__init__(message)
self.code = code
@dataclass(frozen=True)
class NativeRequest:
"""One client-to-native request with trusted execution metadata."""
method: str
params: Mapping[str, Any]
session_id: str
turn_id: str
deadline_ms: int
request_id: str = ""
def to_message(self) -> dict[str, Any]:
"""Serialize the request at the framing-neutral protocol boundary.
Every request passes through here, so it is where the declared method
vocabulary is enforced. Without that the constant would only be
documentation, and a typo would travel to the helper to come back as an
unsupported operation.
"""
if self.method not in NATIVE_METHODS:
raise ComputerUseProtocolError(
"invalid_request",
f"{self.method!r} is not a Computer Use protocol method.",
)
return {
"request_id": self.request_id or uuid.uuid4().hex,
"method": self.method,
"params": dict(self.params),
"meta": {
"session_id": self.session_id,
"turn_id": self.turn_id,
"deadline_ms": self.deadline_ms,
},
"protocol_version": PROTOCOL_VERSION,
}
def parse_response(message: Mapping[str, Any]) -> dict[str, Any]:
"""Normalize a native response or raise a stable protocol error."""
if bool(message.get("ok")):
result = message.get("result", {})
return result if isinstance(result, dict) else {"value": result}
error = message.get("error", {})
if isinstance(error, Mapping):
code = str(error.get("code") or "native_error")
detail = str(error.get("message") or code)
else:
code = "native_error"
detail = str(error or code)
raise ComputerUseProtocolError(code, detail)
def approval_reply(
request_id: str,
*,
allowed: bool,
source: str,
) -> dict[str, Any]:
"""Build the one-time reply for a native App approval request."""
return {
"request_id": request_id,
"result": {
"decision": "allow" if allowed else "deny",
"source": source,
},
"protocol_version": PROTOCOL_VERSION,
}
@@ -0,0 +1,206 @@
# -*- coding: utf-8 -*-
"""Session-scoped controls for the Computer Use plugin page."""
from __future__ import annotations
from typing import Literal
from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel, Field
from qwenpaw.app.approvals import get_approval_service
from qwenpaw.app.computer_use import HostRuntimeProvider
from qwenpaw.security.tool_guard.approval import ApprovalDecision
from .access import (
get_computer_use_access_store,
)
from .client import (
is_computer_use_active,
known_computer_use_sessions,
stop_all_computer_use_turns,
stop_computer_use_session,
)
from .feature_state import get_computer_use_feature_state
class SessionRequest(BaseModel):
"""A page action scoped to exactly one active conversation."""
session_id: str = Field(min_length=1)
class FeatureToggleRequest(BaseModel):
"""A request to enable or disable the Computer Use feature."""
enabled: bool
session_id: str | None = None
class PersistentAccessRequest(BaseModel):
"""A request to revoke one installation-scoped grant."""
canonical_app_id: str = Field(min_length=1)
class PendingDecisionRequest(SessionRequest):
"""A response to one pending native App approval."""
request_id: str = Field(min_length=1)
decision: Literal["deny", "session", "always"]
def _is_computer_use_pending(pending) -> bool:
return pending.extra.get("source_type") == "computer_use_app_access"
def _pending_app(pending) -> tuple[str, str] | None:
details = pending.extra.get("computer_use_app")
if not isinstance(details, dict):
return None
canonical_app_id = str(details.get("canonical_app_id") or "").strip()
display_name = str(details.get("display_name") or canonical_app_id).strip()
if not canonical_app_id or not display_name:
return None
return canonical_app_id, display_name
async def _pending_for_session(session_id: str):
pending = await get_approval_service().list_pending_by_session(session_id)
return [item for item in pending if _is_computer_use_pending(item)]
def build_router() -> APIRouter:
"""Build routes that do not acquire or start the native runtime."""
router = APIRouter()
@router.get("/status")
def get_status() -> dict[str, bool | str]:
capability = HostRuntimeProvider.get_capability()
return {
"runtime_available": HostRuntimeProvider.is_available(),
"connection_active": capability is not None,
"approval_scope": "session_and_persistent",
"enabled": get_computer_use_feature_state().is_enabled(),
}
@router.post("/feature")
async def set_feature_enabled(
request: FeatureToggleRequest,
) -> dict[str, int | bool]:
get_computer_use_feature_state().set_enabled(request.enabled)
if request.enabled:
return {"enabled": True}
# Off is a kill switch for every agent, so every session waiting on an
# approval has to be refused -- not just the one the caller named. A
# session left waiting would otherwise be granted access later and act
# on a desktop the user had already switched the feature off for.
#
# Denying comes before stopping, for the reason given on /session/stop.
sessions = set(known_computer_use_sessions())
if request.session_id:
sessions.add(request.session_id)
denied = 0
for session_id in sorted(sessions):
denied += await _deny_pending(session_id)
stopped = await stop_all_computer_use_turns()
return {"enabled": False, "stopped": stopped, "denied": denied}
@router.get("/access")
def list_persistent_access() -> dict[str, list[dict[str, str]]]:
access = get_computer_use_access_store().list_persistent()
return {
"access": [
{
"canonical_app_id": item.canonical_app_id,
"display_name": item.display_name,
}
for item in access
],
}
@router.delete("/access")
def revoke_persistent_access(
request: PersistentAccessRequest,
) -> dict[str, bool]:
return {
"revoked": get_computer_use_access_store().revoke_persistent(
request.canonical_app_id,
),
}
@router.get("/session")
async def get_session_state(
session_id: str = Query(min_length=1),
) -> dict[str, bool]:
return {"automation_active": is_computer_use_active(session_id)}
@router.post("/session/stop")
async def stop_automation(
request: SessionRequest,
) -> dict[str, int | bool]:
# Deny first, then stop. A pending approval is a wait the helper is
# sitting in, and denying it is what ends that wait -- so it has to
# happen before anything that waits on the action itself, or the
# release would be queued behind the thing it releases.
denied = await _deny_pending(request.session_id)
stopped = await stop_computer_use_session(request.session_id)
return {"stopped": stopped, "denied": denied}
async def _deny_pending(session_id: str) -> int:
"""Refuse every approval this session is waiting on."""
service = get_approval_service()
denied = 0
for pending in await _pending_for_session(session_id):
resolved = await service.resolve_request(
pending.request_id,
ApprovalDecision.DENIED,
)
denied += int(resolved is not None)
return denied
@router.post("/session/pending/decision")
async def decide_pending(
request: PendingDecisionRequest,
) -> dict[str, bool | str]:
pending = await get_approval_service().get_request(request.request_id)
if (
pending is None
or pending.session_id != request.session_id
or not _is_computer_use_pending(pending)
):
raise HTTPException(
status_code=404,
detail="Pending approval not found.",
)
decision = (
ApprovalDecision.DENIED
if request.decision == "deny"
else ApprovalDecision.APPROVED
)
app = _pending_app(pending)
access_store = get_computer_use_access_store()
if request.decision == "always":
if app is None:
raise HTTPException(
status_code=400,
detail="Pending application is invalid.",
)
access_store.record_persistent(*app)
resolved = await get_approval_service().resolve_request(
request.request_id,
decision,
)
if (
resolved is None
and request.decision == "always"
and app is not None
):
access_store.revoke_persistent(app[0])
return {
"resolved": resolved is not None,
"decision": request.decision,
}
return router
@@ -0,0 +1,13 @@
# -*- coding: utf-8 -*-
"""Host capability transports for Computer Use."""
from .base import ComputerUseTransport, ReverseRequestHandler
from .unix_socket import UnixSocketTransport
from .windows_pipe import WindowsPipeTransport
__all__ = [
"ComputerUseTransport",
"ReverseRequestHandler",
"UnixSocketTransport",
"WindowsPipeTransport",
]
@@ -0,0 +1,36 @@
# -*- coding: utf-8 -*-
"""Framing-neutral controlled transport contract."""
from __future__ import annotations
from abc import ABC, abstractmethod
from collections.abc import Awaitable, Callable, Mapping
from typing import Any
ReverseRequestHandler = Callable[
[Mapping[str, Any]],
Awaitable[dict[str, Any]],
]
class ComputerUseTransport(ABC):
"""Authenticated request/response transport with reverse policy events."""
@abstractmethod
async def connect(self) -> None:
"""Connect and authenticate the native endpoint."""
@abstractmethod
async def request(self, message: Mapping[str, Any]) -> dict[str, Any]:
"""Send one request and await its matching response."""
@abstractmethod
async def close(self) -> None:
"""Close the transport and fail outstanding requests."""
@abstractmethod
def set_reverse_request_handler(
self,
handler: ReverseRequestHandler,
) -> None:
"""Install the handler for native-initiated policy requests."""
@@ -0,0 +1,334 @@
# -*- coding: utf-8 -*-
"""Unix domain socket transport for the host-managed Computer Use helper.
This mirrors :class:`WindowsPipeTransport` on macOS. The wire protocol,
handshake, request correlation, and reverse app-approval behavior are
identical; only the byte transport differs (an ``asyncio`` Unix domain
socket instead of a Windows named pipe), so no background thread or
overlapped I/O machinery is needed.
"""
# The capability's endpoint name and secret are deliberately private so they
# cannot leak into tool inputs; a transport is their only intended reader.
# pylint: disable=protected-access
from __future__ import annotations
import asyncio
import contextvars
import json
import socket
import struct
import time
from collections.abc import Mapping
from typing import Any
from qwenpaw.app.computer_use.runtime import RuntimeCapability
from ..protocol import ComputerUseProtocolError, approval_reply
from .base import ComputerUseTransport, ReverseRequestHandler
_MAX_FRAME_BYTES = 64 * 1024 * 1024
_CONNECT_TIMEOUT_SECONDS = 5
_APPROVAL_POLL_SECONDS = 0.25
class UnixSocketTransport(ComputerUseTransport):
"""Length-prefixed JSON transport with native reverse-policy support."""
def __init__(self, capability: RuntimeCapability) -> None:
self._capability = capability
self._reader: asyncio.StreamReader | None = None
self._writer: asyncio.StreamWriter | None = None
self._reader_task: asyncio.Task[None] | None = None
self._loop: asyncio.AbstractEventLoop | None = None
self._pending: dict[str, asyncio.Future[dict[str, Any]]] = {}
self._write_lock = asyncio.Lock()
self._reverse_handler: ReverseRequestHandler | None = None
self._reverse_context: contextvars.Context | None = None
self._approvals_in_flight = 0
self._closed = False
async def connect(self) -> None:
"""Open the host-provided socket and perform the protocol handshake."""
if self._writer is not None:
return
if not hasattr(socket, "AF_UNIX"):
raise ComputerUseProtocolError(
"runtime_unavailable",
"Computer Use Unix socket transport requires a POSIX host.",
)
self._loop = asyncio.get_running_loop()
try:
self._reader, self._writer = await asyncio.wait_for(
asyncio.open_unix_connection(self._capability._pipe_name),
_CONNECT_TIMEOUT_SECONDS,
)
except (OSError, asyncio.TimeoutError) as exc:
raise ComputerUseProtocolError(
"runtime_unavailable",
"Computer Use native runtime is not running.",
) from exc
self._reader_task = asyncio.ensure_future(self._reader_loop())
hello = {
"request_id": "hello",
"method": "hello",
"params": {
"capability": self._capability._secret,
"protocol_version": self._capability.protocol_version,
},
"meta": {"session_id": "", "turn_id": "", "deadline_ms": 5000},
"protocol_version": self._capability.protocol_version,
}
response = await self.request(hello)
result = response.get("result")
if (
not isinstance(result, Mapping)
or int(result.get("protocol_version", 0))
!= self._capability.protocol_version
):
await self.close()
raise ComputerUseProtocolError(
"protocol_mismatch",
"Computer Use helper protocol is incompatible with this "
"plugin.",
)
async def request(self, message: Mapping[str, Any]) -> dict[str, Any]:
"""Send a request and await its native response by request id."""
if self._writer is None or self._closed:
raise ComputerUseProtocolError(
"runtime_unavailable",
"Computer Use native runtime is unavailable.",
)
request_id = str(message.get("request_id") or "")
if not request_id:
raise ComputerUseProtocolError(
"invalid_request",
"Computer Use request is missing its request identifier.",
)
future: asyncio.Future[
dict[str, Any]
] = asyncio.get_running_loop().create_future()
if request_id in self._pending:
raise ComputerUseProtocolError(
"duplicate_request",
"Computer Use request identifier is already in use.",
)
self._pending[request_id] = future
# The helper only issues reverse approval requests while it is
# blocked serving this request, so snapshot the caller's context
# (session, agent, user, channel) for the approval coroutine.
self._reverse_context = contextvars.copy_context()
meta = message.get("meta")
timeout_ms = (
int(meta.get("deadline_ms", 10000))
if isinstance(meta, Mapping)
else 10000
)
timeout = max(0.1, timeout_ms / 1000)
try:
try:
await asyncio.wait_for(
self._write_message(dict(message)),
timeout,
)
except asyncio.TimeoutError:
# A failed or timed-out write may leave a partial frame on the
# socket, so the connection can no longer be trusted.
future.cancel()
await self.close()
raise
except ComputerUseProtocolError:
future.cancel()
await self.close()
raise
except Exception as exc: # noqa: BLE001 - transport boundary
future.cancel()
await self.close()
raise ComputerUseProtocolError(
"runtime_disconnected",
"Computer Use connection failed while sending a request.",
) from exc
return await self._await_response(future, timeout)
except asyncio.TimeoutError as exc:
raise ComputerUseProtocolError(
"request_timeout",
"Computer Use request timed out.",
) from exc
finally:
self._pending.pop(request_id, None)
async def _await_response(
self,
future: asyncio.Future[dict[str, Any]],
timeout: float,
) -> dict[str, Any]:
"""Await a response without charging user approval time as timeout.
The helper blocks serving a request while a reverse app approval
waits for the user, so the machine deadline pauses until the
approval resolves and the helper then gets a fresh execution window.
"""
deadline = time.monotonic() + timeout
while True:
if self._approvals_in_flight > 0:
deadline = time.monotonic() + timeout
remaining = deadline - time.monotonic()
if remaining <= 0:
future.cancel()
raise asyncio.TimeoutError
try:
return await asyncio.wait_for(
asyncio.shield(future),
min(remaining, _APPROVAL_POLL_SECONDS),
)
except asyncio.TimeoutError:
continue
async def close(self) -> None:
"""Close the socket and reject every pending request."""
if self._closed:
return
self._closed = True
writer, self._writer = self._writer, None
self._reader = None
task, self._reader_task = self._reader_task, None
if task is not None and task is not asyncio.current_task():
task.cancel()
if writer is not None:
try:
writer.close()
await writer.wait_closed()
except (
Exception
): # noqa: BLE001 - closing a broken socket may raise
pass
self._fail_pending(
"runtime_disconnected",
"Computer Use connection closed.",
)
def set_reverse_request_handler(
self,
handler: ReverseRequestHandler,
) -> None:
self._reverse_handler = handler
async def _write_message(self, message: dict[str, Any]) -> None:
writer = self._writer
if writer is None:
raise ComputerUseProtocolError(
"runtime_disconnected",
"Computer Use connection is closed.",
)
payload = json.dumps(message, separators=(",", ":")).encode("utf-8")
if len(payload) > _MAX_FRAME_BYTES:
raise ComputerUseProtocolError(
"frame_too_large",
"Computer Use request exceeds the transport limit.",
)
async with self._write_lock:
writer.write(struct.pack("<I", len(payload)) + payload)
await writer.drain()
async def _reader_loop(self) -> None:
reader = self._reader
if reader is None:
return
try:
while not self._closed:
message = await self._read_message(reader)
if message.get("method") == "request_app_approval":
self._schedule_reverse_request(message)
else:
self._resolve_response(message)
except (asyncio.CancelledError, asyncio.IncompleteReadError):
pass
except Exception as exc: # noqa: BLE001 - transport boundary
if not self._closed:
self._fail_pending(
"runtime_disconnected",
f"Computer Use connection failed: {exc}",
)
async def _read_message(
self,
reader: asyncio.StreamReader,
) -> dict[str, Any]:
header = await reader.readexactly(4)
frame_size = struct.unpack("<I", header)[0]
if not 0 < frame_size <= _MAX_FRAME_BYTES:
raise ComputerUseProtocolError(
"invalid_frame",
"Invalid Computer Use frame size.",
)
payload = await reader.readexactly(frame_size)
value = json.loads(payload.decode("utf-8"))
if not isinstance(value, dict):
raise ComputerUseProtocolError(
"invalid_frame",
"Invalid Computer Use message.",
)
return value
def _schedule_reverse_request(self, message: dict[str, Any]) -> None:
loop = self._loop
if loop is None:
return
# Pause pending request deadlines while the user decides.
self._approvals_in_flight += 1
context = self._reverse_context
# Replay the request-time context so the approval coroutine keeps the
# session/agent/user/channel it needs; the bare reader-task context
# would drop those contextvars.
if context is not None:
loop.create_task(
self._reply_to_reverse_request(message),
context=context,
)
else:
loop.create_task(self._reply_to_reverse_request(message))
async def _reply_to_reverse_request(self, message: dict[str, Any]) -> None:
try:
await self._handle_reverse_request(message)
finally:
self._approvals_in_flight -= 1
async def _handle_reverse_request(self, message: dict[str, Any]) -> None:
request_id = str(message.get("request_id") or "")
handler = self._reverse_handler
decision = {"allowed": False, "source": "invalid"}
if request_id and handler is not None:
try:
decision = await handler(message)
except Exception: # noqa: BLE001 - fail closed at the socket edge
decision = {"allowed": False, "source": "error"}
reply = approval_reply(
request_id,
allowed=bool(decision.get("allowed")),
source=str(decision.get("source") or "unknown"),
)
try:
await self._write_message(reply)
except Exception:
self._fail_pending(
"runtime_disconnected",
"Computer Use approval reply could not reach the native "
"runtime.",
)
def _resolve_response(self, message: dict[str, Any]) -> None:
request_id = str(message.get("request_id") or "")
future = self._pending.get(request_id)
if future is None or future.done():
return
future.set_result(message)
def _fail_pending(self, code: str, message: str) -> None:
futures = list(self._pending.values())
self._pending.clear()
for future in futures:
if not future.done():
future.set_exception(ComputerUseProtocolError(code, message))
@@ -0,0 +1,648 @@
# -*- coding: utf-8 -*-
"""Windows named-pipe transport for the host-managed Computer Use helper."""
# The capability's endpoint name and secret are deliberately private so they
# cannot leak into tool inputs; a transport is their only intended reader.
# pylint: disable=protected-access
from __future__ import annotations
import asyncio
import contextvars
import ctypes
import json
import os
import struct
import threading
import time
from collections.abc import Callable, Mapping
from ctypes import wintypes
from typing import Any
from qwenpaw.app.computer_use.runtime import RuntimeCapability
from ..protocol import ComputerUseProtocolError, approval_reply
from .base import ComputerUseTransport, ReverseRequestHandler
_GENERIC_READ = 0x80000000
_GENERIC_WRITE = 0x40000000
_OPEN_EXISTING = 3
_FILE_FLAG_OVERLAPPED = 0x40000000
_ERROR_PIPE_BUSY = 231
_ERROR_FILE_NOT_FOUND = 2
_ERROR_IO_PENDING = 997
_WAIT_OBJECT_0 = 0x0
_WAIT_FAILED = 0xFFFFFFFF
_INFINITE = 0xFFFFFFFF
_MAX_FRAME_BYTES = 64 * 1024 * 1024
_CONNECT_TIMEOUT_SECONDS = 5
_WRITE_TIMEOUT_MS = 10000
_READ_POLL_MS = 500
_APPROVAL_POLL_SECONDS = 0.25
_CLOSE_JOIN_TIMEOUT_SECONDS = 2
class _OVERLAPPED(ctypes.Structure):
_fields_ = [
("Internal", ctypes.c_void_p),
("InternalHigh", ctypes.c_void_p),
("Offset", wintypes.DWORD),
("OffsetHigh", wintypes.DWORD),
("hEvent", wintypes.HANDLE),
]
class WindowsPipeTransport(ComputerUseTransport):
"""Length-prefixed JSON transport with native reverse-policy support."""
def __init__(self, capability: RuntimeCapability) -> None:
self._capability = capability
self._handle: int | None = None
self._loop: asyncio.AbstractEventLoop | None = None
self._reader: threading.Thread | None = None
self._pending: dict[str, asyncio.Future[dict[str, Any]]] = {}
self._pending_lock = threading.Lock()
self._write_lock = threading.Lock()
self._reverse_handler: ReverseRequestHandler | None = None
self._reverse_context: contextvars.Context | None = None
self._approvals_lock = threading.Lock()
self._approvals_in_flight = 0
self._closed = False
async def connect(self) -> None:
"""Open the host-provided pipe and perform the protocol handshake."""
if self._handle is not None:
return
if os.name != "nt":
raise ComputerUseProtocolError(
"runtime_unavailable",
"Computer Use is only available on Windows.",
)
self._loop = asyncio.get_running_loop()
self._handle = await asyncio.to_thread(_connect_pipe, self._pipe_path)
self._reader = threading.Thread(
target=self._reader_loop,
name="computer-use-pipe-reader",
daemon=True,
)
self._reader.start()
hello = {
"request_id": "hello",
"method": "hello",
"params": {
"capability": self._capability._secret,
"protocol_version": self._capability.protocol_version,
},
"meta": {"session_id": "", "turn_id": "", "deadline_ms": 5000},
"protocol_version": self._capability.protocol_version,
}
response = await self.request(hello)
result = response.get("result")
if (
not isinstance(result, Mapping)
or int(result.get("protocol_version", 0))
!= self._capability.protocol_version
):
await self.close()
raise ComputerUseProtocolError(
"protocol_mismatch",
"Computer Use helper protocol is incompatible with this "
"plugin.",
)
async def request(self, message: Mapping[str, Any]) -> dict[str, Any]:
"""Send a request and await its native response by request id."""
if self._handle is None or self._closed:
raise ComputerUseProtocolError(
"runtime_unavailable",
"Computer Use native runtime is unavailable.",
)
request_id = str(message.get("request_id") or "")
if not request_id:
raise ComputerUseProtocolError(
"invalid_request",
"Computer Use request is missing its request identifier.",
)
future: asyncio.Future[
dict[str, Any]
] = asyncio.get_running_loop().create_future()
with self._pending_lock:
if request_id in self._pending:
raise ComputerUseProtocolError(
"duplicate_request",
"Computer Use request identifier is already in use.",
)
self._pending[request_id] = future
# The helper only issues reverse approval requests while it is
# blocked serving this request, so snapshot the caller's context
# (session, agent, user, channel) for the approval coroutine.
self._reverse_context = contextvars.copy_context()
meta = message.get("meta")
timeout_ms = (
int(meta.get("deadline_ms", 10000))
if isinstance(meta, Mapping)
else 10000
)
timeout = max(0.1, timeout_ms / 1000)
try:
try:
await asyncio.wait_for(
asyncio.to_thread(self._write_message, dict(message)),
timeout,
)
except asyncio.TimeoutError:
# A failed or timed-out write may leave a partial frame on the
# pipe, so the connection can no longer be trusted.
future.cancel()
await self.close()
raise
except ComputerUseProtocolError:
future.cancel()
await self.close()
raise
except Exception as exc: # noqa: BLE001 - transport boundary
future.cancel()
await self.close()
raise ComputerUseProtocolError(
"runtime_disconnected",
"Computer Use connection failed while sending a request.",
) from exc
return await self._await_response(future, timeout)
except TimeoutError as exc:
raise ComputerUseProtocolError(
"request_timeout",
"Computer Use request timed out.",
) from exc
finally:
with self._pending_lock:
self._pending.pop(request_id, None)
async def _await_response(
self,
future: asyncio.Future[dict[str, Any]],
timeout: float,
) -> dict[str, Any]:
"""Await a response without charging user approval time as timeout.
The helper blocks serving a request while a reverse app approval
waits for the user, so the machine deadline pauses until the
approval resolves (bounded by the approval service's own timeout)
and the helper then gets a fresh execution window.
"""
deadline = time.monotonic() + timeout
while True:
with self._approvals_lock:
approval_in_flight = self._approvals_in_flight > 0
if approval_in_flight:
deadline = time.monotonic() + timeout
remaining = deadline - time.monotonic()
if remaining <= 0:
future.cancel()
raise TimeoutError
try:
return await asyncio.wait_for(
asyncio.shield(future),
min(remaining, _APPROVAL_POLL_SECONDS),
)
except TimeoutError:
continue
async def close(self) -> None:
"""Close the named pipe and reject every pending request."""
if self._closed:
return
self._closed = True
handle, self._handle = self._handle, None
reader, self._reader = self._reader, None
if handle is not None:
await asyncio.to_thread(_close_handle, handle, reader)
self._fail_pending(
"runtime_disconnected",
"Computer Use connection closed.",
)
def set_reverse_request_handler(
self,
handler: ReverseRequestHandler,
) -> None:
self._reverse_handler = handler
@property
def _pipe_path(self) -> str:
name = self._capability._pipe_name
prefix = "\\\\.\\pipe\\"
return name if name.startswith(prefix) else f"{prefix}{name}"
def _write_message(self, message: dict[str, Any]) -> None:
handle = self._handle
if handle is None:
raise ComputerUseProtocolError(
"runtime_disconnected",
"Computer Use connection is closed.",
)
payload = json.dumps(message, separators=(",", ":")).encode("utf-8")
if len(payload) > _MAX_FRAME_BYTES:
raise ComputerUseProtocolError(
"frame_too_large",
"Computer Use request exceeds the transport limit.",
)
with self._write_lock:
_write_all(handle, struct.pack("<I", len(payload)) + payload)
def _reader_loop(self) -> None:
handle = self._handle
if handle is None:
return
try:
while not self._closed:
message = _read_message(handle, lambda: self._closed)
if message.get("method") == "request_app_approval":
self._schedule_reverse_request(message)
else:
self._resolve_response(message)
except Exception as exc: # noqa: BLE001 - transport boundary
if not self._closed:
self._fail_pending(
"runtime_disconnected",
f"Computer Use connection failed: {exc}",
)
def _schedule_reverse_request(self, message: dict[str, Any]) -> None:
loop = self._loop
if loop is None:
return
# Pause pending request deadlines while the user decides.
with self._approvals_lock:
self._approvals_in_flight += 1
context = self._reverse_context
if context is None:
asyncio.run_coroutine_threadsafe(
self._reply_to_reverse_request(message),
loop,
)
return
# run_coroutine_threadsafe would copy this reader thread's bare
# context, losing the session contextvars, so schedule through
# call_soon_threadsafe with the snapshot captured in request().
loop.call_soon_threadsafe(
self._start_reverse_task,
message,
context=context,
)
def _start_reverse_task(self, message: dict[str, Any]) -> None:
asyncio.ensure_future(self._reply_to_reverse_request(message))
async def _reply_to_reverse_request(self, message: dict[str, Any]) -> None:
try:
await self._handle_reverse_request(message)
finally:
with self._approvals_lock:
self._approvals_in_flight -= 1
async def _handle_reverse_request(self, message: dict[str, Any]) -> None:
request_id = str(message.get("request_id") or "")
handler = self._reverse_handler
decision = {"allowed": False, "source": "invalid"}
if request_id and handler is not None:
try:
decision = await handler(message)
except Exception: # noqa: BLE001 - fail closed at the pipe edge
decision = {"allowed": False, "source": "error"}
reply = approval_reply(
request_id,
allowed=bool(decision.get("allowed")),
source=str(decision.get("source") or "unknown"),
)
try:
await asyncio.to_thread(self._write_message, reply)
except Exception:
self._fail_pending(
"runtime_disconnected",
"Computer Use approval reply could not reach the native "
"runtime.",
)
def _resolve_response(self, message: dict[str, Any]) -> None:
request_id = str(message.get("request_id") or "")
with self._pending_lock:
future = self._pending.get(request_id)
if future is None or future.done() or self._loop is None:
return
self._loop.call_soon_threadsafe(future.set_result, message)
def _fail_pending(self, code: str, message: str) -> None:
with self._pending_lock:
futures = list(self._pending.values())
self._pending.clear()
for future in futures:
if future.done() or self._loop is None:
continue
self._loop.call_soon_threadsafe(
future.set_exception,
ComputerUseProtocolError(code, message),
)
def _connect_pipe(pipe_path: str) -> int:
deadline = time.monotonic() + _CONNECT_TIMEOUT_SECONDS
while True:
handle = _kernel32().CreateFileW(
pipe_path,
_GENERIC_READ | _GENERIC_WRITE,
0,
None,
_OPEN_EXISTING,
_FILE_FLAG_OVERLAPPED,
None,
)
if _valid_handle(handle):
return int(handle)
error = ctypes.get_last_error()
remaining = deadline - time.monotonic()
if (
error in (_ERROR_PIPE_BUSY, _ERROR_FILE_NOT_FOUND)
and remaining > 0
):
if error == _ERROR_PIPE_BUSY:
_kernel32().WaitNamedPipeW(
pipe_path,
max(1, min(250, int(remaining * 1000))),
)
else:
time.sleep(min(0.05, remaining))
continue
if error == _ERROR_FILE_NOT_FOUND:
raise ComputerUseProtocolError(
"runtime_unavailable",
"Computer Use native runtime is not running.",
)
raise OSError(
error,
f"Could not connect to Computer Use pipe {pipe_path!r}",
)
def _read_message(
handle: int,
abort_check: Callable[[], bool] | None = None,
) -> dict[str, Any]:
frame_size = struct.unpack("<I", _read_exact(handle, 4, abort_check))[0]
if not 0 < frame_size <= _MAX_FRAME_BYTES:
raise ComputerUseProtocolError(
"invalid_frame",
"Invalid Computer Use frame size.",
)
value = json.loads(
_read_exact(handle, frame_size, abort_check).decode("utf-8"),
)
if not isinstance(value, dict):
raise ComputerUseProtocolError(
"invalid_frame",
"Invalid Computer Use message.",
)
return value
def _read_exact(
handle: int,
size: int,
abort_check: Callable[[], bool] | None = None,
) -> bytes:
chunks: list[bytes] = []
remaining = size
event = _new_event()
overlapped = _OVERLAPPED()
overlapped.hEvent = event
try:
while remaining:
if abort_check is not None and abort_check():
raise OSError("Computer Use connection closed")
buffer = ctypes.create_string_buffer(remaining)
read = _run_io(
handle,
overlapped,
lambda: _kernel32().ReadFile(
handle,
buffer,
remaining,
None,
ctypes.byref(overlapped),
),
None,
"Computer Use pipe read failed",
abort_check,
)
if not read:
raise OSError("Computer Use pipe closed")
chunks.append(buffer.raw[:read])
remaining -= read
finally:
_kernel32().CloseHandle(event)
return b"".join(chunks)
def _write_all(handle: int, data: bytes) -> None:
offset = 0
event = _new_event()
overlapped = _OVERLAPPED()
overlapped.hEvent = event
try:
while offset < len(data):
chunk = data[offset:]
written = _run_io(
handle,
overlapped,
lambda: _kernel32().WriteFile(
handle,
chunk,
len(chunk),
None,
ctypes.byref(overlapped),
),
_WRITE_TIMEOUT_MS,
"Computer Use pipe write failed",
)
if not written:
raise OSError("Computer Use pipe closed")
offset += written
finally:
_kernel32().CloseHandle(event)
def _run_io(
handle: int,
overlapped: _OVERLAPPED,
start_io: Callable[[], bool],
timeout_ms: int | None,
error_message: str,
abort_check: Callable[[], bool] | None = None,
) -> int:
"""Drive one overlapped ReadFile/WriteFile to completion.
``start_io`` issues the overlapped operation. Returns the number of bytes
transferred. ``timeout_ms`` bounds the total wait. ``abort_check`` (read
path) is polled every ``_READ_POLL_MS`` without touching the pending I/O,
so a racing ``close()`` cannot strand the reader even when its
``CancelIoEx`` lands before the read was issued.
"""
kernel32 = _kernel32()
kernel32.ResetEvent(overlapped.hEvent)
if not start_io():
error = ctypes.get_last_error()
if error != _ERROR_IO_PENDING:
raise OSError(error, error_message)
deadline = (
None
if timeout_ms is None
else time.monotonic() + timeout_ms / 1000
)
while True:
if abort_check is not None:
wait_ms = _READ_POLL_MS
elif deadline is None:
wait_ms = _INFINITE
else:
wait_ms = max(1, int((deadline - time.monotonic()) * 1000))
wait = kernel32.WaitForSingleObject(overlapped.hEvent, wait_ms)
if wait == _WAIT_OBJECT_0:
break
if wait == _WAIT_FAILED:
raise OSError(ctypes.get_last_error(), error_message)
# WAIT_TIMEOUT: poll the abort flag / deadline; the I/O
# stays pending and is only cancelled in the branches below.
if abort_check is not None and abort_check():
_cancel_and_drain(kernel32, handle, overlapped)
raise OSError("Computer Use connection closed")
if deadline is not None and time.monotonic() >= deadline:
_cancel_and_drain(kernel32, handle, overlapped)
raise TimeoutError("Computer Use pipe I/O timed out")
transferred = wintypes.DWORD(0)
if not kernel32.GetOverlappedResult(
handle,
ctypes.byref(overlapped),
ctypes.byref(transferred),
False,
):
raise OSError(ctypes.get_last_error(), error_message)
return transferred.value
def _cancel_and_drain(
kernel32: Any,
handle: int,
overlapped: _OVERLAPPED,
) -> None:
"""Cancel and drain the pending operation before buffers are freed."""
kernel32.CancelIoEx(handle, ctypes.byref(overlapped))
drained = wintypes.DWORD(0)
kernel32.GetOverlappedResult(
handle,
ctypes.byref(overlapped),
ctypes.byref(drained),
True,
)
def _new_event() -> wintypes.HANDLE:
event = _kernel32().CreateEventW(None, True, False, None)
if not event:
raise OSError(
ctypes.get_last_error(),
"Computer Use pipe event creation failed",
)
return event
def _close_handle(handle: int, reader: threading.Thread | None = None) -> None:
"""Cancel pending I/O, wait for the reader thread to unwind, then close."""
_kernel32().CancelIoEx(handle, None)
if (
reader is not None
and reader.is_alive()
and reader is not threading.current_thread()
):
reader.join(timeout=_CLOSE_JOIN_TIMEOUT_SECONDS)
if reader.is_alive():
# The reader did not unwind, so it may still be inside a call on
# this handle. Closing now would let the kernel reuse the handle
# value while that call is in flight, pointing it at whatever
# object lands on the number next. Leaking one handle for the
# life of the process is the safer trade.
return
_kernel32().CloseHandle(handle)
_kernel32_cache = None
_kernel32_cache_lock = threading.Lock()
def _kernel32():
# The signatures are configured once and reused. Rebuilding the WinDLL
# wrapper and re-declaring argtypes on every close/cancel/event call was
# wasted work; the module itself was already loaded once by the OS.
global _kernel32_cache
if _kernel32_cache is None:
with _kernel32_cache_lock:
if _kernel32_cache is None:
_kernel32_cache = _build_kernel32()
return _kernel32_cache
def _build_kernel32():
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
kernel32.CreateFileW.argtypes = [
wintypes.LPCWSTR,
wintypes.DWORD,
wintypes.DWORD,
wintypes.LPVOID,
wintypes.DWORD,
wintypes.DWORD,
wintypes.HANDLE,
]
kernel32.CreateFileW.restype = wintypes.HANDLE
kernel32.WaitNamedPipeW.argtypes = [wintypes.LPCWSTR, wintypes.DWORD]
kernel32.WaitNamedPipeW.restype = wintypes.BOOL
kernel32.CreateEventW.argtypes = [
wintypes.LPVOID,
wintypes.BOOL,
wintypes.BOOL,
wintypes.LPCWSTR,
]
kernel32.CreateEventW.restype = wintypes.HANDLE
kernel32.ResetEvent.argtypes = [wintypes.HANDLE]
kernel32.ResetEvent.restype = wintypes.BOOL
kernel32.WaitForSingleObject.argtypes = [wintypes.HANDLE, wintypes.DWORD]
kernel32.WaitForSingleObject.restype = wintypes.DWORD
kernel32.GetOverlappedResult.argtypes = [
wintypes.HANDLE,
wintypes.LPVOID,
ctypes.POINTER(wintypes.DWORD),
wintypes.BOOL,
]
kernel32.GetOverlappedResult.restype = wintypes.BOOL
kernel32.CancelIoEx.argtypes = [wintypes.HANDLE, wintypes.LPVOID]
kernel32.CancelIoEx.restype = wintypes.BOOL
kernel32.ReadFile.argtypes = [
wintypes.HANDLE,
wintypes.LPVOID,
wintypes.DWORD,
ctypes.POINTER(wintypes.DWORD),
wintypes.LPVOID,
]
kernel32.ReadFile.restype = wintypes.BOOL
kernel32.WriteFile.argtypes = [
wintypes.HANDLE,
wintypes.LPCVOID,
wintypes.DWORD,
ctypes.POINTER(wintypes.DWORD),
wintypes.LPVOID,
]
kernel32.WriteFile.restype = wintypes.BOOL
kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
kernel32.CloseHandle.restype = wintypes.BOOL
return kernel32
def _valid_handle(handle: int | None) -> bool:
return handle not in (None, 0, ctypes.c_void_p(-1).value)