源码快照 @ 2026-09-01 (source-only, 剔除非源码素材)
Creator App Tests / PR Spam Gate (push) Skipped
Frontend Tests / PR Spam Gate (push) Skipped
NPM Format (website & console) / PR Spam Gate (push) Skipped
Pre-commit Checks / PR Spam Gate (push) Skipped
QwenPaw Creator Plugin Release / check-version (push) Canceled after 0s
QwenPaw Creator Plugin Release / upload-oss (push) Canceled after 0s
Tests / PR Spam Gate (push) Skipped
CodeQL / CodeQL (python) (push) Canceled after 0s
CodeQL / CodeQL (typescript) (push) Canceled after 0s
Creator App Tests / Creator Backend Tests (push) Canceled after 0s
Creator App Tests / Creator UI Build & Tests (push) Canceled after 0s
E2E Smoke Tests / Playwright UI Smoke (push) Canceled after 0s
Frontend Tests / Detect console changes (push) Canceled after 0s
NPM Format (website & console) / website format check (push) Canceled after 0s
NPM Format (website & console) / console format check (push) Canceled after 0s
Pre-commit Checks / Pre-commit (push) Canceled after 0s
QwenPaw Data Plugin Release / check-version (push) Canceled after 0s
Tests / Detect code changes (push) Canceled after 0s
Frontend Tests / Vitest Unit Tests (push) Canceled after 0s
Frontend Tests / Frontend Summary (push) Canceled after 0s
QwenPaw Data Plugin Release / upload-oss (push) Canceled after 0s
Tests / Maintainer Approval (push) Canceled after 0s
Tests / Unit Tests - py3.11 - macos-latest (push) Canceled after 0s
Tests / Unit Tests - py3.11 - ubuntu-latest (push) Canceled after 0s
Tests / Unit Tests - py3.13 - ubuntu-latest (push) Canceled after 0s
Tests / Unit Tests - py3.11 - windows-latest (push) Canceled after 0s
Tests / Contract Tests - py3.11 - macos-latest (push) Canceled after 0s
Tests / Contract Tests - py3.11 - ubuntu-latest (push) Canceled after 0s
Tests / Contract Tests - py3.13 - ubuntu-latest (push) Canceled after 0s
Tests / Contract Tests - py3.11 - windows-latest (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - macos-latest - p0 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - macos-latest - p1 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - macos-latest - p2 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - ubuntu-latest - fallback (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - ubuntu-latest - p0 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - ubuntu-latest - p1 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - ubuntu-latest - p2 (push) Canceled after 0s
Tests / Integrated Tests - py3.13 - ubuntu-latest - p0 (push) Canceled after 0s
Tests / Integrated Tests - py3.13 - ubuntu-latest - p1 (push) Canceled after 0s
Tests / Integrated Tests - py3.13 - ubuntu-latest - p2 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - windows-latest - p0 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - windows-latest - p1 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - windows-latest - p2 (push) Canceled after 0s
Tests / Coverage Report (push) Canceled after 0s
Tests / Test Summary (push) Canceled after 0s
Creator App Tests / PR Spam Gate (push) Skipped
Frontend Tests / PR Spam Gate (push) Skipped
NPM Format (website & console) / PR Spam Gate (push) Skipped
Pre-commit Checks / PR Spam Gate (push) Skipped
QwenPaw Creator Plugin Release / check-version (push) Canceled after 0s
QwenPaw Creator Plugin Release / upload-oss (push) Canceled after 0s
Tests / PR Spam Gate (push) Skipped
CodeQL / CodeQL (python) (push) Canceled after 0s
CodeQL / CodeQL (typescript) (push) Canceled after 0s
Creator App Tests / Creator Backend Tests (push) Canceled after 0s
Creator App Tests / Creator UI Build & Tests (push) Canceled after 0s
E2E Smoke Tests / Playwright UI Smoke (push) Canceled after 0s
Frontend Tests / Detect console changes (push) Canceled after 0s
NPM Format (website & console) / website format check (push) Canceled after 0s
NPM Format (website & console) / console format check (push) Canceled after 0s
Pre-commit Checks / Pre-commit (push) Canceled after 0s
QwenPaw Data Plugin Release / check-version (push) Canceled after 0s
Tests / Detect code changes (push) Canceled after 0s
Frontend Tests / Vitest Unit Tests (push) Canceled after 0s
Frontend Tests / Frontend Summary (push) Canceled after 0s
QwenPaw Data Plugin Release / upload-oss (push) Canceled after 0s
Tests / Maintainer Approval (push) Canceled after 0s
Tests / Unit Tests - py3.11 - macos-latest (push) Canceled after 0s
Tests / Unit Tests - py3.11 - ubuntu-latest (push) Canceled after 0s
Tests / Unit Tests - py3.13 - ubuntu-latest (push) Canceled after 0s
Tests / Unit Tests - py3.11 - windows-latest (push) Canceled after 0s
Tests / Contract Tests - py3.11 - macos-latest (push) Canceled after 0s
Tests / Contract Tests - py3.11 - ubuntu-latest (push) Canceled after 0s
Tests / Contract Tests - py3.13 - ubuntu-latest (push) Canceled after 0s
Tests / Contract Tests - py3.11 - windows-latest (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - macos-latest - p0 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - macos-latest - p1 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - macos-latest - p2 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - ubuntu-latest - fallback (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - ubuntu-latest - p0 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - ubuntu-latest - p1 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - ubuntu-latest - p2 (push) Canceled after 0s
Tests / Integrated Tests - py3.13 - ubuntu-latest - p0 (push) Canceled after 0s
Tests / Integrated Tests - py3.13 - ubuntu-latest - p1 (push) Canceled after 0s
Tests / Integrated Tests - py3.13 - ubuntu-latest - p2 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - windows-latest - p0 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - windows-latest - p1 (push) Canceled after 0s
Tests / Integrated Tests - py3.11 - windows-latest - p2 (push) Canceled after 0s
Tests / Coverage Report (push) Canceled after 0s
Tests / Test Summary (push) Canceled after 0s
This commit is contained in:
@@ -0,0 +1,20 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""computer_use tool package (plugin-provided)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .dispatch import computer_use
|
||||
|
||||
__all__ = ["computer_use"]
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Load the host-bound tool entry point only when it is requested."""
|
||||
if name != "computer_use":
|
||||
raise AttributeError(name)
|
||||
from .dispatch import computer_use
|
||||
|
||||
return computer_use
|
||||
@@ -0,0 +1,197 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Application access decisions owned by the Computer Use plugin."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from threading import RLock
|
||||
from typing import Mapping
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AppApprovalRequest:
|
||||
"""A native-resolved application identity awaiting a user decision."""
|
||||
|
||||
request_id: str
|
||||
session_id: str
|
||||
canonical_app_id: str
|
||||
display_name: str
|
||||
identity_evidence: Mapping[str, str]
|
||||
risk: str = "unknown"
|
||||
warning: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AppAccessDecision:
|
||||
"""A resolved Computer Use application access decision."""
|
||||
|
||||
allowed: bool
|
||||
source: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PersistentAppAccess:
|
||||
"""One explicit, installation-scoped allow decision."""
|
||||
|
||||
canonical_app_id: str
|
||||
display_name: str
|
||||
|
||||
|
||||
def _normalize_app_id(app_id: str) -> str:
|
||||
"""Collapse equivalent spellings of one application identifier.
|
||||
|
||||
Executable identifiers can arrive as either an extended-length
|
||||
(``\\\\?\\``) path or a plain drive path, and with mixed casing across
|
||||
the discovery sources. Stripping the prefix and folding case yields a
|
||||
single key so an application is approved once per session regardless of
|
||||
the spelling that reached the store.
|
||||
"""
|
||||
text = app_id.strip()
|
||||
scheme, separator, remainder = text.partition(":")
|
||||
if separator and scheme == "process":
|
||||
verbatim_prefix = "\\\\?\\"
|
||||
if remainder.startswith(verbatim_prefix):
|
||||
remainder = remainder[len(verbatim_prefix) :]
|
||||
return f"{scheme}:{remainder.lower()}"
|
||||
# Only the Windows ``process:`` scheme is case-folded, because NTFS is
|
||||
# case-insensitive. The macOS ``app:`` path is left verbatim on purpose: a
|
||||
# case-sensitive APFS volume treats two casings as two files, so folding
|
||||
# here would merge identifiers the filesystem keeps distinct -- the same
|
||||
# mistake that once left canonical bundle paths unlaunchable. The path is
|
||||
# already canonicalized at its source, so the casing is consistent without
|
||||
# folding.
|
||||
return text
|
||||
|
||||
|
||||
def _default_persistent_path() -> Path:
|
||||
from qwenpaw.constant import WORKING_DIR
|
||||
|
||||
return (
|
||||
Path(WORKING_DIR)
|
||||
/ "plugin_runtime"
|
||||
/ "computer-use"
|
||||
/ "app_access.json"
|
||||
)
|
||||
|
||||
|
||||
class ComputerUseAccessStore:
|
||||
"""Keep session decisions in memory and explicit grants on this host."""
|
||||
|
||||
def __init__(self, persistent_path: Path | None = None) -> None:
|
||||
self._lock = RLock()
|
||||
self._persistent_path = persistent_path or _default_persistent_path()
|
||||
self._session_decisions: dict[tuple[str, str], bool] = {}
|
||||
self._persistent_decisions = self._load_persistent()
|
||||
|
||||
def resolve(self, request: AppApprovalRequest) -> AppAccessDecision | None:
|
||||
"""Return an existing decision, or ``None`` when input is needed."""
|
||||
app_id = _normalize_app_id(request.canonical_app_id)
|
||||
session_key = (request.session_id, app_id)
|
||||
with self._lock:
|
||||
session_decision = self._session_decisions.get(session_key)
|
||||
if session_decision is not None:
|
||||
return AppAccessDecision(session_decision, "session")
|
||||
if app_id in self._persistent_decisions:
|
||||
return AppAccessDecision(True, "persistent")
|
||||
return None
|
||||
|
||||
def record_session(
|
||||
self,
|
||||
request: AppApprovalRequest,
|
||||
*,
|
||||
allowed: bool,
|
||||
) -> None:
|
||||
"""Keep a decision for the current in-memory session only."""
|
||||
app_id = _normalize_app_id(request.canonical_app_id)
|
||||
with self._lock:
|
||||
self._session_decisions[(request.session_id, app_id)] = allowed
|
||||
|
||||
def record_persistent(
|
||||
self,
|
||||
canonical_app_id: str,
|
||||
display_name: str,
|
||||
) -> None:
|
||||
"""Persist an explicit allow for this QwenPaw installation."""
|
||||
app_id = _normalize_app_id(canonical_app_id)
|
||||
decision = PersistentAppAccess(app_id, display_name)
|
||||
with self._lock:
|
||||
self._persistent_decisions[app_id] = decision
|
||||
self._save_persistent_locked()
|
||||
|
||||
def list_persistent(self) -> list[PersistentAppAccess]:
|
||||
"""List application grants shared by this installation's agents."""
|
||||
with self._lock:
|
||||
decisions = list(self._persistent_decisions.values())
|
||||
return sorted(
|
||||
decisions,
|
||||
key=lambda decision: decision.display_name.casefold(),
|
||||
)
|
||||
|
||||
def revoke_persistent(self, canonical_app_id: str) -> bool:
|
||||
"""Remove one persistent allow so future access asks again."""
|
||||
app_id = _normalize_app_id(canonical_app_id)
|
||||
with self._lock:
|
||||
removed = self._persistent_decisions.pop(app_id, None)
|
||||
if removed is not None:
|
||||
self._save_persistent_locked()
|
||||
return removed is not None
|
||||
|
||||
def _load_persistent(self) -> dict[str, PersistentAppAccess]:
|
||||
try:
|
||||
with self._persistent_path.open(encoding="utf-8") as file:
|
||||
payload = json.load(file)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return {}
|
||||
records = payload.get("allowed_apps", [])
|
||||
if not isinstance(records, list):
|
||||
return {}
|
||||
return {
|
||||
app_id: PersistentAppAccess(app_id, display_name)
|
||||
for record in records
|
||||
if isinstance(record, dict)
|
||||
and (
|
||||
app_id := _normalize_app_id(
|
||||
str(record.get("canonical_app_id") or "").strip(),
|
||||
)
|
||||
)
|
||||
and (
|
||||
display_name := str(
|
||||
record.get("display_name") or app_id,
|
||||
).strip()
|
||||
)
|
||||
}
|
||||
|
||||
def _save_persistent_locked(self) -> None:
|
||||
self._persistent_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary_path = self._persistent_path.with_suffix(".tmp")
|
||||
payload = {
|
||||
"allowed_apps": [
|
||||
asdict(decision) for decision in self.list_persistent()
|
||||
],
|
||||
}
|
||||
temporary_path.write_text(
|
||||
json.dumps(payload, indent=2, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.replace(temporary_path, self._persistent_path)
|
||||
|
||||
|
||||
_access_store: ComputerUseAccessStore | None = None
|
||||
_access_store_lock = RLock()
|
||||
|
||||
|
||||
def get_computer_use_access_store() -> ComputerUseAccessStore:
|
||||
"""Return the process-wide Computer Use plugin access authority."""
|
||||
global _access_store
|
||||
# Double-checked: the fast path skips the lock once built, and the lock
|
||||
# keeps two callers on different event-loop threads from each building a
|
||||
# store and racing to install it. Under the GIL this is belt-and-braces,
|
||||
# but the guarantee it leans on is the interpreter's, not the code's.
|
||||
if _access_store is None:
|
||||
with _access_store_lock:
|
||||
if _access_store is None:
|
||||
_access_store = ComputerUseAccessStore()
|
||||
return _access_store
|
||||
@@ -0,0 +1,164 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Translate native App approval requests into the Core approval service."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from qwenpaw.app.approvals import ApprovalRequestSummary, get_approval_service
|
||||
from qwenpaw.config.context import (
|
||||
get_current_session_id as get_tool_session_id,
|
||||
)
|
||||
from qwenpaw.constant import TOOL_GUARD_APPROVAL_TIMEOUT_SECONDS
|
||||
from qwenpaw.security.tool_guard.approval import ApprovalDecision
|
||||
|
||||
from .access import (
|
||||
AppApprovalRequest,
|
||||
get_computer_use_access_store,
|
||||
)
|
||||
|
||||
_POST_APPROVAL_SETTLE_SECONDS = 0.8
|
||||
|
||||
|
||||
def _agent_context():
|
||||
"""Load request-scoped context only while resolving an approval."""
|
||||
from qwenpaw.app import agent_context
|
||||
|
||||
return agent_context
|
||||
|
||||
|
||||
class ComputerUseApprovalCoordinator:
|
||||
"""The plugin-side adapter for native-originated App approval events.
|
||||
|
||||
Holds no exemption state. A newly approved request waits out the native
|
||||
recency window before returning, so the approval click does not cause a
|
||||
spurious refusal while later user input remains guarded normally.
|
||||
"""
|
||||
|
||||
async def decide(self, message: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Resolve one reverse approval request without widening its scope."""
|
||||
request = self._app_request(message)
|
||||
if request is None:
|
||||
return {"allowed": False, "source": "invalid"}
|
||||
if not self._matches_active_session(request):
|
||||
return {"allowed": False, "source": "session_mismatch"}
|
||||
|
||||
store = get_computer_use_access_store()
|
||||
existing = store.resolve(request)
|
||||
if existing is not None:
|
||||
return {"allowed": existing.allowed, "source": existing.source}
|
||||
|
||||
pending = await self._create_pending(request)
|
||||
try:
|
||||
decision = await get_approval_service().wait_for_approval(
|
||||
pending.request_id,
|
||||
TOOL_GUARD_APPROVAL_TIMEOUT_SECONDS,
|
||||
)
|
||||
except Exception: # noqa: BLE001 - approval failures deny by default
|
||||
decision = ApprovalDecision.DENIED
|
||||
allowed = decision == ApprovalDecision.APPROVED
|
||||
store.record_session(request, allowed=allowed)
|
||||
if allowed:
|
||||
# The approving click is real user input. Let the native recency
|
||||
# guard age it out instead of weakening that guard with a bypass.
|
||||
await asyncio.sleep(_POST_APPROVAL_SETTLE_SECONDS)
|
||||
return {"allowed": allowed, "source": "session"}
|
||||
|
||||
@staticmethod
|
||||
def _matches_active_session(request: AppApprovalRequest) -> bool:
|
||||
agent_context = _agent_context()
|
||||
active_session = (
|
||||
agent_context.get_current_session_id()
|
||||
or get_tool_session_id()
|
||||
or ""
|
||||
)
|
||||
return bool(active_session and active_session == request.session_id)
|
||||
|
||||
@staticmethod
|
||||
def _app_request(message: Mapping[str, Any]) -> AppApprovalRequest | None:
|
||||
params = message.get("params")
|
||||
meta = message.get("meta")
|
||||
if not isinstance(params, Mapping) or not isinstance(meta, Mapping):
|
||||
return None
|
||||
app_id = str(params.get("canonical_app_id") or "").strip()
|
||||
session_id = str(meta.get("session_id") or "").strip()
|
||||
request_id = str(message.get("request_id") or "").strip()
|
||||
evidence = params.get("identity_evidence")
|
||||
if (
|
||||
not app_id
|
||||
or not session_id
|
||||
or not request_id
|
||||
or not isinstance(evidence, Mapping)
|
||||
):
|
||||
return None
|
||||
return AppApprovalRequest(
|
||||
request_id=request_id,
|
||||
session_id=session_id,
|
||||
canonical_app_id=app_id,
|
||||
display_name=str(params.get("display_name") or app_id),
|
||||
identity_evidence={
|
||||
str(key): str(value) for key, value in evidence.items()
|
||||
},
|
||||
risk=str(params.get("risk") or "unknown"),
|
||||
warning=str(params.get("warning") or ""),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _create_pending(request: AppApprovalRequest):
|
||||
agent_context = _agent_context()
|
||||
current_session = (
|
||||
agent_context.get_current_session_id() or request.session_id
|
||||
)
|
||||
root_session = (
|
||||
agent_context.get_current_root_session_id() or current_session
|
||||
)
|
||||
agent_id = agent_context.get_current_agent_id() or "unknown"
|
||||
summary = (
|
||||
f"Computer Use requests access to {request.display_name} "
|
||||
"for this session."
|
||||
)
|
||||
return await get_approval_service().create_pending_summary(
|
||||
session_id=current_session,
|
||||
root_session_id=root_session,
|
||||
owner_agent_id=agent_id,
|
||||
user_id=agent_context.get_current_user_id() or "",
|
||||
channel=agent_context.get_current_channel() or "",
|
||||
agent_id=agent_id,
|
||||
summary=ApprovalRequestSummary(
|
||||
source_type="computer_use_app_access",
|
||||
name="Computer Use",
|
||||
severity="medium",
|
||||
findings_count=1,
|
||||
result_summary=summary,
|
||||
payload={
|
||||
"canonical_app_id": request.canonical_app_id,
|
||||
"display_name": request.display_name,
|
||||
"risk": request.risk,
|
||||
},
|
||||
),
|
||||
timeout_seconds=TOOL_GUARD_APPROVAL_TIMEOUT_SECONDS,
|
||||
extra={
|
||||
"display": {
|
||||
"tool_name": "Computer Use",
|
||||
"tool_source": "app access",
|
||||
"exact_target": f"{request.display_name} for this session",
|
||||
"is_generalized": False,
|
||||
},
|
||||
"tool_call": {
|
||||
"id": f"computer_use:{request.request_id}",
|
||||
"name": "Computer Use",
|
||||
"input": {
|
||||
"canonical_app_id": request.canonical_app_id,
|
||||
"display_name": request.display_name,
|
||||
"risk": request.risk,
|
||||
"warning": request.warning,
|
||||
},
|
||||
},
|
||||
"computer_use_app": {
|
||||
"canonical_app_id": request.canonical_app_id,
|
||||
"display_name": request.display_name,
|
||||
},
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,662 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Controlled client for the host-managed Computer Use native runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from typing import Any
|
||||
|
||||
from qwenpaw.app.computer_use import (
|
||||
HostRuntimeProvider,
|
||||
RuntimeCapability,
|
||||
get_current_computer_use_turn_id,
|
||||
)
|
||||
from qwenpaw.config.context import (
|
||||
get_current_session_id as get_tool_session_id,
|
||||
)
|
||||
|
||||
from .approval import ComputerUseApprovalCoordinator
|
||||
from .protocol import (
|
||||
PROTOCOL_VERSION,
|
||||
ComputerUseProtocolError,
|
||||
NativeRequest,
|
||||
parse_response,
|
||||
)
|
||||
from .transport import (
|
||||
ComputerUseTransport,
|
||||
UnixSocketTransport,
|
||||
WindowsPipeTransport,
|
||||
)
|
||||
|
||||
_DEFAULT_DEADLINE_MS = 10000
|
||||
# The desktop host spawns the helper process while answering acquire; the
|
||||
# first spawn after an update can be slowed by antivirus scanning, and
|
||||
# frozen backends still use a short per-attempt socket timeout, so retry
|
||||
# the idempotent acquire a few times to cover that cold-start window.
|
||||
_ACQUIRE_ATTEMPTS = 5
|
||||
# A capability can name a helper that exited after announcing readiness but
|
||||
# before the client connected. Starting a native session is still idempotent
|
||||
# at that point: only the hello handshake has been sent, so discard the dead
|
||||
# endpoint and ask the host to start a fresh helper within the same tool call.
|
||||
_START_ATTEMPTS = 3
|
||||
# A broken connection can be replaced transparently only when repeating the
|
||||
# request cannot change the desktop. Mutating requests may have reached the
|
||||
# helper before the connection failed, so replaying them could act twice.
|
||||
_READ_ONLY_METHODS = frozenset(
|
||||
{"list_apps", "list_windows", "observe_window"},
|
||||
)
|
||||
_OBSERVED_METHODS = frozenset(
|
||||
{
|
||||
"click",
|
||||
"close_window",
|
||||
"drag",
|
||||
"invoke_element",
|
||||
"press_key",
|
||||
"scroll",
|
||||
"sequence",
|
||||
"set_value",
|
||||
"type_text",
|
||||
},
|
||||
)
|
||||
_REQUEST_ATTEMPTS = 2
|
||||
_BROKEN_TRANSPORT_ERRORS = frozenset(
|
||||
{
|
||||
"invalid_frame",
|
||||
"request_timeout",
|
||||
"runtime_disconnected",
|
||||
"runtime_unavailable",
|
||||
},
|
||||
)
|
||||
_RETRYABLE_READ_ONLY_ERRORS = _BROKEN_TRANSPORT_ERRORS | {
|
||||
# ScreenCaptureKit can miss its first callback while the capture service
|
||||
# wakes up. Observation is side-effect free, so one retry is safe; input
|
||||
# methods still never replay after an ambiguous failure.
|
||||
"capture_failed",
|
||||
}
|
||||
_DEAD_ENDPOINT_ERRORS = frozenset(
|
||||
{"invalid_frame", "runtime_disconnected", "runtime_unavailable"},
|
||||
)
|
||||
|
||||
# The helper refuses rather than queues when another session holds the desktop,
|
||||
# so the waiting is done here. Five attempts with doubling delays give a little
|
||||
# over two seconds, which covers one action's worth of contention; beyond that
|
||||
# the model is better told the desktop is busy than left waiting.
|
||||
_DESKTOP_BUSY_ATTEMPTS = 5
|
||||
_DESKTOP_BUSY_DELAY_SECONDS = 0.15
|
||||
_ACQUIRE_RETRY_DELAY_SECONDS = 0.5
|
||||
TransportFactory = Callable[[], ComputerUseTransport]
|
||||
|
||||
|
||||
class ComputerUseClient:
|
||||
"""Own one authenticated native connection for one QwenPaw session."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
session_id: str,
|
||||
transport_factory: TransportFactory | None = None,
|
||||
) -> None:
|
||||
self._session_id = session_id
|
||||
self._transport_factory = transport_factory
|
||||
self._transport: ComputerUseTransport | None = None
|
||||
# The capability this client's transport was built from, kept so a dead
|
||||
# endpoint can be reported back rather than reconnected to forever.
|
||||
self._capability: RuntimeCapability | None = None
|
||||
self._turn_id: str | None = None
|
||||
# Native observations are concurrency tokens, not model input. Keeping
|
||||
# the current token beside the connection prevents an agent from
|
||||
# copying, guessing, or crossing it with another action while native
|
||||
# still rejects stale state at the trust boundary.
|
||||
self._observation_id: str | None = None
|
||||
# The turn a stop applied to. Kept client-side so requests already
|
||||
# queued on this lock cannot cross the turn boundary after Native has
|
||||
# released its per-connection state.
|
||||
self._stopped_turn: str | None = None
|
||||
self._lock = asyncio.Lock()
|
||||
# The loop that created the transport, its reader task and this lock.
|
||||
# They may only be touched from there, and control routes arrive on the
|
||||
# HTTP server's loop instead.
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
self._approvals = ComputerUseApprovalCoordinator()
|
||||
|
||||
async def execute(
|
||||
self,
|
||||
method: str,
|
||||
params: Mapping[str, Any],
|
||||
*,
|
||||
deadline_ms: int = _DEFAULT_DEADLINE_MS,
|
||||
) -> dict[str, Any]:
|
||||
"""Execute one native operation through the authenticated transport."""
|
||||
turn_id = get_current_computer_use_turn_id()
|
||||
if not turn_id:
|
||||
raise ComputerUseProtocolError(
|
||||
"turn_unavailable",
|
||||
"Computer Use is unavailable outside an active agent turn.",
|
||||
)
|
||||
self._check_turn_active(turn_id)
|
||||
async with self._lock:
|
||||
# A stop can arrive while this request is queued behind another
|
||||
# operation. Recheck after acquiring the lock so queued work cannot
|
||||
# cross the turn boundary and recreate a transport afterwards.
|
||||
self._check_turn_active(turn_id, clear_previous=True)
|
||||
transport = await self._ensure_transport()
|
||||
if self._turn_id and self._turn_id != turn_id:
|
||||
await self._end_turn(transport, self._turn_id)
|
||||
self._observation_id = None
|
||||
self._turn_id = turn_id
|
||||
native_params = self._native_params(method, params)
|
||||
request_attempts = (
|
||||
_REQUEST_ATTEMPTS if method in _READ_ONLY_METHODS else 1
|
||||
)
|
||||
for request_attempt in range(request_attempts):
|
||||
try:
|
||||
result = await self._request_with_contention(
|
||||
transport,
|
||||
method,
|
||||
native_params,
|
||||
turn_id,
|
||||
deadline_ms,
|
||||
)
|
||||
self._check_turn_active(turn_id)
|
||||
return self._accept_result(method, result)
|
||||
except asyncio.CancelledError:
|
||||
# The native operation may already have changed the
|
||||
# desktop. Stop this turn and abandon only its connection;
|
||||
# the shared helper lets the bounded operation finish and
|
||||
# remains available to other sessions.
|
||||
self._stopped_turn = turn_id
|
||||
await self._discard_transport()
|
||||
raise
|
||||
except ComputerUseProtocolError as error:
|
||||
# A failed observation never replaces the old snapshot, and
|
||||
# a failed action may have changed the desktop before it
|
||||
# reported the error. Require fresh state in both cases.
|
||||
if (
|
||||
method == "observe_window"
|
||||
or method in _OBSERVED_METHODS
|
||||
):
|
||||
self._observation_id = None
|
||||
if error.code == "request_timeout":
|
||||
if method not in _READ_ONLY_METHODS:
|
||||
self._stopped_turn = turn_id
|
||||
await self._discard_transport()
|
||||
elif error.code in _BROKEN_TRANSPORT_ERRORS:
|
||||
await self._discard_transport()
|
||||
if error.code in _DEAD_ENDPOINT_ERRORS:
|
||||
# The endpoint is unusable, so the next acquire must
|
||||
# ask the host to verify or restart the helper.
|
||||
self._forget_capability()
|
||||
if (
|
||||
error.code not in _RETRYABLE_READ_ONLY_ERRORS
|
||||
or request_attempt + 1 >= request_attempts
|
||||
or turn_id == self._stopped_turn
|
||||
):
|
||||
raise
|
||||
transport = await self._ensure_transport()
|
||||
self._turn_id = turn_id
|
||||
raise ComputerUseProtocolError(
|
||||
"runtime_unavailable",
|
||||
"Computer Use native runtime is unavailable.",
|
||||
)
|
||||
|
||||
def _check_turn_active(
|
||||
self,
|
||||
turn_id: str,
|
||||
*,
|
||||
clear_previous: bool = False,
|
||||
) -> None:
|
||||
"""Reject the stopped turn and optionally retire an older marker."""
|
||||
if turn_id == self._stopped_turn:
|
||||
self._observation_id = None
|
||||
raise ComputerUseProtocolError(
|
||||
"turn_stopped",
|
||||
"Computer Use was stopped for this turn.",
|
||||
)
|
||||
if clear_previous:
|
||||
self._stopped_turn = None
|
||||
|
||||
def _native_params(
|
||||
self,
|
||||
method: str,
|
||||
params: Mapping[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Attach the current native observation without exposing its ID."""
|
||||
native_params = dict(params)
|
||||
if method not in _OBSERVED_METHODS:
|
||||
return native_params
|
||||
if not self._observation_id:
|
||||
raise ComputerUseProtocolError(
|
||||
"observation_required",
|
||||
"Observe a window before performing this action.",
|
||||
)
|
||||
native_params["observation_id"] = self._observation_id
|
||||
return native_params
|
||||
|
||||
def _accept_result(
|
||||
self,
|
||||
method: str,
|
||||
result: Mapping[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
"""Advance the observation and return only model-facing data."""
|
||||
public_result = dict(result)
|
||||
observation_id = public_result.pop("observation_id", None)
|
||||
if method == "observe_window" or method in _OBSERVED_METHODS:
|
||||
self._observation_id = (
|
||||
observation_id
|
||||
if isinstance(observation_id, str) and observation_id
|
||||
else None
|
||||
)
|
||||
elif method == "launch_app":
|
||||
self._observation_id = None
|
||||
return public_result
|
||||
|
||||
async def _request_with_contention(
|
||||
self,
|
||||
transport: ComputerUseTransport,
|
||||
method: str,
|
||||
params: Mapping[str, Any],
|
||||
turn_id: str,
|
||||
deadline_ms: int,
|
||||
) -> dict[str, Any]:
|
||||
"""Send once, except for refusals known to happen before any action."""
|
||||
for attempt in range(_DESKTOP_BUSY_ATTEMPTS):
|
||||
request = NativeRequest(
|
||||
request_id=uuid.uuid4().hex,
|
||||
method=method,
|
||||
params=params,
|
||||
session_id=self._session_id,
|
||||
turn_id=turn_id,
|
||||
deadline_ms=max(100, deadline_ms),
|
||||
)
|
||||
try:
|
||||
return parse_response(
|
||||
await transport.request(request.to_message()),
|
||||
)
|
||||
except ComputerUseProtocolError as error:
|
||||
if error.code != "desktop_busy":
|
||||
raise
|
||||
if attempt + 1 >= _DESKTOP_BUSY_ATTEMPTS:
|
||||
raise
|
||||
# Another session holds the desktop. The helper refuses before
|
||||
# touching it, so retrying cannot duplicate an action. Waiting
|
||||
# here also lets a stop interrupt the retry promptly.
|
||||
await asyncio.sleep(
|
||||
_DESKTOP_BUSY_DELAY_SECONDS * (2**attempt),
|
||||
)
|
||||
if turn_id == self._stopped_turn:
|
||||
raise ComputerUseProtocolError(
|
||||
"turn_stopped",
|
||||
"Computer Use was stopped for this turn.",
|
||||
) from error
|
||||
raise ComputerUseProtocolError(
|
||||
"desktop_busy",
|
||||
"Another Computer Use session is using the desktop.",
|
||||
)
|
||||
|
||||
@property
|
||||
def has_active_turn(self) -> bool:
|
||||
"""Whether this session currently owns a native Computer Use turn."""
|
||||
return self._transport is not None and self._turn_id is not None
|
||||
|
||||
async def stop_turn(self) -> bool:
|
||||
"""Stop this session's active turn without affecting other clients."""
|
||||
return await self._on_owner_loop(self._stop_turn_here)
|
||||
|
||||
async def _stop_turn_here(self) -> bool:
|
||||
"""Stop this turn after any already-dispatched operation settles."""
|
||||
turn_id = self._turn_id
|
||||
if self._transport is None or not turn_id:
|
||||
return False
|
||||
# Mark before waiting: requests already queued on the lock will see the
|
||||
# marker when they acquire it and must not reach the native connection.
|
||||
self._stopped_turn = turn_id
|
||||
self._observation_id = None
|
||||
async with self._lock:
|
||||
transport = self._transport
|
||||
if transport is not None and self._turn_id == turn_id:
|
||||
await self._end_turn(transport, turn_id)
|
||||
self._turn_id = None
|
||||
return True
|
||||
|
||||
async def close(self) -> None:
|
||||
"""End the active turn and close the client transport."""
|
||||
await self._on_owner_loop(self._close_here)
|
||||
|
||||
async def end_turn(self) -> bool:
|
||||
"""Release the native turn this session has finished with.
|
||||
|
||||
Keeps the connection, since the next turn will want it: the helper
|
||||
drops
|
||||
the turn's screenshots and accessibility handles and carries on
|
||||
serving.
|
||||
"""
|
||||
return await self._on_owner_loop(self._end_turn_here)
|
||||
|
||||
async def _end_turn_here(self) -> bool:
|
||||
async with self._lock:
|
||||
transport = self._transport
|
||||
turn_id = self._turn_id
|
||||
if transport is None or not turn_id:
|
||||
return False
|
||||
self._turn_id = None
|
||||
self._observation_id = None
|
||||
await self._end_turn(transport, turn_id)
|
||||
return True
|
||||
|
||||
@property
|
||||
def owner_loop(self) -> asyncio.AbstractEventLoop | None:
|
||||
"""The loop this client's transport and lock belong to, if
|
||||
connected."""
|
||||
return self._loop
|
||||
|
||||
async def _close_here(self) -> None:
|
||||
transport = self._transport
|
||||
if transport is None:
|
||||
return
|
||||
try:
|
||||
if self._turn_id:
|
||||
await self._end_turn(transport, self._turn_id)
|
||||
finally:
|
||||
self._turn_id = None
|
||||
self._observation_id = None
|
||||
self._transport = None
|
||||
await transport.close()
|
||||
|
||||
async def _on_owner_loop(
|
||||
self,
|
||||
operation: Callable[[], Awaitable[Any]],
|
||||
) -> Any:
|
||||
"""Run a client operation on the loop that owns its asyncio state.
|
||||
|
||||
The host runs one event loop per workspace, each on its own thread, and
|
||||
the control routes run on the HTTP server's loop. The transport's
|
||||
streams, its reader task and this client's lock all belong to whichever
|
||||
loop built them, so a coroutine touching them is handed back there
|
||||
rather than awaited here.
|
||||
"""
|
||||
loop = self._loop
|
||||
if loop is None or loop is asyncio.get_running_loop():
|
||||
return await operation()
|
||||
try:
|
||||
handle = asyncio.run_coroutine_threadsafe(operation(), loop)
|
||||
except RuntimeError:
|
||||
# The owning loop is gone, so its transport is unusable anyway.
|
||||
self._transport = None
|
||||
self._turn_id = None
|
||||
return None
|
||||
return await asyncio.wrap_future(handle)
|
||||
|
||||
async def _ensure_transport(self) -> ComputerUseTransport:
|
||||
if self._transport is not None:
|
||||
return self._transport
|
||||
# Everything created below belongs to this loop, so record it before
|
||||
# anything else can be asked to touch it from elsewhere.
|
||||
self._loop = asyncio.get_running_loop()
|
||||
if self._transport_factory is not None:
|
||||
transport = self._transport_factory()
|
||||
transport.set_reverse_request_handler(self._approvals.decide)
|
||||
await transport.connect()
|
||||
self._transport = transport
|
||||
return transport
|
||||
|
||||
for attempt in range(_START_ATTEMPTS):
|
||||
capability = await self._acquire_capability()
|
||||
if capability is None:
|
||||
raise ComputerUseProtocolError(
|
||||
"runtime_unavailable",
|
||||
"Computer Use native runtime is unavailable.",
|
||||
)
|
||||
transport = (
|
||||
WindowsPipeTransport(capability)
|
||||
if sys.platform == "win32"
|
||||
else UnixSocketTransport(capability)
|
||||
)
|
||||
# Remembered so a dead endpoint can be reported back to the
|
||||
# provider; the next acquire then asks the host for a live one.
|
||||
self._capability = capability
|
||||
transport.set_reverse_request_handler(self._approvals.decide)
|
||||
try:
|
||||
await transport.connect()
|
||||
except ComputerUseProtocolError as error:
|
||||
try:
|
||||
await transport.close()
|
||||
except Exception: # noqa: BLE001 - preserve connect failure
|
||||
pass
|
||||
# The endpoint named by this capability did not answer, which
|
||||
# is what a helper that has gone away looks like from here. A
|
||||
# later acquire must ask the host again.
|
||||
self._forget_capability()
|
||||
if (
|
||||
error.code
|
||||
not in {
|
||||
"invalid_frame",
|
||||
"request_timeout",
|
||||
"runtime_disconnected",
|
||||
"runtime_unavailable",
|
||||
}
|
||||
or attempt + 1 >= _START_ATTEMPTS
|
||||
):
|
||||
raise
|
||||
await asyncio.sleep(_ACQUIRE_RETRY_DELAY_SECONDS)
|
||||
continue
|
||||
self._transport = transport
|
||||
return transport
|
||||
raise ComputerUseProtocolError(
|
||||
"runtime_unavailable",
|
||||
"Computer Use native runtime is unavailable.",
|
||||
)
|
||||
|
||||
def _forget_capability(self) -> None:
|
||||
"""Report this client's endpoint as dead, so a fresh one is issued."""
|
||||
capability, self._capability = self._capability, None
|
||||
if capability is not None:
|
||||
HostRuntimeProvider.invalidate_capability(capability)
|
||||
|
||||
@staticmethod
|
||||
async def _acquire_capability():
|
||||
"""Acquire the host capability, retrying cold-start misses."""
|
||||
for attempt in range(_ACQUIRE_ATTEMPTS):
|
||||
# The provider call blocks on a control socket; keep it off the
|
||||
# event loop so other sessions stay responsive.
|
||||
capability = await asyncio.to_thread(
|
||||
HostRuntimeProvider.acquire_capability,
|
||||
)
|
||||
if capability is not None:
|
||||
if capability.protocol_version != PROTOCOL_VERSION:
|
||||
raise ComputerUseProtocolError(
|
||||
"protocol_mismatch",
|
||||
"Computer Use plugin and desktop runtime "
|
||||
"versions are incompatible.",
|
||||
)
|
||||
return capability
|
||||
if attempt + 1 < _ACQUIRE_ATTEMPTS:
|
||||
await asyncio.sleep(_ACQUIRE_RETRY_DELAY_SECONDS)
|
||||
return None
|
||||
|
||||
async def _end_turn(
|
||||
self,
|
||||
transport: ComputerUseTransport,
|
||||
turn_id: str,
|
||||
) -> None:
|
||||
request = NativeRequest(
|
||||
request_id=uuid.uuid4().hex,
|
||||
method="end_turn",
|
||||
params={},
|
||||
session_id=self._session_id,
|
||||
turn_id=turn_id,
|
||||
deadline_ms=2000,
|
||||
)
|
||||
try:
|
||||
parse_response(await transport.request(request.to_message()))
|
||||
except ComputerUseProtocolError:
|
||||
pass
|
||||
|
||||
async def _discard_transport(self) -> None:
|
||||
"""Detach and close the current transport, ignoring shutdown errors."""
|
||||
transport = self._transport
|
||||
self._transport = None
|
||||
self._turn_id = None
|
||||
self._observation_id = None
|
||||
if transport is None:
|
||||
return
|
||||
try:
|
||||
await transport.close()
|
||||
except Exception:
|
||||
# Closing a broken pipe can raise transport errors; ignore them so
|
||||
# the caller can re-raise its own original failure.
|
||||
pass
|
||||
|
||||
|
||||
_clients: dict[str, ComputerUseClient] = {}
|
||||
# The cache is read and written from more than one event loop -- the host runs
|
||||
# one per workspace on its own thread -- so a plain dict could be mutated while
|
||||
# another thread iterates it during eviction. The lock covers the get-or-create
|
||||
# and eviction paths; per-client work happens outside it, guarded by the
|
||||
# client's own async lock.
|
||||
_clients_lock = threading.Lock()
|
||||
|
||||
# A client caches the per-session native turn that outlives a single tool call.
|
||||
# Nothing tells the plugin when a session is gone, so the cache is bounded
|
||||
# instead: on insert, idle sessions are dropped oldest-first. The backend is a
|
||||
# long-lived desktop process, so an unbounded dict would keep every session
|
||||
# ever seen.
|
||||
_MAX_CACHED_CLIENTS = 64
|
||||
|
||||
|
||||
def _retire(client: ComputerUseClient) -> None:
|
||||
"""Close an evicted client's connection, best effort.
|
||||
|
||||
Dropping the reference alone would leave the pipe or socket open until the
|
||||
object happened to be collected. Closing needs to await, and this runs from
|
||||
synchronous code holding a threading lock, so the coroutine is handed to
|
||||
the
|
||||
loop that owns the transport and not waited on.
|
||||
"""
|
||||
loop = client.owner_loop
|
||||
if loop is None:
|
||||
return
|
||||
try:
|
||||
asyncio.run_coroutine_threadsafe(client.close(), loop)
|
||||
except RuntimeError:
|
||||
# That loop has stopped, so its transport is already unusable.
|
||||
pass
|
||||
|
||||
|
||||
def _evict_idle_clients() -> list[ComputerUseClient]:
|
||||
"""Drop cached clients for sessions with no turn in flight.
|
||||
|
||||
The caller holds ``_clients_lock``. Returns the clients removed so the
|
||||
caller can close them outside the lock.
|
||||
"""
|
||||
if len(_clients) < _MAX_CACHED_CLIENTS:
|
||||
return []
|
||||
evicted = []
|
||||
for session_id, client in list(_clients.items()):
|
||||
if len(_clients) < _MAX_CACHED_CLIENTS:
|
||||
break
|
||||
if not client.has_active_turn:
|
||||
del _clients[session_id]
|
||||
evicted.append(client)
|
||||
return evicted
|
||||
|
||||
|
||||
def get_computer_use_client() -> ComputerUseClient:
|
||||
"""Return the controlled client for the active QwenPaw session."""
|
||||
# The request-context module initializes the web workspace stack. Defer it
|
||||
# until session lookup so transport and protocol code remain lightweight.
|
||||
from qwenpaw.app.agent_context import get_current_session_id
|
||||
|
||||
session_id = get_current_session_id() or get_tool_session_id() or ""
|
||||
if not session_id:
|
||||
raise ComputerUseProtocolError(
|
||||
"session_unavailable",
|
||||
"Computer Use requires an active session.",
|
||||
)
|
||||
with _clients_lock:
|
||||
client = _clients.get(session_id)
|
||||
evicted: list[ComputerUseClient] = []
|
||||
if client is None:
|
||||
evicted = _evict_idle_clients()
|
||||
if len(_clients) >= _MAX_CACHED_CLIENTS:
|
||||
# Every cached session still claims a turn. Refusing keeps the
|
||||
# bound real and makes the situation visible, where growing the
|
||||
# cache would quietly hold a connection per session forever.
|
||||
raise ComputerUseProtocolError(
|
||||
"too_many_sessions",
|
||||
"Too many Computer Use sessions are active; "
|
||||
"finish or stop one before starting another.",
|
||||
)
|
||||
client = ComputerUseClient(session_id)
|
||||
_clients[session_id] = client
|
||||
for retired in evicted:
|
||||
_retire(retired)
|
||||
return client
|
||||
|
||||
|
||||
def _cached_client(session_id: str) -> ComputerUseClient | None:
|
||||
"""Look up a session's client under the cache lock.
|
||||
|
||||
Control routes reach the cache from the HTTP server's thread while a
|
||||
workspace thread may be inserting or evicting, so every read takes the lock
|
||||
the rest of this module already uses.
|
||||
"""
|
||||
with _clients_lock:
|
||||
return _clients.get(session_id)
|
||||
|
||||
|
||||
def is_computer_use_active(session_id: str) -> bool:
|
||||
"""Return whether a session owns an active native Computer Use turn."""
|
||||
client = _cached_client(session_id)
|
||||
return client.has_active_turn if client is not None else False
|
||||
|
||||
|
||||
async def stop_computer_use_session(session_id: str) -> bool:
|
||||
"""Stop the native Computer Use turn currently owned by one session."""
|
||||
client = _cached_client(session_id)
|
||||
return await client.stop_turn() if client is not None else False
|
||||
|
||||
|
||||
async def end_computer_use_turn(session_id: str) -> bool:
|
||||
"""Release the native turn a finished request was holding.
|
||||
|
||||
The turn id is minted per request by the host, and nothing used to retire
|
||||
it: a session that used the tool once kept its turn -- and the helper's
|
||||
screenshots and accessibility handles -- until the next call happened to
|
||||
supply a new id. That also made the cache bound unreachable, since a client
|
||||
holding a turn is never evicted.
|
||||
"""
|
||||
client = _cached_client(session_id)
|
||||
return await client.end_turn() if client is not None else False
|
||||
|
||||
|
||||
def known_computer_use_sessions() -> list[str]:
|
||||
"""Every session this process holds a Computer Use client for.
|
||||
|
||||
A pending approval can only exist for one of these: the helper asks through
|
||||
the connection a client owns, so the request carries that client's session.
|
||||
Turning the feature off therefore has to reach all of them, not only the
|
||||
session whoever flipped the switch happened to be looking at.
|
||||
"""
|
||||
with _clients_lock:
|
||||
return list(_clients)
|
||||
|
||||
|
||||
async def stop_all_computer_use_turns() -> int:
|
||||
"""Stop every active native turn across all known sessions.
|
||||
|
||||
Used when the feature is switched off so no automation keeps running.
|
||||
Returns the number of turns that were actually stopped.
|
||||
"""
|
||||
stopped = 0
|
||||
# Snapshot under the lock, then stop turns without holding it: stop_turn
|
||||
# awaits native I/O, and the lock is a sync primitive that must not be held
|
||||
# across an await.
|
||||
with _clients_lock:
|
||||
clients = list(_clients.values())
|
||||
for client in clients:
|
||||
if await client.stop_turn():
|
||||
stopped += 1
|
||||
return stopped
|
||||
@@ -0,0 +1,553 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Thin tool adapter for the host-managed Computer Use runtime."""
|
||||
|
||||
# NOTE: no `from __future__ import annotations` here, deliberately. The tool
|
||||
# entry point below is handed to the runtime's JSON-schema builder, which
|
||||
# resolves annotations in a namespace without our typing imports; stringized
|
||||
# annotations would abort the toolkit build. Under Python 3.11 every
|
||||
# annotation in this module evaluates fine at definition time.
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from typing import Any, Literal, Mapping
|
||||
|
||||
from agentscope.message import (
|
||||
Base64Source,
|
||||
DataBlock,
|
||||
TextBlock,
|
||||
ToolResultState,
|
||||
URLSource,
|
||||
)
|
||||
from agentscope.tool import ToolChunk
|
||||
|
||||
from qwenpaw.runtime.tool_registry import tool_descriptor
|
||||
|
||||
from .client import get_computer_use_client
|
||||
from .feature_state import get_computer_use_feature_state
|
||||
from .protocol import ComputerUseProtocolError
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
_MAX_ACTIONS_PER_MINUTE = 60
|
||||
_action_times: list[float] = []
|
||||
_rate_limit_lock = threading.Lock()
|
||||
_SCREENSHOT_URL_PLACEHOLDER = "<image delivered as a separate attachment>"
|
||||
_MAX_ACCESSIBILITY_DEPTH = 40
|
||||
|
||||
ComputerUseAction = Literal[
|
||||
"list_apps",
|
||||
"list_windows",
|
||||
"observe_window",
|
||||
"launch_app",
|
||||
"close_window",
|
||||
"click",
|
||||
"double_click",
|
||||
"right_click",
|
||||
"scroll",
|
||||
"drag",
|
||||
"type",
|
||||
"press_key",
|
||||
"sequence",
|
||||
"invoke",
|
||||
"begin_text_edit",
|
||||
"set_value",
|
||||
"wait",
|
||||
"stop",
|
||||
]
|
||||
|
||||
|
||||
def _check_rate_limit(cost: int = 1) -> None:
|
||||
# The tool can be entered from more than one event loop -- the host runs
|
||||
# per-workspace loops on their own threads -- so the guard is a threading
|
||||
# lock rather than an asyncio one, which serialises only within a single
|
||||
# loop. Under the GIL the unguarded check-then-append is narrow enough that
|
||||
# overshooting the cap could not be provoked, but that is a property of the
|
||||
# interpreter rather than of this code, and a free-threaded build removes
|
||||
# it. The body has no await, so the lock is held briefly.
|
||||
with _rate_limit_lock:
|
||||
now = time.monotonic()
|
||||
_action_times[:] = [
|
||||
value for value in _action_times if now - value < 60
|
||||
]
|
||||
if len(_action_times) + cost > _MAX_ACTIONS_PER_MINUTE:
|
||||
raise ComputerUseProtocolError(
|
||||
"rate_limited",
|
||||
"Computer Use rate limit exceeded; wait before continuing.",
|
||||
)
|
||||
_action_times.extend([now] * cost)
|
||||
|
||||
|
||||
def _sequence_steps(steps: Any) -> list[dict[str, str]]:
|
||||
"""Validate the bounded keyboard-only sequence contract."""
|
||||
if isinstance(steps, str):
|
||||
try:
|
||||
steps = json.loads(steps)
|
||||
except json.JSONDecodeError as error:
|
||||
raise ValueError(
|
||||
"sequence steps must be a JSON array.",
|
||||
) from error
|
||||
if not isinstance(steps, list) or not 1 <= len(steps) <= 20:
|
||||
raise ValueError("sequence requires 1 to 20 steps.")
|
||||
normalized = []
|
||||
text_length = 0
|
||||
for index, step in enumerate(steps):
|
||||
if not isinstance(step, Mapping):
|
||||
raise ValueError(f"sequence step {index} must be an object.")
|
||||
action = str(step.get("action") or "").strip().lower()
|
||||
if action not in {"type", "press_key"}:
|
||||
raise ValueError(
|
||||
f"sequence step {index} must use type or press_key.",
|
||||
)
|
||||
field = "text" if action == "type" else "key"
|
||||
value = step.get(field)
|
||||
if (
|
||||
not isinstance(value, str)
|
||||
or not value
|
||||
or (action == "press_key" and not value.strip())
|
||||
):
|
||||
raise ValueError(
|
||||
f"sequence step {index} requires non-empty {field}.",
|
||||
)
|
||||
if set(step) != {"action", field}:
|
||||
raise ValueError(
|
||||
f"sequence step {index} accepts only action and {field}.",
|
||||
)
|
||||
if action == "type":
|
||||
text_length += len(value)
|
||||
if text_length > 512:
|
||||
raise ValueError("sequence text is limited to 512 characters.")
|
||||
normalized.append({"action": action, field: value})
|
||||
return normalized
|
||||
|
||||
|
||||
def _without_screenshot_urls(
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
attached: bool,
|
||||
) -> Mapping[str, Any]:
|
||||
"""Remove image data from text output, retaining metadata when attached.
|
||||
|
||||
Screenshots are attached as image blocks; repeating the base64 data
|
||||
URL inside the JSON text block would double a multi-megabyte payload
|
||||
and pollute the model's text context. Post-action refreshes are semantic
|
||||
and do not capture images; a visual target starts with a fresh observation.
|
||||
"""
|
||||
screenshots = payload.get("screenshots")
|
||||
if not isinstance(screenshots, list):
|
||||
return payload
|
||||
if not attached:
|
||||
return {
|
||||
key: value
|
||||
for key, value in payload.items()
|
||||
if key != "screenshots"
|
||||
}
|
||||
sanitized: list[Any] = []
|
||||
for screenshot in screenshots:
|
||||
if isinstance(screenshot, Mapping) and "url" in screenshot:
|
||||
sanitized.append(
|
||||
{**screenshot, "url": _SCREENSHOT_URL_PLACEHOLDER},
|
||||
)
|
||||
else:
|
||||
sanitized.append(screenshot)
|
||||
return {**payload, "screenshots": sanitized}
|
||||
|
||||
|
||||
def _element_line(element: Mapping[str, Any]) -> str:
|
||||
"""Render one accessibility element as a single compact line.
|
||||
|
||||
Only the model reads these elements, so the JSON scaffolding around
|
||||
them is pure overhead. Coordinates come from the current screenshot;
|
||||
accessibility lines expose only semantic element metadata.
|
||||
"""
|
||||
parts = [
|
||||
str(element.get("id") or "?"),
|
||||
str(element.get("control_type_name") or element.get("role") or "?"),
|
||||
f'"{element.get("name") or ""}"',
|
||||
]
|
||||
value = element.get("value")
|
||||
if isinstance(value, str) and value:
|
||||
parts.append(f"={value}")
|
||||
identifier = element.get("identifier") or element.get("automation_id")
|
||||
if isinstance(identifier, str) and identifier:
|
||||
parts.append(f"[identifier={identifier}]")
|
||||
# Both states stay visible: an offscreen entry may become reachable
|
||||
# after scrolling, and a disabled control tells the model not to try.
|
||||
if element.get("enabled") is False:
|
||||
parts.append("[disabled]")
|
||||
if element.get("offscreen") is True:
|
||||
parts.append("[offscreen]")
|
||||
if element.get("selected") is True:
|
||||
parts.append("[selected]")
|
||||
if element.get("settable") is True:
|
||||
parts.append("[settable]")
|
||||
if element.get("resource_backed") is True:
|
||||
parts.append("[resource-backed]")
|
||||
actions = element.get("actions")
|
||||
if isinstance(actions, list):
|
||||
names = [str(action) for action in actions if str(action)]
|
||||
if names:
|
||||
parts.append(f"[actions={','.join(names)}]")
|
||||
depth = element.get("depth")
|
||||
indent = (
|
||||
" " * min(depth, _MAX_ACCESSIBILITY_DEPTH)
|
||||
if isinstance(depth, int) and not isinstance(depth, bool) and depth > 0
|
||||
else ""
|
||||
)
|
||||
return indent + " ".join(parts)
|
||||
|
||||
|
||||
def _with_compact_elements(payload: Mapping[str, Any]) -> Mapping[str, Any]:
|
||||
"""Replace the accessibility element objects with one line each."""
|
||||
accessibility = payload.get("accessibility")
|
||||
if not isinstance(accessibility, Mapping):
|
||||
return payload
|
||||
elements = accessibility.get("elements")
|
||||
if not isinstance(elements, list):
|
||||
return payload
|
||||
lines = [
|
||||
_element_line(element)
|
||||
for element in elements
|
||||
if isinstance(element, Mapping)
|
||||
]
|
||||
compact = {
|
||||
key: value for key, value in accessibility.items() if key != "elements"
|
||||
}
|
||||
compact["elements"] = "\n".join(lines)
|
||||
return {**payload, "accessibility": compact}
|
||||
|
||||
|
||||
def _screenshot_source(url: str) -> Base64Source | URLSource:
|
||||
header, separator, data = url.partition(",")
|
||||
if (
|
||||
separator
|
||||
and header.casefold().startswith("data:")
|
||||
and ";base64" in header.casefold()
|
||||
):
|
||||
media_type = header[5:].split(";", 1)[0] or "image/*"
|
||||
return Base64Source(data=data, media_type=media_type)
|
||||
return URLSource(url=url, media_type="image/*")
|
||||
|
||||
|
||||
def _response(
|
||||
payload: Mapping[str, Any],
|
||||
*,
|
||||
include_images: bool = False,
|
||||
state: ToolResultState = ToolResultState.SUCCESS,
|
||||
) -> ToolChunk:
|
||||
content: list[Any] = []
|
||||
if include_images:
|
||||
for screenshot in payload.get("screenshots", []):
|
||||
if isinstance(screenshot, Mapping) and isinstance(
|
||||
screenshot.get("url"),
|
||||
str,
|
||||
):
|
||||
content.append(
|
||||
DataBlock(
|
||||
source=_screenshot_source(screenshot["url"]),
|
||||
),
|
||||
)
|
||||
content.append(
|
||||
TextBlock(
|
||||
type="text",
|
||||
text=json.dumps(
|
||||
_with_compact_elements(
|
||||
_without_screenshot_urls(payload, attached=include_images),
|
||||
),
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
),
|
||||
),
|
||||
)
|
||||
return ToolChunk(content=content, state=state, is_last=True)
|
||||
|
||||
|
||||
def _error(code: str, message: str) -> ToolChunk:
|
||||
payload = {
|
||||
"ok": False,
|
||||
"error": {"code": code, "message": message},
|
||||
}
|
||||
if code in {
|
||||
"desktop_busy",
|
||||
"focus_failed",
|
||||
"input_failed",
|
||||
"observation_required",
|
||||
"stale_observation",
|
||||
"target_not_at_point",
|
||||
"unknown_screenshot",
|
||||
"user_intervention",
|
||||
}:
|
||||
payload["requires_observe"] = True
|
||||
payload["next_action"] = "observe_window"
|
||||
elif code in {"stale_window", "window_not_found"}:
|
||||
payload["requires_observe"] = True
|
||||
payload["next_action"] = "list_windows"
|
||||
return _response(
|
||||
payload,
|
||||
state=ToolResultState.ERROR,
|
||||
)
|
||||
|
||||
|
||||
@tool_descriptor(
|
||||
name="computer_use",
|
||||
enabled_by_default=True,
|
||||
async_execution=True,
|
||||
description=(
|
||||
"Control approved desktop applications through the native "
|
||||
"Computer Use runtime. Observe a window before acting; the runtime "
|
||||
"keeps the current observation synchronized between actions."
|
||||
),
|
||||
requires_skills=("computer_use",),
|
||||
)
|
||||
async def computer_use(
|
||||
action: ComputerUseAction,
|
||||
app: str = "",
|
||||
window_id: str = "",
|
||||
screenshot_id: str = "",
|
||||
element_id: str = "",
|
||||
x: int = 0,
|
||||
y: int = 0,
|
||||
start_x: int = 0,
|
||||
start_y: int = 0,
|
||||
end_x: int = 0,
|
||||
end_y: int = 0,
|
||||
source_element_id: str = "",
|
||||
target_element_id: str = "",
|
||||
button: str = "left",
|
||||
count: int = 1,
|
||||
delta_y: int = 0,
|
||||
text: str = "",
|
||||
value: str = "",
|
||||
key: str = "",
|
||||
steps: list[dict[str, Any]] | str | None = None,
|
||||
wait_ms: int = 500,
|
||||
timeout_ms: int = 10000,
|
||||
) -> ToolChunk:
|
||||
"""Control one observed window at a time.
|
||||
|
||||
Use ``list_apps`` or ``list_windows`` first. Observe a target with
|
||||
``observe_window`` before acting. The client advances the native
|
||||
observation after every successful action; native rejects stale state.
|
||||
``launch_app`` accepts an App ID returned by ``list_apps`` or an absolute
|
||||
platform-native application path.
|
||||
``observe_window`` returns screenshots and accessibility text.
|
||||
Coordinate actions require the ``id`` of an attached screenshot as
|
||||
``screenshot_id``; coordinates are local to that image.
|
||||
Inspect the replacement observation after an action changes selection,
|
||||
focus, menus, editors, dialogs, or windows. Confirm editable focus before
|
||||
typing, and observe again after committing an edit.
|
||||
"""
|
||||
# Each early return maps to one refusal reason the model must be able to
|
||||
# tell apart, so they are reported individually rather than merged.
|
||||
# pylint: disable=too-many-return-statements
|
||||
try:
|
||||
action = str(action or "").strip().lower()
|
||||
if not action:
|
||||
raise ValueError("action is required.")
|
||||
if not get_computer_use_feature_state().is_enabled():
|
||||
return _error(
|
||||
"feature_disabled",
|
||||
"Computer Use is turned off. Enable it in the Computer Use "
|
||||
"panel to allow desktop automation.",
|
||||
)
|
||||
if action == "wait":
|
||||
_check_rate_limit()
|
||||
await asyncio.sleep(max(0, min(wait_ms, 30_000)) / 1000)
|
||||
return _response(
|
||||
{"ok": True, "action": action, "waited_ms": wait_ms},
|
||||
)
|
||||
|
||||
client = get_computer_use_client()
|
||||
if action == "stop":
|
||||
_check_rate_limit()
|
||||
await client.stop_turn()
|
||||
return _response({"ok": True, "action": action})
|
||||
|
||||
method, params, include_images = _native_request(
|
||||
action,
|
||||
app=app,
|
||||
window_id=window_id,
|
||||
screenshot_id=screenshot_id,
|
||||
element_id=element_id,
|
||||
x=x,
|
||||
y=y,
|
||||
start_x=start_x,
|
||||
start_y=start_y,
|
||||
end_x=end_x,
|
||||
end_y=end_y,
|
||||
source_element_id=source_element_id,
|
||||
target_element_id=target_element_id,
|
||||
button=button,
|
||||
count=count,
|
||||
delta_y=delta_y,
|
||||
text=text,
|
||||
value=value,
|
||||
key=key,
|
||||
steps=steps,
|
||||
)
|
||||
if method == "sequence":
|
||||
_check_rate_limit(len(params["steps"]))
|
||||
else:
|
||||
_check_rate_limit()
|
||||
result = await client.execute(
|
||||
method,
|
||||
params,
|
||||
deadline_ms=max(100, min(timeout_ms, 30_000)),
|
||||
)
|
||||
failed = action == "sequence" and isinstance(
|
||||
result.get("error"),
|
||||
Mapping,
|
||||
)
|
||||
payload = {"ok": not failed, "action": action, **result}
|
||||
return _response(
|
||||
payload,
|
||||
include_images=include_images,
|
||||
state=ToolResultState.ERROR if failed else ToolResultState.SUCCESS,
|
||||
)
|
||||
except ComputerUseProtocolError as error:
|
||||
return _error(error.code, str(error))
|
||||
except ValueError as error:
|
||||
return _error("invalid_request", str(error))
|
||||
except (
|
||||
Exception
|
||||
) as error: # noqa: BLE001 - tool calls must not escape errors
|
||||
# A tool entry point must not raise, but the errors that reach here are
|
||||
# the unexpected ones -- an attribute error, a bad type, a broken
|
||||
# import -- not the protocol failures handled above. Collapsing them to
|
||||
# one message keeps the turn alive; logging the traceback first keeps
|
||||
# them diagnosable rather than lost behind "Computer Use failed".
|
||||
_LOGGER.exception("Computer Use tool call failed unexpectedly")
|
||||
return _error("tool_failed", f"Computer Use failed: {error}")
|
||||
|
||||
|
||||
def _native_request(
|
||||
action: str,
|
||||
**values: Any,
|
||||
) -> tuple[str, dict[str, Any], bool]:
|
||||
# One branch per action keeps the whole request contract readable in a
|
||||
# single place; splitting it per action would scatter the protocol.
|
||||
# pylint: disable=too-many-return-statements
|
||||
# pylint: disable=too-many-branches, too-many-statements
|
||||
if action == "list_apps":
|
||||
return action, {}, False
|
||||
if action == "list_windows":
|
||||
app = str(values["app"] or "").strip()
|
||||
return action, ({"app": app} if app else {}), False
|
||||
if action == "launch_app":
|
||||
app = str(values["app"] or "").strip()
|
||||
if not app:
|
||||
raise ValueError(
|
||||
"launch_app requires an App ID or an absolute .exe path.",
|
||||
)
|
||||
return action, {"app": app}, False
|
||||
|
||||
if action == "observe_window":
|
||||
window_id = str(values["window_id"] or "").strip()
|
||||
if not window_id:
|
||||
raise ValueError(
|
||||
"observe_window requires window_id from list_windows.",
|
||||
)
|
||||
return action, {"window_id": window_id}, True
|
||||
if action == "close_window":
|
||||
return action, {}, False
|
||||
if action in {"click", "double_click", "right_click"}:
|
||||
params = {}
|
||||
element_id = str(values.get("element_id") or "").strip()
|
||||
if element_id:
|
||||
params["element_id"] = element_id
|
||||
else:
|
||||
params["screenshot_id"] = _screenshot_id(values)
|
||||
params["x"] = values["x"]
|
||||
params["y"] = values["y"]
|
||||
params["button"] = (
|
||||
"right" if action == "right_click" else values["button"]
|
||||
)
|
||||
params["count"] = 2 if action == "double_click" else values["count"]
|
||||
return "click", params, False
|
||||
if action == "scroll":
|
||||
params = {
|
||||
"screenshot_id": _screenshot_id(values),
|
||||
"x": values["x"],
|
||||
"y": values["y"],
|
||||
}
|
||||
params["delta_y"] = values["delta_y"]
|
||||
return action, params, False
|
||||
if action == "drag":
|
||||
source_element_id = str(
|
||||
values.get("source_element_id") or "",
|
||||
).strip()
|
||||
target_element_id = str(
|
||||
values.get("target_element_id") or "",
|
||||
).strip()
|
||||
if bool(source_element_id) != bool(target_element_id):
|
||||
raise ValueError(
|
||||
"drag requires both source_element_id and "
|
||||
"target_element_id, or neither.",
|
||||
)
|
||||
params = {}
|
||||
if source_element_id:
|
||||
params.update(
|
||||
source_element_id=source_element_id,
|
||||
target_element_id=target_element_id,
|
||||
)
|
||||
else:
|
||||
params.update(
|
||||
screenshot_id=_screenshot_id(values),
|
||||
start_x=values["start_x"],
|
||||
start_y=values["start_y"],
|
||||
end_x=values["end_x"],
|
||||
end_y=values["end_y"],
|
||||
)
|
||||
return action, params, False
|
||||
if action == "type":
|
||||
text = str(values["text"] or "")
|
||||
if not text:
|
||||
raise ValueError("type requires non-empty text.")
|
||||
return (
|
||||
"type_text",
|
||||
{"text": text},
|
||||
False,
|
||||
)
|
||||
if action in {"invoke", "begin_text_edit", "set_value"}:
|
||||
element_id = str(values["element_id"] or "").strip()
|
||||
if not element_id:
|
||||
raise ValueError(
|
||||
f"{action} requires element_id from observe_window.",
|
||||
)
|
||||
params = {"element_id": element_id}
|
||||
if action == "begin_text_edit":
|
||||
params["expects_text_input"] = True
|
||||
if action == "set_value":
|
||||
params["value"] = str(values["value"] or "")
|
||||
return (
|
||||
(
|
||||
"invoke_element"
|
||||
if action in {"invoke", "begin_text_edit"}
|
||||
else action
|
||||
),
|
||||
params,
|
||||
False,
|
||||
)
|
||||
if action == "press_key":
|
||||
key = str(values["key"] or "").strip()
|
||||
if not key:
|
||||
raise ValueError("press_key requires key.")
|
||||
return action, {"key": key}, False
|
||||
if action == "sequence":
|
||||
return action, {"steps": _sequence_steps(values.get("steps"))}, False
|
||||
raise ValueError(
|
||||
"Unknown action. Valid actions: list_apps, list_windows, "
|
||||
"observe_window, launch_app, close_window, click, "
|
||||
"double_click, right_click, scroll, drag, type, press_key, invoke, "
|
||||
"begin_text_edit, set_value, sequence, wait, stop.",
|
||||
)
|
||||
|
||||
|
||||
def _screenshot_id(values: Mapping[str, Any]) -> str:
|
||||
screenshot_id = str(values.get("screenshot_id") or "").strip()
|
||||
if not screenshot_id:
|
||||
raise ValueError(
|
||||
"Coordinate input requires screenshot_id from observe_window.",
|
||||
)
|
||||
return screenshot_id
|
||||
@@ -0,0 +1,106 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Installation-scoped on/off switch for the Computer Use feature."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from threading import RLock
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _default_state_path() -> Path:
|
||||
from qwenpaw.constant import WORKING_DIR
|
||||
|
||||
return (
|
||||
Path(WORKING_DIR)
|
||||
/ "plugin_runtime"
|
||||
/ "computer-use"
|
||||
/ "feature_state.json"
|
||||
)
|
||||
|
||||
|
||||
class ComputerUseFeatureState:
|
||||
"""Persist whether the Computer Use feature is allowed on this host.
|
||||
|
||||
The feature is enabled by default so existing installations keep their
|
||||
current behaviour; the user can turn it off from the plugin page. The
|
||||
flag is installation-scoped and survives restarts.
|
||||
|
||||
That default applies only when no decision has been recorded. A state file
|
||||
that exists but cannot be read is an unknown, not a default, and an unknown
|
||||
resolves to off -- the alternative is quietly restoring desktop access that
|
||||
the user may have switched off.
|
||||
"""
|
||||
|
||||
def __init__(self, persistent_path: Path | None = None) -> None:
|
||||
self._lock = RLock()
|
||||
self._persistent_path = persistent_path or _default_state_path()
|
||||
self._enabled = self._load()
|
||||
|
||||
def is_enabled(self) -> bool:
|
||||
"""Return whether desktop automation is currently allowed."""
|
||||
with self._lock:
|
||||
return self._enabled
|
||||
|
||||
def set_enabled(self, value: bool) -> None:
|
||||
"""Persist a new enabled/disabled decision for this installation."""
|
||||
with self._lock:
|
||||
self._enabled = bool(value)
|
||||
self._save_locked()
|
||||
|
||||
def _load(self) -> bool:
|
||||
try:
|
||||
with self._persistent_path.open(encoding="utf-8") as file:
|
||||
payload = json.load(file)
|
||||
except FileNotFoundError:
|
||||
# No decision has been recorded yet, so the installation default
|
||||
# applies. Nothing is lost by starting from it: the feature being
|
||||
# on means an agent may ask, and every action on an application
|
||||
# still waits for the user to approve that application.
|
||||
return True
|
||||
except (OSError, json.JSONDecodeError) as error:
|
||||
# A file that exists but cannot be read is different: the user may
|
||||
# have turned the feature off, and reading that as on would put a
|
||||
# decision about desktop access back the way they did not choose.
|
||||
_LOGGER.warning(
|
||||
"Computer Use feature state at %s is unreadable (%s); "
|
||||
"treating the feature as off until it is set again.",
|
||||
self._persistent_path,
|
||||
error,
|
||||
)
|
||||
return False
|
||||
enabled = payload.get("enabled")
|
||||
if isinstance(enabled, bool):
|
||||
return enabled
|
||||
# Parsed, but says nothing this code understands -- the same unknown as
|
||||
# an unreadable file.
|
||||
_LOGGER.warning(
|
||||
"Computer Use feature state at %s has no usable 'enabled' flag; "
|
||||
"treating the feature as off until it is set again.",
|
||||
self._persistent_path,
|
||||
)
|
||||
return False
|
||||
|
||||
def _save_locked(self) -> None:
|
||||
self._persistent_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary_path = self._persistent_path.with_suffix(".tmp")
|
||||
temporary_path.write_text(
|
||||
json.dumps({"enabled": self._enabled}, indent=2, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.replace(temporary_path, self._persistent_path)
|
||||
|
||||
|
||||
_feature_state: ComputerUseFeatureState | None = None
|
||||
|
||||
|
||||
def get_computer_use_feature_state() -> ComputerUseFeatureState:
|
||||
"""Return the process-wide Computer Use feature switch."""
|
||||
global _feature_state
|
||||
if _feature_state is None:
|
||||
_feature_state = ComputerUseFeatureState()
|
||||
return _feature_state
|
||||
@@ -0,0 +1,53 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Retiring a Computer Use turn when the request that opened it finishes.
|
||||
|
||||
The host mints a fresh turn id for every request, in a request-setup hook. This
|
||||
is the other half of that: a hook in the FINALLY phase, which the runtime runs
|
||||
inside a `finally` block, so the turn is released whether the request
|
||||
succeeded,
|
||||
failed or was cancelled.
|
||||
|
||||
Without it nothing ever told the plugin a turn was over. A session that used
|
||||
the
|
||||
tool once went on claiming its turn -- holding the connection, and holding the
|
||||
helper's screenshots and accessibility handles for that turn -- until some
|
||||
later
|
||||
request happened to arrive with a different id.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from qwenpaw.hooks.base import LifecycleHook
|
||||
from qwenpaw.runtime.hooks import HookContext, HookResult
|
||||
from qwenpaw.runtime.phases import Phase
|
||||
|
||||
from .client import end_computer_use_turn
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ComputerUseTurnEndHook(LifecycleHook):
|
||||
"""Release the native turn a finished request was holding."""
|
||||
|
||||
phase = Phase.FINALLY
|
||||
name = "computer_use_turn_end"
|
||||
priority = 50
|
||||
|
||||
async def run(self, ctx: HookContext) -> HookResult:
|
||||
session_id = getattr(ctx, "session_id", "") or ""
|
||||
if not session_id:
|
||||
return HookResult()
|
||||
try:
|
||||
await end_computer_use_turn(session_id)
|
||||
except Exception: # noqa: BLE001 - cleanup must not fail a request
|
||||
logger.debug(
|
||||
"computer_use_turn_end: release failed session=%s",
|
||||
session_id,
|
||||
exc_info=True,
|
||||
)
|
||||
return HookResult()
|
||||
|
||||
|
||||
__all__ = ["ComputerUseTurnEndHook"]
|
||||
@@ -0,0 +1,120 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Protocol models shared by the Computer Use client and transports."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Mapping
|
||||
|
||||
# The plugin can be updated independently of the desktop host. Keep its
|
||||
# expected version local so mismatched installations fail at the handshake.
|
||||
PROTOCOL_VERSION = 2
|
||||
|
||||
|
||||
# Every method name this adapter may put on the wire. The helper matches on
|
||||
# these, so the two sides have to agree exactly -- a name only one of them
|
||||
# knows
|
||||
# fails as an unsupported operation at run time, with nothing to catch it
|
||||
# earlier. A contract test compares this set against the helper's dispatch.
|
||||
#
|
||||
# Tool-level action names are a separate vocabulary: several of them map onto
|
||||
# one
|
||||
# method here (a double click and a right click are both ``click``), and some
|
||||
# never reach the helper at all.
|
||||
NATIVE_METHODS = frozenset(
|
||||
{
|
||||
"click",
|
||||
"close_window",
|
||||
"drag",
|
||||
"end_turn",
|
||||
"hello",
|
||||
"invoke_element",
|
||||
"launch_app",
|
||||
"list_apps",
|
||||
"list_windows",
|
||||
"observe_window",
|
||||
"press_key",
|
||||
"scroll",
|
||||
"sequence",
|
||||
"set_value",
|
||||
"type_text",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class ComputerUseProtocolError(RuntimeError):
|
||||
"""A stable native protocol failure with an actionable error code."""
|
||||
|
||||
def __init__(self, code: str, message: str) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NativeRequest:
|
||||
"""One client-to-native request with trusted execution metadata."""
|
||||
|
||||
method: str
|
||||
params: Mapping[str, Any]
|
||||
session_id: str
|
||||
turn_id: str
|
||||
deadline_ms: int
|
||||
request_id: str = ""
|
||||
|
||||
def to_message(self) -> dict[str, Any]:
|
||||
"""Serialize the request at the framing-neutral protocol boundary.
|
||||
|
||||
Every request passes through here, so it is where the declared method
|
||||
vocabulary is enforced. Without that the constant would only be
|
||||
documentation, and a typo would travel to the helper to come back as an
|
||||
unsupported operation.
|
||||
"""
|
||||
if self.method not in NATIVE_METHODS:
|
||||
raise ComputerUseProtocolError(
|
||||
"invalid_request",
|
||||
f"{self.method!r} is not a Computer Use protocol method.",
|
||||
)
|
||||
return {
|
||||
"request_id": self.request_id or uuid.uuid4().hex,
|
||||
"method": self.method,
|
||||
"params": dict(self.params),
|
||||
"meta": {
|
||||
"session_id": self.session_id,
|
||||
"turn_id": self.turn_id,
|
||||
"deadline_ms": self.deadline_ms,
|
||||
},
|
||||
"protocol_version": PROTOCOL_VERSION,
|
||||
}
|
||||
|
||||
|
||||
def parse_response(message: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Normalize a native response or raise a stable protocol error."""
|
||||
if bool(message.get("ok")):
|
||||
result = message.get("result", {})
|
||||
return result if isinstance(result, dict) else {"value": result}
|
||||
error = message.get("error", {})
|
||||
if isinstance(error, Mapping):
|
||||
code = str(error.get("code") or "native_error")
|
||||
detail = str(error.get("message") or code)
|
||||
else:
|
||||
code = "native_error"
|
||||
detail = str(error or code)
|
||||
raise ComputerUseProtocolError(code, detail)
|
||||
|
||||
|
||||
def approval_reply(
|
||||
request_id: str,
|
||||
*,
|
||||
allowed: bool,
|
||||
source: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Build the one-time reply for a native App approval request."""
|
||||
return {
|
||||
"request_id": request_id,
|
||||
"result": {
|
||||
"decision": "allow" if allowed else "deny",
|
||||
"source": source,
|
||||
},
|
||||
"protocol_version": PROTOCOL_VERSION,
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Session-scoped controls for the Computer Use plugin page."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from qwenpaw.app.approvals import get_approval_service
|
||||
from qwenpaw.app.computer_use import HostRuntimeProvider
|
||||
from qwenpaw.security.tool_guard.approval import ApprovalDecision
|
||||
|
||||
from .access import (
|
||||
get_computer_use_access_store,
|
||||
)
|
||||
|
||||
from .client import (
|
||||
is_computer_use_active,
|
||||
known_computer_use_sessions,
|
||||
stop_all_computer_use_turns,
|
||||
stop_computer_use_session,
|
||||
)
|
||||
from .feature_state import get_computer_use_feature_state
|
||||
|
||||
|
||||
class SessionRequest(BaseModel):
|
||||
"""A page action scoped to exactly one active conversation."""
|
||||
|
||||
session_id: str = Field(min_length=1)
|
||||
|
||||
|
||||
class FeatureToggleRequest(BaseModel):
|
||||
"""A request to enable or disable the Computer Use feature."""
|
||||
|
||||
enabled: bool
|
||||
session_id: str | None = None
|
||||
|
||||
|
||||
class PersistentAccessRequest(BaseModel):
|
||||
"""A request to revoke one installation-scoped grant."""
|
||||
|
||||
canonical_app_id: str = Field(min_length=1)
|
||||
|
||||
|
||||
class PendingDecisionRequest(SessionRequest):
|
||||
"""A response to one pending native App approval."""
|
||||
|
||||
request_id: str = Field(min_length=1)
|
||||
decision: Literal["deny", "session", "always"]
|
||||
|
||||
|
||||
def _is_computer_use_pending(pending) -> bool:
|
||||
return pending.extra.get("source_type") == "computer_use_app_access"
|
||||
|
||||
|
||||
def _pending_app(pending) -> tuple[str, str] | None:
|
||||
details = pending.extra.get("computer_use_app")
|
||||
if not isinstance(details, dict):
|
||||
return None
|
||||
canonical_app_id = str(details.get("canonical_app_id") or "").strip()
|
||||
display_name = str(details.get("display_name") or canonical_app_id).strip()
|
||||
if not canonical_app_id or not display_name:
|
||||
return None
|
||||
return canonical_app_id, display_name
|
||||
|
||||
|
||||
async def _pending_for_session(session_id: str):
|
||||
pending = await get_approval_service().list_pending_by_session(session_id)
|
||||
return [item for item in pending if _is_computer_use_pending(item)]
|
||||
|
||||
|
||||
def build_router() -> APIRouter:
|
||||
"""Build routes that do not acquire or start the native runtime."""
|
||||
router = APIRouter()
|
||||
|
||||
@router.get("/status")
|
||||
def get_status() -> dict[str, bool | str]:
|
||||
capability = HostRuntimeProvider.get_capability()
|
||||
return {
|
||||
"runtime_available": HostRuntimeProvider.is_available(),
|
||||
"connection_active": capability is not None,
|
||||
"approval_scope": "session_and_persistent",
|
||||
"enabled": get_computer_use_feature_state().is_enabled(),
|
||||
}
|
||||
|
||||
@router.post("/feature")
|
||||
async def set_feature_enabled(
|
||||
request: FeatureToggleRequest,
|
||||
) -> dict[str, int | bool]:
|
||||
get_computer_use_feature_state().set_enabled(request.enabled)
|
||||
if request.enabled:
|
||||
return {"enabled": True}
|
||||
# Off is a kill switch for every agent, so every session waiting on an
|
||||
# approval has to be refused -- not just the one the caller named. A
|
||||
# session left waiting would otherwise be granted access later and act
|
||||
# on a desktop the user had already switched the feature off for.
|
||||
#
|
||||
# Denying comes before stopping, for the reason given on /session/stop.
|
||||
sessions = set(known_computer_use_sessions())
|
||||
if request.session_id:
|
||||
sessions.add(request.session_id)
|
||||
denied = 0
|
||||
for session_id in sorted(sessions):
|
||||
denied += await _deny_pending(session_id)
|
||||
stopped = await stop_all_computer_use_turns()
|
||||
return {"enabled": False, "stopped": stopped, "denied": denied}
|
||||
|
||||
@router.get("/access")
|
||||
def list_persistent_access() -> dict[str, list[dict[str, str]]]:
|
||||
access = get_computer_use_access_store().list_persistent()
|
||||
return {
|
||||
"access": [
|
||||
{
|
||||
"canonical_app_id": item.canonical_app_id,
|
||||
"display_name": item.display_name,
|
||||
}
|
||||
for item in access
|
||||
],
|
||||
}
|
||||
|
||||
@router.delete("/access")
|
||||
def revoke_persistent_access(
|
||||
request: PersistentAccessRequest,
|
||||
) -> dict[str, bool]:
|
||||
return {
|
||||
"revoked": get_computer_use_access_store().revoke_persistent(
|
||||
request.canonical_app_id,
|
||||
),
|
||||
}
|
||||
|
||||
@router.get("/session")
|
||||
async def get_session_state(
|
||||
session_id: str = Query(min_length=1),
|
||||
) -> dict[str, bool]:
|
||||
return {"automation_active": is_computer_use_active(session_id)}
|
||||
|
||||
@router.post("/session/stop")
|
||||
async def stop_automation(
|
||||
request: SessionRequest,
|
||||
) -> dict[str, int | bool]:
|
||||
# Deny first, then stop. A pending approval is a wait the helper is
|
||||
# sitting in, and denying it is what ends that wait -- so it has to
|
||||
# happen before anything that waits on the action itself, or the
|
||||
# release would be queued behind the thing it releases.
|
||||
denied = await _deny_pending(request.session_id)
|
||||
stopped = await stop_computer_use_session(request.session_id)
|
||||
return {"stopped": stopped, "denied": denied}
|
||||
|
||||
async def _deny_pending(session_id: str) -> int:
|
||||
"""Refuse every approval this session is waiting on."""
|
||||
service = get_approval_service()
|
||||
denied = 0
|
||||
for pending in await _pending_for_session(session_id):
|
||||
resolved = await service.resolve_request(
|
||||
pending.request_id,
|
||||
ApprovalDecision.DENIED,
|
||||
)
|
||||
denied += int(resolved is not None)
|
||||
return denied
|
||||
|
||||
@router.post("/session/pending/decision")
|
||||
async def decide_pending(
|
||||
request: PendingDecisionRequest,
|
||||
) -> dict[str, bool | str]:
|
||||
pending = await get_approval_service().get_request(request.request_id)
|
||||
if (
|
||||
pending is None
|
||||
or pending.session_id != request.session_id
|
||||
or not _is_computer_use_pending(pending)
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail="Pending approval not found.",
|
||||
)
|
||||
decision = (
|
||||
ApprovalDecision.DENIED
|
||||
if request.decision == "deny"
|
||||
else ApprovalDecision.APPROVED
|
||||
)
|
||||
app = _pending_app(pending)
|
||||
access_store = get_computer_use_access_store()
|
||||
if request.decision == "always":
|
||||
if app is None:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Pending application is invalid.",
|
||||
)
|
||||
access_store.record_persistent(*app)
|
||||
resolved = await get_approval_service().resolve_request(
|
||||
request.request_id,
|
||||
decision,
|
||||
)
|
||||
if (
|
||||
resolved is None
|
||||
and request.decision == "always"
|
||||
and app is not None
|
||||
):
|
||||
access_store.revoke_persistent(app[0])
|
||||
return {
|
||||
"resolved": resolved is not None,
|
||||
"decision": request.decision,
|
||||
}
|
||||
|
||||
return router
|
||||
@@ -0,0 +1,13 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Host capability transports for Computer Use."""
|
||||
|
||||
from .base import ComputerUseTransport, ReverseRequestHandler
|
||||
from .unix_socket import UnixSocketTransport
|
||||
from .windows_pipe import WindowsPipeTransport
|
||||
|
||||
__all__ = [
|
||||
"ComputerUseTransport",
|
||||
"ReverseRequestHandler",
|
||||
"UnixSocketTransport",
|
||||
"WindowsPipeTransport",
|
||||
]
|
||||
@@ -0,0 +1,36 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Framing-neutral controlled transport contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from typing import Any
|
||||
|
||||
ReverseRequestHandler = Callable[
|
||||
[Mapping[str, Any]],
|
||||
Awaitable[dict[str, Any]],
|
||||
]
|
||||
|
||||
|
||||
class ComputerUseTransport(ABC):
|
||||
"""Authenticated request/response transport with reverse policy events."""
|
||||
|
||||
@abstractmethod
|
||||
async def connect(self) -> None:
|
||||
"""Connect and authenticate the native endpoint."""
|
||||
|
||||
@abstractmethod
|
||||
async def request(self, message: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Send one request and await its matching response."""
|
||||
|
||||
@abstractmethod
|
||||
async def close(self) -> None:
|
||||
"""Close the transport and fail outstanding requests."""
|
||||
|
||||
@abstractmethod
|
||||
def set_reverse_request_handler(
|
||||
self,
|
||||
handler: ReverseRequestHandler,
|
||||
) -> None:
|
||||
"""Install the handler for native-initiated policy requests."""
|
||||
@@ -0,0 +1,334 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Unix domain socket transport for the host-managed Computer Use helper.
|
||||
|
||||
This mirrors :class:`WindowsPipeTransport` on macOS. The wire protocol,
|
||||
handshake, request correlation, and reverse app-approval behavior are
|
||||
identical; only the byte transport differs (an ``asyncio`` Unix domain
|
||||
socket instead of a Windows named pipe), so no background thread or
|
||||
overlapped I/O machinery is needed.
|
||||
"""
|
||||
|
||||
# The capability's endpoint name and secret are deliberately private so they
|
||||
# cannot leak into tool inputs; a transport is their only intended reader.
|
||||
# pylint: disable=protected-access
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextvars
|
||||
import json
|
||||
import socket
|
||||
import struct
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from qwenpaw.app.computer_use.runtime import RuntimeCapability
|
||||
|
||||
from ..protocol import ComputerUseProtocolError, approval_reply
|
||||
from .base import ComputerUseTransport, ReverseRequestHandler
|
||||
|
||||
_MAX_FRAME_BYTES = 64 * 1024 * 1024
|
||||
_CONNECT_TIMEOUT_SECONDS = 5
|
||||
_APPROVAL_POLL_SECONDS = 0.25
|
||||
|
||||
|
||||
class UnixSocketTransport(ComputerUseTransport):
|
||||
"""Length-prefixed JSON transport with native reverse-policy support."""
|
||||
|
||||
def __init__(self, capability: RuntimeCapability) -> None:
|
||||
self._capability = capability
|
||||
self._reader: asyncio.StreamReader | None = None
|
||||
self._writer: asyncio.StreamWriter | None = None
|
||||
self._reader_task: asyncio.Task[None] | None = None
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
self._pending: dict[str, asyncio.Future[dict[str, Any]]] = {}
|
||||
self._write_lock = asyncio.Lock()
|
||||
self._reverse_handler: ReverseRequestHandler | None = None
|
||||
self._reverse_context: contextvars.Context | None = None
|
||||
self._approvals_in_flight = 0
|
||||
self._closed = False
|
||||
|
||||
async def connect(self) -> None:
|
||||
"""Open the host-provided socket and perform the protocol handshake."""
|
||||
if self._writer is not None:
|
||||
return
|
||||
if not hasattr(socket, "AF_UNIX"):
|
||||
raise ComputerUseProtocolError(
|
||||
"runtime_unavailable",
|
||||
"Computer Use Unix socket transport requires a POSIX host.",
|
||||
)
|
||||
self._loop = asyncio.get_running_loop()
|
||||
try:
|
||||
self._reader, self._writer = await asyncio.wait_for(
|
||||
asyncio.open_unix_connection(self._capability._pipe_name),
|
||||
_CONNECT_TIMEOUT_SECONDS,
|
||||
)
|
||||
except (OSError, asyncio.TimeoutError) as exc:
|
||||
raise ComputerUseProtocolError(
|
||||
"runtime_unavailable",
|
||||
"Computer Use native runtime is not running.",
|
||||
) from exc
|
||||
self._reader_task = asyncio.ensure_future(self._reader_loop())
|
||||
hello = {
|
||||
"request_id": "hello",
|
||||
"method": "hello",
|
||||
"params": {
|
||||
"capability": self._capability._secret,
|
||||
"protocol_version": self._capability.protocol_version,
|
||||
},
|
||||
"meta": {"session_id": "", "turn_id": "", "deadline_ms": 5000},
|
||||
"protocol_version": self._capability.protocol_version,
|
||||
}
|
||||
response = await self.request(hello)
|
||||
result = response.get("result")
|
||||
if (
|
||||
not isinstance(result, Mapping)
|
||||
or int(result.get("protocol_version", 0))
|
||||
!= self._capability.protocol_version
|
||||
):
|
||||
await self.close()
|
||||
raise ComputerUseProtocolError(
|
||||
"protocol_mismatch",
|
||||
"Computer Use helper protocol is incompatible with this "
|
||||
"plugin.",
|
||||
)
|
||||
|
||||
async def request(self, message: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Send a request and await its native response by request id."""
|
||||
if self._writer is None or self._closed:
|
||||
raise ComputerUseProtocolError(
|
||||
"runtime_unavailable",
|
||||
"Computer Use native runtime is unavailable.",
|
||||
)
|
||||
request_id = str(message.get("request_id") or "")
|
||||
if not request_id:
|
||||
raise ComputerUseProtocolError(
|
||||
"invalid_request",
|
||||
"Computer Use request is missing its request identifier.",
|
||||
)
|
||||
future: asyncio.Future[
|
||||
dict[str, Any]
|
||||
] = asyncio.get_running_loop().create_future()
|
||||
if request_id in self._pending:
|
||||
raise ComputerUseProtocolError(
|
||||
"duplicate_request",
|
||||
"Computer Use request identifier is already in use.",
|
||||
)
|
||||
self._pending[request_id] = future
|
||||
# The helper only issues reverse approval requests while it is
|
||||
# blocked serving this request, so snapshot the caller's context
|
||||
# (session, agent, user, channel) for the approval coroutine.
|
||||
self._reverse_context = contextvars.copy_context()
|
||||
meta = message.get("meta")
|
||||
timeout_ms = (
|
||||
int(meta.get("deadline_ms", 10000))
|
||||
if isinstance(meta, Mapping)
|
||||
else 10000
|
||||
)
|
||||
timeout = max(0.1, timeout_ms / 1000)
|
||||
try:
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._write_message(dict(message)),
|
||||
timeout,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
# A failed or timed-out write may leave a partial frame on the
|
||||
# socket, so the connection can no longer be trusted.
|
||||
future.cancel()
|
||||
await self.close()
|
||||
raise
|
||||
except ComputerUseProtocolError:
|
||||
future.cancel()
|
||||
await self.close()
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - transport boundary
|
||||
future.cancel()
|
||||
await self.close()
|
||||
raise ComputerUseProtocolError(
|
||||
"runtime_disconnected",
|
||||
"Computer Use connection failed while sending a request.",
|
||||
) from exc
|
||||
return await self._await_response(future, timeout)
|
||||
except asyncio.TimeoutError as exc:
|
||||
raise ComputerUseProtocolError(
|
||||
"request_timeout",
|
||||
"Computer Use request timed out.",
|
||||
) from exc
|
||||
finally:
|
||||
self._pending.pop(request_id, None)
|
||||
|
||||
async def _await_response(
|
||||
self,
|
||||
future: asyncio.Future[dict[str, Any]],
|
||||
timeout: float,
|
||||
) -> dict[str, Any]:
|
||||
"""Await a response without charging user approval time as timeout.
|
||||
|
||||
The helper blocks serving a request while a reverse app approval
|
||||
waits for the user, so the machine deadline pauses until the
|
||||
approval resolves and the helper then gets a fresh execution window.
|
||||
"""
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
if self._approvals_in_flight > 0:
|
||||
deadline = time.monotonic() + timeout
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
future.cancel()
|
||||
raise asyncio.TimeoutError
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
asyncio.shield(future),
|
||||
min(remaining, _APPROVAL_POLL_SECONDS),
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Close the socket and reject every pending request."""
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
writer, self._writer = self._writer, None
|
||||
self._reader = None
|
||||
task, self._reader_task = self._reader_task, None
|
||||
if task is not None and task is not asyncio.current_task():
|
||||
task.cancel()
|
||||
if writer is not None:
|
||||
try:
|
||||
writer.close()
|
||||
await writer.wait_closed()
|
||||
except (
|
||||
Exception
|
||||
): # noqa: BLE001 - closing a broken socket may raise
|
||||
pass
|
||||
self._fail_pending(
|
||||
"runtime_disconnected",
|
||||
"Computer Use connection closed.",
|
||||
)
|
||||
|
||||
def set_reverse_request_handler(
|
||||
self,
|
||||
handler: ReverseRequestHandler,
|
||||
) -> None:
|
||||
self._reverse_handler = handler
|
||||
|
||||
async def _write_message(self, message: dict[str, Any]) -> None:
|
||||
writer = self._writer
|
||||
if writer is None:
|
||||
raise ComputerUseProtocolError(
|
||||
"runtime_disconnected",
|
||||
"Computer Use connection is closed.",
|
||||
)
|
||||
payload = json.dumps(message, separators=(",", ":")).encode("utf-8")
|
||||
if len(payload) > _MAX_FRAME_BYTES:
|
||||
raise ComputerUseProtocolError(
|
||||
"frame_too_large",
|
||||
"Computer Use request exceeds the transport limit.",
|
||||
)
|
||||
async with self._write_lock:
|
||||
writer.write(struct.pack("<I", len(payload)) + payload)
|
||||
await writer.drain()
|
||||
|
||||
async def _reader_loop(self) -> None:
|
||||
reader = self._reader
|
||||
if reader is None:
|
||||
return
|
||||
try:
|
||||
while not self._closed:
|
||||
message = await self._read_message(reader)
|
||||
if message.get("method") == "request_app_approval":
|
||||
self._schedule_reverse_request(message)
|
||||
else:
|
||||
self._resolve_response(message)
|
||||
except (asyncio.CancelledError, asyncio.IncompleteReadError):
|
||||
pass
|
||||
except Exception as exc: # noqa: BLE001 - transport boundary
|
||||
if not self._closed:
|
||||
self._fail_pending(
|
||||
"runtime_disconnected",
|
||||
f"Computer Use connection failed: {exc}",
|
||||
)
|
||||
|
||||
async def _read_message(
|
||||
self,
|
||||
reader: asyncio.StreamReader,
|
||||
) -> dict[str, Any]:
|
||||
header = await reader.readexactly(4)
|
||||
frame_size = struct.unpack("<I", header)[0]
|
||||
if not 0 < frame_size <= _MAX_FRAME_BYTES:
|
||||
raise ComputerUseProtocolError(
|
||||
"invalid_frame",
|
||||
"Invalid Computer Use frame size.",
|
||||
)
|
||||
payload = await reader.readexactly(frame_size)
|
||||
value = json.loads(payload.decode("utf-8"))
|
||||
if not isinstance(value, dict):
|
||||
raise ComputerUseProtocolError(
|
||||
"invalid_frame",
|
||||
"Invalid Computer Use message.",
|
||||
)
|
||||
return value
|
||||
|
||||
def _schedule_reverse_request(self, message: dict[str, Any]) -> None:
|
||||
loop = self._loop
|
||||
if loop is None:
|
||||
return
|
||||
# Pause pending request deadlines while the user decides.
|
||||
self._approvals_in_flight += 1
|
||||
context = self._reverse_context
|
||||
# Replay the request-time context so the approval coroutine keeps the
|
||||
# session/agent/user/channel it needs; the bare reader-task context
|
||||
# would drop those contextvars.
|
||||
if context is not None:
|
||||
loop.create_task(
|
||||
self._reply_to_reverse_request(message),
|
||||
context=context,
|
||||
)
|
||||
else:
|
||||
loop.create_task(self._reply_to_reverse_request(message))
|
||||
|
||||
async def _reply_to_reverse_request(self, message: dict[str, Any]) -> None:
|
||||
try:
|
||||
await self._handle_reverse_request(message)
|
||||
finally:
|
||||
self._approvals_in_flight -= 1
|
||||
|
||||
async def _handle_reverse_request(self, message: dict[str, Any]) -> None:
|
||||
request_id = str(message.get("request_id") or "")
|
||||
handler = self._reverse_handler
|
||||
decision = {"allowed": False, "source": "invalid"}
|
||||
if request_id and handler is not None:
|
||||
try:
|
||||
decision = await handler(message)
|
||||
except Exception: # noqa: BLE001 - fail closed at the socket edge
|
||||
decision = {"allowed": False, "source": "error"}
|
||||
reply = approval_reply(
|
||||
request_id,
|
||||
allowed=bool(decision.get("allowed")),
|
||||
source=str(decision.get("source") or "unknown"),
|
||||
)
|
||||
try:
|
||||
await self._write_message(reply)
|
||||
except Exception:
|
||||
self._fail_pending(
|
||||
"runtime_disconnected",
|
||||
"Computer Use approval reply could not reach the native "
|
||||
"runtime.",
|
||||
)
|
||||
|
||||
def _resolve_response(self, message: dict[str, Any]) -> None:
|
||||
request_id = str(message.get("request_id") or "")
|
||||
future = self._pending.get(request_id)
|
||||
if future is None or future.done():
|
||||
return
|
||||
future.set_result(message)
|
||||
|
||||
def _fail_pending(self, code: str, message: str) -> None:
|
||||
futures = list(self._pending.values())
|
||||
self._pending.clear()
|
||||
for future in futures:
|
||||
if not future.done():
|
||||
future.set_exception(ComputerUseProtocolError(code, message))
|
||||
@@ -0,0 +1,648 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Windows named-pipe transport for the host-managed Computer Use helper."""
|
||||
|
||||
# The capability's endpoint name and secret are deliberately private so they
|
||||
# cannot leak into tool inputs; a transport is their only intended reader.
|
||||
# pylint: disable=protected-access
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextvars
|
||||
import ctypes
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Callable, Mapping
|
||||
from ctypes import wintypes
|
||||
from typing import Any
|
||||
|
||||
from qwenpaw.app.computer_use.runtime import RuntimeCapability
|
||||
|
||||
from ..protocol import ComputerUseProtocolError, approval_reply
|
||||
from .base import ComputerUseTransport, ReverseRequestHandler
|
||||
|
||||
_GENERIC_READ = 0x80000000
|
||||
_GENERIC_WRITE = 0x40000000
|
||||
_OPEN_EXISTING = 3
|
||||
_FILE_FLAG_OVERLAPPED = 0x40000000
|
||||
_ERROR_PIPE_BUSY = 231
|
||||
_ERROR_FILE_NOT_FOUND = 2
|
||||
_ERROR_IO_PENDING = 997
|
||||
_WAIT_OBJECT_0 = 0x0
|
||||
_WAIT_FAILED = 0xFFFFFFFF
|
||||
_INFINITE = 0xFFFFFFFF
|
||||
_MAX_FRAME_BYTES = 64 * 1024 * 1024
|
||||
_CONNECT_TIMEOUT_SECONDS = 5
|
||||
_WRITE_TIMEOUT_MS = 10000
|
||||
_READ_POLL_MS = 500
|
||||
_APPROVAL_POLL_SECONDS = 0.25
|
||||
_CLOSE_JOIN_TIMEOUT_SECONDS = 2
|
||||
|
||||
|
||||
class _OVERLAPPED(ctypes.Structure):
|
||||
_fields_ = [
|
||||
("Internal", ctypes.c_void_p),
|
||||
("InternalHigh", ctypes.c_void_p),
|
||||
("Offset", wintypes.DWORD),
|
||||
("OffsetHigh", wintypes.DWORD),
|
||||
("hEvent", wintypes.HANDLE),
|
||||
]
|
||||
|
||||
|
||||
class WindowsPipeTransport(ComputerUseTransport):
|
||||
"""Length-prefixed JSON transport with native reverse-policy support."""
|
||||
|
||||
def __init__(self, capability: RuntimeCapability) -> None:
|
||||
self._capability = capability
|
||||
self._handle: int | None = None
|
||||
self._loop: asyncio.AbstractEventLoop | None = None
|
||||
self._reader: threading.Thread | None = None
|
||||
self._pending: dict[str, asyncio.Future[dict[str, Any]]] = {}
|
||||
self._pending_lock = threading.Lock()
|
||||
self._write_lock = threading.Lock()
|
||||
self._reverse_handler: ReverseRequestHandler | None = None
|
||||
self._reverse_context: contextvars.Context | None = None
|
||||
self._approvals_lock = threading.Lock()
|
||||
self._approvals_in_flight = 0
|
||||
self._closed = False
|
||||
|
||||
async def connect(self) -> None:
|
||||
"""Open the host-provided pipe and perform the protocol handshake."""
|
||||
if self._handle is not None:
|
||||
return
|
||||
if os.name != "nt":
|
||||
raise ComputerUseProtocolError(
|
||||
"runtime_unavailable",
|
||||
"Computer Use is only available on Windows.",
|
||||
)
|
||||
self._loop = asyncio.get_running_loop()
|
||||
self._handle = await asyncio.to_thread(_connect_pipe, self._pipe_path)
|
||||
self._reader = threading.Thread(
|
||||
target=self._reader_loop,
|
||||
name="computer-use-pipe-reader",
|
||||
daemon=True,
|
||||
)
|
||||
self._reader.start()
|
||||
hello = {
|
||||
"request_id": "hello",
|
||||
"method": "hello",
|
||||
"params": {
|
||||
"capability": self._capability._secret,
|
||||
"protocol_version": self._capability.protocol_version,
|
||||
},
|
||||
"meta": {"session_id": "", "turn_id": "", "deadline_ms": 5000},
|
||||
"protocol_version": self._capability.protocol_version,
|
||||
}
|
||||
response = await self.request(hello)
|
||||
result = response.get("result")
|
||||
if (
|
||||
not isinstance(result, Mapping)
|
||||
or int(result.get("protocol_version", 0))
|
||||
!= self._capability.protocol_version
|
||||
):
|
||||
await self.close()
|
||||
raise ComputerUseProtocolError(
|
||||
"protocol_mismatch",
|
||||
"Computer Use helper protocol is incompatible with this "
|
||||
"plugin.",
|
||||
)
|
||||
|
||||
async def request(self, message: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""Send a request and await its native response by request id."""
|
||||
if self._handle is None or self._closed:
|
||||
raise ComputerUseProtocolError(
|
||||
"runtime_unavailable",
|
||||
"Computer Use native runtime is unavailable.",
|
||||
)
|
||||
request_id = str(message.get("request_id") or "")
|
||||
if not request_id:
|
||||
raise ComputerUseProtocolError(
|
||||
"invalid_request",
|
||||
"Computer Use request is missing its request identifier.",
|
||||
)
|
||||
future: asyncio.Future[
|
||||
dict[str, Any]
|
||||
] = asyncio.get_running_loop().create_future()
|
||||
with self._pending_lock:
|
||||
if request_id in self._pending:
|
||||
raise ComputerUseProtocolError(
|
||||
"duplicate_request",
|
||||
"Computer Use request identifier is already in use.",
|
||||
)
|
||||
self._pending[request_id] = future
|
||||
# The helper only issues reverse approval requests while it is
|
||||
# blocked serving this request, so snapshot the caller's context
|
||||
# (session, agent, user, channel) for the approval coroutine.
|
||||
self._reverse_context = contextvars.copy_context()
|
||||
meta = message.get("meta")
|
||||
timeout_ms = (
|
||||
int(meta.get("deadline_ms", 10000))
|
||||
if isinstance(meta, Mapping)
|
||||
else 10000
|
||||
)
|
||||
timeout = max(0.1, timeout_ms / 1000)
|
||||
try:
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
asyncio.to_thread(self._write_message, dict(message)),
|
||||
timeout,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
# A failed or timed-out write may leave a partial frame on the
|
||||
# pipe, so the connection can no longer be trusted.
|
||||
future.cancel()
|
||||
await self.close()
|
||||
raise
|
||||
except ComputerUseProtocolError:
|
||||
future.cancel()
|
||||
await self.close()
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - transport boundary
|
||||
future.cancel()
|
||||
await self.close()
|
||||
raise ComputerUseProtocolError(
|
||||
"runtime_disconnected",
|
||||
"Computer Use connection failed while sending a request.",
|
||||
) from exc
|
||||
return await self._await_response(future, timeout)
|
||||
except TimeoutError as exc:
|
||||
raise ComputerUseProtocolError(
|
||||
"request_timeout",
|
||||
"Computer Use request timed out.",
|
||||
) from exc
|
||||
finally:
|
||||
with self._pending_lock:
|
||||
self._pending.pop(request_id, None)
|
||||
|
||||
async def _await_response(
|
||||
self,
|
||||
future: asyncio.Future[dict[str, Any]],
|
||||
timeout: float,
|
||||
) -> dict[str, Any]:
|
||||
"""Await a response without charging user approval time as timeout.
|
||||
|
||||
The helper blocks serving a request while a reverse app approval
|
||||
waits for the user, so the machine deadline pauses until the
|
||||
approval resolves (bounded by the approval service's own timeout)
|
||||
and the helper then gets a fresh execution window.
|
||||
"""
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
with self._approvals_lock:
|
||||
approval_in_flight = self._approvals_in_flight > 0
|
||||
if approval_in_flight:
|
||||
deadline = time.monotonic() + timeout
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
future.cancel()
|
||||
raise TimeoutError
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
asyncio.shield(future),
|
||||
min(remaining, _APPROVAL_POLL_SECONDS),
|
||||
)
|
||||
except TimeoutError:
|
||||
continue
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Close the named pipe and reject every pending request."""
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
handle, self._handle = self._handle, None
|
||||
reader, self._reader = self._reader, None
|
||||
if handle is not None:
|
||||
await asyncio.to_thread(_close_handle, handle, reader)
|
||||
self._fail_pending(
|
||||
"runtime_disconnected",
|
||||
"Computer Use connection closed.",
|
||||
)
|
||||
|
||||
def set_reverse_request_handler(
|
||||
self,
|
||||
handler: ReverseRequestHandler,
|
||||
) -> None:
|
||||
self._reverse_handler = handler
|
||||
|
||||
@property
|
||||
def _pipe_path(self) -> str:
|
||||
name = self._capability._pipe_name
|
||||
prefix = "\\\\.\\pipe\\"
|
||||
return name if name.startswith(prefix) else f"{prefix}{name}"
|
||||
|
||||
def _write_message(self, message: dict[str, Any]) -> None:
|
||||
handle = self._handle
|
||||
if handle is None:
|
||||
raise ComputerUseProtocolError(
|
||||
"runtime_disconnected",
|
||||
"Computer Use connection is closed.",
|
||||
)
|
||||
payload = json.dumps(message, separators=(",", ":")).encode("utf-8")
|
||||
if len(payload) > _MAX_FRAME_BYTES:
|
||||
raise ComputerUseProtocolError(
|
||||
"frame_too_large",
|
||||
"Computer Use request exceeds the transport limit.",
|
||||
)
|
||||
with self._write_lock:
|
||||
_write_all(handle, struct.pack("<I", len(payload)) + payload)
|
||||
|
||||
def _reader_loop(self) -> None:
|
||||
handle = self._handle
|
||||
if handle is None:
|
||||
return
|
||||
try:
|
||||
while not self._closed:
|
||||
message = _read_message(handle, lambda: self._closed)
|
||||
if message.get("method") == "request_app_approval":
|
||||
self._schedule_reverse_request(message)
|
||||
else:
|
||||
self._resolve_response(message)
|
||||
except Exception as exc: # noqa: BLE001 - transport boundary
|
||||
if not self._closed:
|
||||
self._fail_pending(
|
||||
"runtime_disconnected",
|
||||
f"Computer Use connection failed: {exc}",
|
||||
)
|
||||
|
||||
def _schedule_reverse_request(self, message: dict[str, Any]) -> None:
|
||||
loop = self._loop
|
||||
if loop is None:
|
||||
return
|
||||
# Pause pending request deadlines while the user decides.
|
||||
with self._approvals_lock:
|
||||
self._approvals_in_flight += 1
|
||||
context = self._reverse_context
|
||||
if context is None:
|
||||
asyncio.run_coroutine_threadsafe(
|
||||
self._reply_to_reverse_request(message),
|
||||
loop,
|
||||
)
|
||||
return
|
||||
# run_coroutine_threadsafe would copy this reader thread's bare
|
||||
# context, losing the session contextvars, so schedule through
|
||||
# call_soon_threadsafe with the snapshot captured in request().
|
||||
loop.call_soon_threadsafe(
|
||||
self._start_reverse_task,
|
||||
message,
|
||||
context=context,
|
||||
)
|
||||
|
||||
def _start_reverse_task(self, message: dict[str, Any]) -> None:
|
||||
asyncio.ensure_future(self._reply_to_reverse_request(message))
|
||||
|
||||
async def _reply_to_reverse_request(self, message: dict[str, Any]) -> None:
|
||||
try:
|
||||
await self._handle_reverse_request(message)
|
||||
finally:
|
||||
with self._approvals_lock:
|
||||
self._approvals_in_flight -= 1
|
||||
|
||||
async def _handle_reverse_request(self, message: dict[str, Any]) -> None:
|
||||
request_id = str(message.get("request_id") or "")
|
||||
handler = self._reverse_handler
|
||||
decision = {"allowed": False, "source": "invalid"}
|
||||
if request_id and handler is not None:
|
||||
try:
|
||||
decision = await handler(message)
|
||||
except Exception: # noqa: BLE001 - fail closed at the pipe edge
|
||||
decision = {"allowed": False, "source": "error"}
|
||||
reply = approval_reply(
|
||||
request_id,
|
||||
allowed=bool(decision.get("allowed")),
|
||||
source=str(decision.get("source") or "unknown"),
|
||||
)
|
||||
try:
|
||||
await asyncio.to_thread(self._write_message, reply)
|
||||
except Exception:
|
||||
self._fail_pending(
|
||||
"runtime_disconnected",
|
||||
"Computer Use approval reply could not reach the native "
|
||||
"runtime.",
|
||||
)
|
||||
|
||||
def _resolve_response(self, message: dict[str, Any]) -> None:
|
||||
request_id = str(message.get("request_id") or "")
|
||||
with self._pending_lock:
|
||||
future = self._pending.get(request_id)
|
||||
if future is None or future.done() or self._loop is None:
|
||||
return
|
||||
self._loop.call_soon_threadsafe(future.set_result, message)
|
||||
|
||||
def _fail_pending(self, code: str, message: str) -> None:
|
||||
with self._pending_lock:
|
||||
futures = list(self._pending.values())
|
||||
self._pending.clear()
|
||||
for future in futures:
|
||||
if future.done() or self._loop is None:
|
||||
continue
|
||||
self._loop.call_soon_threadsafe(
|
||||
future.set_exception,
|
||||
ComputerUseProtocolError(code, message),
|
||||
)
|
||||
|
||||
|
||||
def _connect_pipe(pipe_path: str) -> int:
|
||||
deadline = time.monotonic() + _CONNECT_TIMEOUT_SECONDS
|
||||
while True:
|
||||
handle = _kernel32().CreateFileW(
|
||||
pipe_path,
|
||||
_GENERIC_READ | _GENERIC_WRITE,
|
||||
0,
|
||||
None,
|
||||
_OPEN_EXISTING,
|
||||
_FILE_FLAG_OVERLAPPED,
|
||||
None,
|
||||
)
|
||||
if _valid_handle(handle):
|
||||
return int(handle)
|
||||
error = ctypes.get_last_error()
|
||||
remaining = deadline - time.monotonic()
|
||||
if (
|
||||
error in (_ERROR_PIPE_BUSY, _ERROR_FILE_NOT_FOUND)
|
||||
and remaining > 0
|
||||
):
|
||||
if error == _ERROR_PIPE_BUSY:
|
||||
_kernel32().WaitNamedPipeW(
|
||||
pipe_path,
|
||||
max(1, min(250, int(remaining * 1000))),
|
||||
)
|
||||
else:
|
||||
time.sleep(min(0.05, remaining))
|
||||
continue
|
||||
if error == _ERROR_FILE_NOT_FOUND:
|
||||
raise ComputerUseProtocolError(
|
||||
"runtime_unavailable",
|
||||
"Computer Use native runtime is not running.",
|
||||
)
|
||||
raise OSError(
|
||||
error,
|
||||
f"Could not connect to Computer Use pipe {pipe_path!r}",
|
||||
)
|
||||
|
||||
|
||||
def _read_message(
|
||||
handle: int,
|
||||
abort_check: Callable[[], bool] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
frame_size = struct.unpack("<I", _read_exact(handle, 4, abort_check))[0]
|
||||
if not 0 < frame_size <= _MAX_FRAME_BYTES:
|
||||
raise ComputerUseProtocolError(
|
||||
"invalid_frame",
|
||||
"Invalid Computer Use frame size.",
|
||||
)
|
||||
value = json.loads(
|
||||
_read_exact(handle, frame_size, abort_check).decode("utf-8"),
|
||||
)
|
||||
if not isinstance(value, dict):
|
||||
raise ComputerUseProtocolError(
|
||||
"invalid_frame",
|
||||
"Invalid Computer Use message.",
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def _read_exact(
|
||||
handle: int,
|
||||
size: int,
|
||||
abort_check: Callable[[], bool] | None = None,
|
||||
) -> bytes:
|
||||
chunks: list[bytes] = []
|
||||
remaining = size
|
||||
event = _new_event()
|
||||
overlapped = _OVERLAPPED()
|
||||
overlapped.hEvent = event
|
||||
try:
|
||||
while remaining:
|
||||
if abort_check is not None and abort_check():
|
||||
raise OSError("Computer Use connection closed")
|
||||
buffer = ctypes.create_string_buffer(remaining)
|
||||
read = _run_io(
|
||||
handle,
|
||||
overlapped,
|
||||
lambda: _kernel32().ReadFile(
|
||||
handle,
|
||||
buffer,
|
||||
remaining,
|
||||
None,
|
||||
ctypes.byref(overlapped),
|
||||
),
|
||||
None,
|
||||
"Computer Use pipe read failed",
|
||||
abort_check,
|
||||
)
|
||||
if not read:
|
||||
raise OSError("Computer Use pipe closed")
|
||||
chunks.append(buffer.raw[:read])
|
||||
remaining -= read
|
||||
finally:
|
||||
_kernel32().CloseHandle(event)
|
||||
return b"".join(chunks)
|
||||
|
||||
|
||||
def _write_all(handle: int, data: bytes) -> None:
|
||||
offset = 0
|
||||
event = _new_event()
|
||||
overlapped = _OVERLAPPED()
|
||||
overlapped.hEvent = event
|
||||
try:
|
||||
while offset < len(data):
|
||||
chunk = data[offset:]
|
||||
written = _run_io(
|
||||
handle,
|
||||
overlapped,
|
||||
lambda: _kernel32().WriteFile(
|
||||
handle,
|
||||
chunk,
|
||||
len(chunk),
|
||||
None,
|
||||
ctypes.byref(overlapped),
|
||||
),
|
||||
_WRITE_TIMEOUT_MS,
|
||||
"Computer Use pipe write failed",
|
||||
)
|
||||
if not written:
|
||||
raise OSError("Computer Use pipe closed")
|
||||
offset += written
|
||||
finally:
|
||||
_kernel32().CloseHandle(event)
|
||||
|
||||
|
||||
def _run_io(
|
||||
handle: int,
|
||||
overlapped: _OVERLAPPED,
|
||||
start_io: Callable[[], bool],
|
||||
timeout_ms: int | None,
|
||||
error_message: str,
|
||||
abort_check: Callable[[], bool] | None = None,
|
||||
) -> int:
|
||||
"""Drive one overlapped ReadFile/WriteFile to completion.
|
||||
|
||||
``start_io`` issues the overlapped operation. Returns the number of bytes
|
||||
transferred. ``timeout_ms`` bounds the total wait. ``abort_check`` (read
|
||||
path) is polled every ``_READ_POLL_MS`` without touching the pending I/O,
|
||||
so a racing ``close()`` cannot strand the reader even when its
|
||||
``CancelIoEx`` lands before the read was issued.
|
||||
"""
|
||||
kernel32 = _kernel32()
|
||||
kernel32.ResetEvent(overlapped.hEvent)
|
||||
if not start_io():
|
||||
error = ctypes.get_last_error()
|
||||
if error != _ERROR_IO_PENDING:
|
||||
raise OSError(error, error_message)
|
||||
deadline = (
|
||||
None
|
||||
if timeout_ms is None
|
||||
else time.monotonic() + timeout_ms / 1000
|
||||
)
|
||||
while True:
|
||||
if abort_check is not None:
|
||||
wait_ms = _READ_POLL_MS
|
||||
elif deadline is None:
|
||||
wait_ms = _INFINITE
|
||||
else:
|
||||
wait_ms = max(1, int((deadline - time.monotonic()) * 1000))
|
||||
wait = kernel32.WaitForSingleObject(overlapped.hEvent, wait_ms)
|
||||
if wait == _WAIT_OBJECT_0:
|
||||
break
|
||||
if wait == _WAIT_FAILED:
|
||||
raise OSError(ctypes.get_last_error(), error_message)
|
||||
# WAIT_TIMEOUT: poll the abort flag / deadline; the I/O
|
||||
# stays pending and is only cancelled in the branches below.
|
||||
if abort_check is not None and abort_check():
|
||||
_cancel_and_drain(kernel32, handle, overlapped)
|
||||
raise OSError("Computer Use connection closed")
|
||||
if deadline is not None and time.monotonic() >= deadline:
|
||||
_cancel_and_drain(kernel32, handle, overlapped)
|
||||
raise TimeoutError("Computer Use pipe I/O timed out")
|
||||
transferred = wintypes.DWORD(0)
|
||||
if not kernel32.GetOverlappedResult(
|
||||
handle,
|
||||
ctypes.byref(overlapped),
|
||||
ctypes.byref(transferred),
|
||||
False,
|
||||
):
|
||||
raise OSError(ctypes.get_last_error(), error_message)
|
||||
return transferred.value
|
||||
|
||||
|
||||
def _cancel_and_drain(
|
||||
kernel32: Any,
|
||||
handle: int,
|
||||
overlapped: _OVERLAPPED,
|
||||
) -> None:
|
||||
"""Cancel and drain the pending operation before buffers are freed."""
|
||||
kernel32.CancelIoEx(handle, ctypes.byref(overlapped))
|
||||
drained = wintypes.DWORD(0)
|
||||
kernel32.GetOverlappedResult(
|
||||
handle,
|
||||
ctypes.byref(overlapped),
|
||||
ctypes.byref(drained),
|
||||
True,
|
||||
)
|
||||
|
||||
|
||||
def _new_event() -> wintypes.HANDLE:
|
||||
event = _kernel32().CreateEventW(None, True, False, None)
|
||||
if not event:
|
||||
raise OSError(
|
||||
ctypes.get_last_error(),
|
||||
"Computer Use pipe event creation failed",
|
||||
)
|
||||
return event
|
||||
|
||||
|
||||
def _close_handle(handle: int, reader: threading.Thread | None = None) -> None:
|
||||
"""Cancel pending I/O, wait for the reader thread to unwind, then close."""
|
||||
_kernel32().CancelIoEx(handle, None)
|
||||
if (
|
||||
reader is not None
|
||||
and reader.is_alive()
|
||||
and reader is not threading.current_thread()
|
||||
):
|
||||
reader.join(timeout=_CLOSE_JOIN_TIMEOUT_SECONDS)
|
||||
if reader.is_alive():
|
||||
# The reader did not unwind, so it may still be inside a call on
|
||||
# this handle. Closing now would let the kernel reuse the handle
|
||||
# value while that call is in flight, pointing it at whatever
|
||||
# object lands on the number next. Leaking one handle for the
|
||||
# life of the process is the safer trade.
|
||||
return
|
||||
_kernel32().CloseHandle(handle)
|
||||
|
||||
|
||||
_kernel32_cache = None
|
||||
_kernel32_cache_lock = threading.Lock()
|
||||
|
||||
|
||||
def _kernel32():
|
||||
# The signatures are configured once and reused. Rebuilding the WinDLL
|
||||
# wrapper and re-declaring argtypes on every close/cancel/event call was
|
||||
# wasted work; the module itself was already loaded once by the OS.
|
||||
global _kernel32_cache
|
||||
if _kernel32_cache is None:
|
||||
with _kernel32_cache_lock:
|
||||
if _kernel32_cache is None:
|
||||
_kernel32_cache = _build_kernel32()
|
||||
return _kernel32_cache
|
||||
|
||||
|
||||
def _build_kernel32():
|
||||
kernel32 = ctypes.WinDLL("kernel32", use_last_error=True)
|
||||
kernel32.CreateFileW.argtypes = [
|
||||
wintypes.LPCWSTR,
|
||||
wintypes.DWORD,
|
||||
wintypes.DWORD,
|
||||
wintypes.LPVOID,
|
||||
wintypes.DWORD,
|
||||
wintypes.DWORD,
|
||||
wintypes.HANDLE,
|
||||
]
|
||||
kernel32.CreateFileW.restype = wintypes.HANDLE
|
||||
kernel32.WaitNamedPipeW.argtypes = [wintypes.LPCWSTR, wintypes.DWORD]
|
||||
kernel32.WaitNamedPipeW.restype = wintypes.BOOL
|
||||
kernel32.CreateEventW.argtypes = [
|
||||
wintypes.LPVOID,
|
||||
wintypes.BOOL,
|
||||
wintypes.BOOL,
|
||||
wintypes.LPCWSTR,
|
||||
]
|
||||
kernel32.CreateEventW.restype = wintypes.HANDLE
|
||||
kernel32.ResetEvent.argtypes = [wintypes.HANDLE]
|
||||
kernel32.ResetEvent.restype = wintypes.BOOL
|
||||
kernel32.WaitForSingleObject.argtypes = [wintypes.HANDLE, wintypes.DWORD]
|
||||
kernel32.WaitForSingleObject.restype = wintypes.DWORD
|
||||
kernel32.GetOverlappedResult.argtypes = [
|
||||
wintypes.HANDLE,
|
||||
wintypes.LPVOID,
|
||||
ctypes.POINTER(wintypes.DWORD),
|
||||
wintypes.BOOL,
|
||||
]
|
||||
kernel32.GetOverlappedResult.restype = wintypes.BOOL
|
||||
kernel32.CancelIoEx.argtypes = [wintypes.HANDLE, wintypes.LPVOID]
|
||||
kernel32.CancelIoEx.restype = wintypes.BOOL
|
||||
kernel32.ReadFile.argtypes = [
|
||||
wintypes.HANDLE,
|
||||
wintypes.LPVOID,
|
||||
wintypes.DWORD,
|
||||
ctypes.POINTER(wintypes.DWORD),
|
||||
wintypes.LPVOID,
|
||||
]
|
||||
kernel32.ReadFile.restype = wintypes.BOOL
|
||||
kernel32.WriteFile.argtypes = [
|
||||
wintypes.HANDLE,
|
||||
wintypes.LPCVOID,
|
||||
wintypes.DWORD,
|
||||
ctypes.POINTER(wintypes.DWORD),
|
||||
wintypes.LPVOID,
|
||||
]
|
||||
kernel32.WriteFile.restype = wintypes.BOOL
|
||||
kernel32.CloseHandle.argtypes = [wintypes.HANDLE]
|
||||
kernel32.CloseHandle.restype = wintypes.BOOL
|
||||
return kernel32
|
||||
|
||||
|
||||
def _valid_handle(handle: int | None) -> bool:
|
||||
return handle not in (None, 0, ctypes.c_void_p(-1).value)
|
||||
Reference in New Issue
Block a user