Gitinbox/OpenOPC
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: preserve company resume control and agent identity

Browse Source
This commit is contained in:
LZH-YS1998
2026-07-14 16:59:24 +08:00
parent b8202bbe9e
commit 297295a4aa
12 changed files with 2459 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
opc/engine.py
+636 -61
View File
@@ -175,6 +175,13 @@ from opc.layer3_agent.native_agent import NativeAgent
from opc.layer3_agent.prompt_harness.builder import _final_decider_role_id, _memory_skill_user_facing
from opc.layer3_agent.adapters.registry import AdapterRegistry
from opc.layer3_agent.external_broker import ExternalAgentBroker
from opc.layer3_agent.external_session_identity import (
external_session_matches_provider_token,
external_session_status_allows_resume,
is_provider_session_token,
provider_token_from_external_session,
select_best_external_resume_session,
)
from opc.layer4_tools.registry import ToolRegistry, ToolDefinition
from opc.layer4_tools.shell import create_shell_tool, create_shell_tools
from opc.layer4_tools.file_ops import create_file_tools
@@ -4661,6 +4668,70 @@ class OPCEngine:
async def _assign_task_execution_agent(self, task: Task, role: Any | None = None) -> str | None:
assert self.org_engine
task.metadata = dict(task.metadata)
resume_pin = dict(
task.metadata.get("_company_runtime_resume_execution_agent_pin", {})
or {}
)
if resume_pin:
available_for_audit: list[str] = []
selected_name = normalize_recruitment_agent_choice(
resume_pin.get("selected_execution_agent"),
default=(
str(resume_pin.get("assigned_external_agent", "") or "").strip()
or "native"
),
) or "native"
assigned_name = str(
resume_pin.get("assigned_external_agent", "") or ""
).strip()
if selected_name == "native":
if assigned_name:
raise RuntimeError(
f"company runtime resume agent pin is inconsistent for task {task.id}"
)
selected: str | None = None
else:
if assigned_name and assigned_name != selected_name:
raise RuntimeError(
f"company runtime resume agent pin is inconsistent for task {task.id}"
)
available_for_audit = self._available_external_agents()
if selected_name not in available_for_audit:
raise RuntimeError(
"company runtime resume requires unavailable external agent "
f"{selected_name!r} for task {task.id}"
)
selected = selected_name
task.assigned_external_agent = selected
task.metadata["selected_execution_agent"] = selected_name
task.metadata["preferred_external_agent"] = selected
task.metadata["agent_selection"] = {
"selected": selected_name,
"strategy": (
WorkItemExecutionStrategy.NATIVE.value
if selected_name == "native"
else WorkItemExecutionStrategy.EXTERNAL.value
),
"role_id": task.assigned_to
or task.metadata.get("work_item_role_id", ""),
"decision_reason": "company_runtime_resume_checkpoint_pin",
"selection_source": "company_runtime_resume_checkpoint",
"checkpoint_id": str(resume_pin.get("checkpoint_id", "") or ""),
"original_selection_source": str(
resume_pin.get("selected_execution_agent_source", "") or ""
),
"available_external_agents": (
available_for_audit
),
}
# The pin belongs to this dispatch attempt. Persisting the
# resulting choice is useful audit state, but leaving the pin set
# would silently turn an adaptive role into a permanent lock.
task.metadata.pop(
"_company_runtime_resume_execution_agent_pin",
None,
)
return selected
locked_agent = normalize_recruitment_agent_choice(
task.metadata.get("selected_execution_agent"),
default=("native" if not str(task.assigned_external_agent or "").strip() else str(task.assigned_external_agent or "").strip()),
@@ -4724,6 +4795,16 @@ class OPCEngine:
if not preferred_adapter:
return ordered
selection = dict(task.metadata.get("agent_selection", {}) or {})
if (
str(selection.get("selection_source", "") or "").strip()
== "company_runtime_resume_checkpoint"
):
# Resume owns one exact execution backend. Returning alternates
# here would undermine the checkpoint pin after the selector has
# consumed its one-shot marker.
return [(preferred, preferred_adapter)]
remaining = [(name, adapter) for name, adapter in ordered if name != preferred]
return [(preferred, preferred_adapter), *remaining]
@@ -5100,20 +5181,54 @@ class OPCEngine:
return [str(item) for item in progress[-limit:]]
@staticmethod
def _external_resume_status_allows_token(status: Any) -> bool:
normalized = str(status or "").strip().lower()
return normalized not in {
"failed",
"cancelled",
"denied",
"rejected",
"hard_timeout",
"idle_timeout",
"startup_timeout",
}
def _task_effective_execution_agent_identity(
task: Task,
) -> tuple[str, str, str]:
"""Return the backend that this Task attempt actually executes on.
Recruitment's ``selected_execution_agent`` is policy/default input and
can remain unchanged after an unlocked adaptive selection. Runtime
assignment and its audit record are the attempt identity; recruitment
metadata is only a fallback for tasks that have not recorded either.
"""
metadata = dict(task.metadata or {})
selection = dict(metadata.get("agent_selection", {}) or {})
selection_agent = normalize_recruitment_agent_choice(
selection.get("selected")
)
assigned_agent = normalize_recruitment_agent_choice(
task.assigned_external_agent
)
if bool(metadata.get("force_native_execution")) or selection_agent == "native":
selected_agent = "native"
assigned_external_agent = ""
elif assigned_agent and assigned_agent != "native":
selected_agent = assigned_agent
assigned_external_agent = assigned_agent
elif selection_agent and selection_agent != "native":
selected_agent = selection_agent
assigned_external_agent = selection_agent
else:
selected_agent = normalize_recruitment_agent_choice(
metadata.get("selected_execution_agent"),
default="native",
) or "native"
assigned_external_agent = (
selected_agent if selected_agent != "native" else ""
)
selection_source = str(
selection.get("selection_source")
or metadata.get("selected_execution_agent_source")
or ""
).strip()
return selected_agent, assigned_external_agent, selection_source
async def _external_resume_snapshot_for_task(self, task: Task) -> dict[str, Any]:
session = await self._load_latest_external_session_for_task(task)
session = (
await self._load_best_external_resume_session_for_task(task)
or await self._load_latest_external_session_for_task(task)
)
if not session:
return {}
metadata = dict(getattr(session, "metadata", {}) or {})
@@ -5162,8 +5277,6 @@ class OPCEngine:
company_profile = company_profile or str(metadata.get("company_profile", "") or "").strip()
role_session_id = str(metadata.get("delegation_role_session_id", "") or "").strip()
seat_state_id = str(metadata.get("delegation_seat_state_id", "") or "").strip()
if role_session_id:
role_runtime_session_ids.append(role_session_id)
if seat_state_id:
seat_state_ids.append(seat_state_id)
raw_runtime_resume = task.context_snapshot.get("runtime_resume", {}) if isinstance(task.context_snapshot, dict) else {}
@@ -5174,15 +5287,37 @@ class OPCEngine:
external_snapshot = await self._external_resume_snapshot_for_task(task)
if external_snapshot:
external_sessions_by_task[task.id] = external_snapshot
if role_session_id and callable(get_role_session):
try:
role_session = await get_role_session(role_session_id)
except Exception:
role_session = None
if role_session is not None:
adapter_session_state_by_role[role_session_id] = dict(
getattr(role_session, "adapter_session_state", {}) or {}
)
(
selected_execution_agent,
assigned_external_agent,
agent_selection_source,
) = self._task_effective_execution_agent_identity(task)
employee_assignment = dict(metadata.get("employee_assignment", {}) or {})
execution_identity = {
"role_id": str(
task.assigned_to
or metadata.get("work_item_role_id", "")
or ""
).strip(),
"seat_id": str(
metadata.get("delegation_seat_id", "")
or metadata.get("seat_id", "")
or ""
).strip(),
"role_runtime_session_id": role_session_id,
"employee_id": str(employee_assignment.get("employee_id", "") or "").strip(),
"employee_assignment": copy.deepcopy(employee_assignment),
"selected_execution_agent": selected_execution_agent,
"assigned_external_agent": assigned_external_agent,
"preferred_external_agent": str(
metadata.get("preferred_external_agent", "") or ""
).strip(),
"execution_agent_locked": bool(metadata.get("execution_agent_locked", False)),
"selected_execution_agent_source": str(
metadata.get("selected_execution_agent_source", "") or ""
).strip(),
"agent_selection_source": agent_selection_source,
}
work_item_id = linked_work_item_id_for_task(task)
work_item_snapshot: dict[str, Any] = {}
@@ -5208,8 +5343,46 @@ class OPCEngine:
"kind": str(getattr(work_item, "kind", "") or ""),
"metadata": dict(getattr(work_item, "metadata", {}) or {}),
}
execution_identity["seat_id"] = (
work_item_snapshot["seat_id"]
or execution_identity["seat_id"]
)
execution_identity["role_id"] = (
work_item_snapshot["role_id"]
or execution_identity["role_id"]
)
execution_identity["role_runtime_session_id"] = (
work_item_snapshot["role_runtime_session_id"]
or execution_identity["role_runtime_session_id"]
)
work_item_assignment = dict(
work_item_snapshot["metadata"].get("employee_assignment", {})
or {}
)
if "employee_assignment" in work_item_snapshot["metadata"]:
execution_identity["employee_id"] = str(
work_item_assignment.get("employee_id", "") or ""
).strip()
execution_identity["employee_assignment"] = copy.deepcopy(
work_item_assignment
)
active_work_items.append(work_item_snapshot)
role_session_id = str(
execution_identity["role_runtime_session_id"] or ""
).strip()
if role_session_id:
role_runtime_session_ids.append(role_session_id)
if role_session_id and callable(get_role_session):
try:
role_session = await get_role_session(role_session_id)
except Exception:
role_session = None
if role_session is not None:
adapter_session_state_by_role[role_session_id] = dict(
getattr(role_session, "adapter_session_state", {}) or {}
)
task_snapshots.append({
"task_id": task.id,
"session_id": task.session_id,
@@ -5217,7 +5390,9 @@ class OPCEngine:
"status": task.status.value if isinstance(task.status, TaskStatus) else str(task.status),
"title": task.title,
"assigned_to": task.assigned_to,
"assigned_external_agent": task.assigned_external_agent,
"assigned_external_agent": assigned_external_agent,
"selected_execution_agent": selected_execution_agent,
"execution_identity": execution_identity,
"work_item_id": work_item_id,
"projection_id": projection_id_for_task(task),
"turn_type": turn_type_for_task(task, fallback=""),
@@ -5797,25 +5972,6 @@ class OPCEngine:
"idempotent": idempotent,
}
@staticmethod
def _external_resume_token_is_provider_token(
token: str,
*,
task: Task,
agent_type: str,
) -> bool:
value = str(token or "").strip()
if not value:
return False
project_id = str(task.project_id or "").strip()
if agent_type and project_id and value.startswith(f"{agent_type}:{project_id}:"):
return False
if agent_type and value.startswith(f"{agent_type}:"):
parts = value.split(":")
if len(parts) >= 3:
return False
return True
@staticmethod
def _company_runtime_dependencies_satisfied(
work_item: DelegationWorkItem,
@@ -5894,6 +6050,232 @@ class OPCEngine:
return Phase.READY
return original_phase
@staticmethod
def _checkpoint_task_execution_identity(
task_snapshot: dict[str, Any],
) -> dict[str, Any]:
"""Return the immutable execution identity captured at suspension.
Checkpoints created before the explicit ``execution_identity`` field
already contain enough role/agent data to derive a safe identity. The
derivation is intentionally local to checkpoint consumption; it is not
a second runtime resolver.
"""
explicit = dict(task_snapshot.get("execution_identity", {}) or {})
work_item_snapshot = dict(task_snapshot.get("work_item", {}) or {})
work_item_metadata = dict(work_item_snapshot.get("metadata", {}) or {})
employee_assignment = dict(
explicit.get("employee_assignment", {})
or work_item_metadata.get("employee_assignment", {})
or {}
)
assigned_external_agent = str(
explicit.get("assigned_external_agent")
if "assigned_external_agent" in explicit
else task_snapshot.get("assigned_external_agent", "")
or ""
).strip()
selected_execution_agent = normalize_recruitment_agent_choice(
explicit.get("selected_execution_agent")
or task_snapshot.get("selected_execution_agent"),
default=assigned_external_agent or "native",
) or "native"
return {
"role_id": str(
explicit.get("role_id")
or work_item_snapshot.get("role_id")
or task_snapshot.get("assigned_to")
or ""
).strip(),
"seat_id": str(
explicit.get("seat_id")
or work_item_snapshot.get("seat_id")
or ""
).strip(),
"role_runtime_session_id": str(
explicit.get("role_runtime_session_id")
or work_item_snapshot.get("role_runtime_session_id")
or task_snapshot.get("role_session_id")
or ""
).strip(),
"employee_id": str(
explicit.get("employee_id")
or employee_assignment.get("employee_id")
or ""
).strip(),
"employee_assignment": copy.deepcopy(employee_assignment),
"selected_execution_agent": selected_execution_agent,
"assigned_external_agent": assigned_external_agent,
"preferred_external_agent": str(
explicit.get("preferred_external_agent", "") or ""
).strip(),
"execution_agent_locked": (
bool(explicit.get("execution_agent_locked", False))
if "execution_agent_locked" in explicit
else None
),
"selected_execution_agent_source": str(
explicit.get("agent_selection_source")
or explicit.get("selected_execution_agent_source", "")
or ""
).strip(),
"explicit": bool(explicit),
}
@classmethod
def _restore_and_pin_company_resume_execution_identity(
cls,
task: Task,
work_item: DelegationWorkItem | None,
task_snapshot: dict[str, Any],
role_session: DelegationRoleSession | None,
*,
checkpoint_id: str,
) -> None:
"""Validate durable role identity and pin this resumed attempt's agent.
A resume must continue the suspended actor, not re-run recruitment or
adaptive backend selection. Non-empty durable values that disagree
with the checkpoint fail closed. Missing Task projection fields are
restored from the checkpoint after the authoritative WorkItem has also
been checked.
"""
identity = cls._checkpoint_task_execution_identity(task_snapshot)
metadata = dict(task.metadata or {})
task_role_id = str(
task.assigned_to or metadata.get("work_item_role_id", "") or ""
).strip()
task_seat_id = str(
metadata.get("delegation_seat_id", "")
or metadata.get("seat_id", "")
or ""
).strip()
task_role_session_id = str(
metadata.get("delegation_role_session_id", "") or ""
).strip()
task_assignment = dict(metadata.get("employee_assignment", {}) or {})
task_employee_id = str(task_assignment.get("employee_id", "") or "").strip()
work_item_metadata = dict(getattr(work_item, "metadata", {}) or {})
work_item_assignment = dict(
work_item_metadata.get("employee_assignment", {}) or {}
)
if work_item is not None:
current_values: dict[str, list[str]] = {
"role_id": [str(getattr(work_item, "role_id", "") or "").strip()],
"seat_id": [str(getattr(work_item, "seat_id", "") or "").strip()],
"role_runtime_session_id": [str(
getattr(work_item, "role_runtime_session_id", "") or ""
).strip()],
"employee_id": [str(
work_item_assignment.get("employee_id", "") or ""
).strip()],
}
else:
current_values = {
"role_id": [task_role_id],
"seat_id": [task_seat_id],
"role_runtime_session_id": [task_role_session_id],
"employee_id": [task_employee_id],
}
for field_name, values in current_values.items():
expected = str(identity.get(field_name, "") or "").strip()
if not expected:
continue
for current in values:
if current and current != expected:
raise RuntimeError(
"company runtime resume identity mismatch for "
f"task {task.id}: {field_name}={current!r}, "
f"checkpoint={expected!r}"
)
expected_role_session_id = str(
identity.get("role_runtime_session_id", "") or ""
).strip()
if expected_role_session_id and identity.get("explicit") and role_session is None:
raise RuntimeError(
"company runtime resume identity mismatch for "
f"task {task.id}: role runtime session {expected_role_session_id!r} is missing"
)
if role_session is not None:
role_session_values = {
"role_id": str(getattr(role_session, "role_id", "") or "").strip(),
"seat_id": str(getattr(role_session, "seat_id", "") or "").strip(),
"employee_id": str(getattr(role_session, "employee_id", "") or "").strip(),
}
for field_name, current in role_session_values.items():
expected = str(identity.get(field_name, "") or "").strip()
if expected and current and current != expected:
raise RuntimeError(
"company runtime resume identity mismatch for "
f"task {task.id}: role session {field_name}={current!r}, "
f"checkpoint={expected!r}"
)
expected_agent = str(
identity.get("selected_execution_agent", "") or "native"
).strip()
expected_assigned_agent = str(
identity.get("assigned_external_agent", "") or ""
).strip()
if expected_agent == "native":
if expected_assigned_agent:
raise RuntimeError(
f"company runtime resume checkpoint has inconsistent native agent for task {task.id}"
)
elif expected_assigned_agent and expected_assigned_agent != expected_agent:
raise RuntimeError(
f"company runtime resume checkpoint has inconsistent external agent for task {task.id}"
)
else:
expected_assigned_agent = expected_agent
current_agent, current_assigned_agent, _current_source = (
cls._task_effective_execution_agent_identity(task)
)
if (
current_agent != expected_agent
or current_assigned_agent != expected_assigned_agent
):
raise RuntimeError(
"company runtime resume identity mismatch for "
f"task {task.id}: execution_agent={current_agent!r}, "
f"checkpoint={expected_agent!r}"
)
expected_source = str(
identity.get("selected_execution_agent_source", "") or ""
).strip()
metadata["work_item_role_id"] = identity["role_id"] or task_role_id
if identity["seat_id"]:
metadata["delegation_seat_id"] = identity["seat_id"]
if identity["role_runtime_session_id"]:
metadata["delegation_role_session_id"] = identity[
"role_runtime_session_id"
]
if identity.get("explicit") or identity["employee_assignment"]:
metadata["employee_assignment"] = copy.deepcopy(
identity["employee_assignment"]
)
metadata["selected_execution_agent"] = expected_agent
metadata["preferred_external_agent"] = (
str(identity.get("preferred_external_agent", "") or "").strip()
or (expected_assigned_agent if expected_agent != "native" else None)
)
metadata["_company_runtime_resume_execution_agent_pin"] = {
"checkpoint_id": str(checkpoint_id or "").strip(),
"selected_execution_agent": expected_agent,
"assigned_external_agent": expected_assigned_agent,
"selected_execution_agent_source": expected_source,
}
task.assigned_to = identity["role_id"] or task_role_id
task.assigned_external_agent = expected_assigned_agent or None
task.metadata = metadata
async def _prepare_company_runtime_tasks_for_resume(
self,
tasks: list[Task],
@@ -5903,7 +6285,6 @@ class OPCEngine:
) -> list[Task]:
assert self.store
adapter_state_by_role = dict(payload.get("adapter_session_state", {}) or {})
task_snapshot_by_id = {
str(item.get("task_id", "") or "").strip(): dict(item)
for item in list(payload.get("task_snapshots", []) or [])
@@ -5916,6 +6297,7 @@ class OPCEngine:
}
refreshed: list[Task] = []
get_work_item = getattr(self.store, "get_delegation_work_item", None)
get_role_session = getattr(self.store, "get_delegation_role_session", None)
list_work_items = getattr(self.store, "list_delegation_work_items", None)
update_role_session = getattr(self.store, "update_delegation_role_session", None)
update_work_item = getattr(self.store, "update_delegation_work_item", None)
@@ -5973,6 +6355,25 @@ class OPCEngine:
if task_is_terminal and not work_item_is_nonterminal:
refreshed.append(task)
continue
task_snapshot = task_snapshot_by_id.get(task.id, {})
if not task_snapshot:
raise RuntimeError(
f"company runtime resume checkpoint has no task identity snapshot for {task.id}"
)
identity = self._checkpoint_task_execution_identity(task_snapshot)
expected_role_session_id = str(
identity.get("role_runtime_session_id", "") or ""
).strip()
role_session = None
if expected_role_session_id and callable(get_role_session):
role_session = await get_role_session(expected_role_session_id)
self._restore_and_pin_company_resume_execution_identity(
task,
work_item,
task_snapshot,
role_session,
checkpoint_id=str(payload.get("checkpoint_id", "") or ""),
)
task.metadata = dict(task.metadata or {})
task.context_snapshot = dict(task.context_snapshot or {})
runtime_resume = dict(payload.get("native_runtime_resume", {}) or {}).get(task.id)
@@ -5981,13 +6382,18 @@ class OPCEngine:
task.metadata["runtime_v2"] = dict(runtime_resume)
external_sessions = dict(payload.get("external_sessions", {}) or {})
external_session = external_sessions.get(task.id)
task.metadata.pop("external_resume_checkpoint_session_updated_at", None)
task.metadata.pop("external_resume_checkpoint_session_status", None)
if isinstance(external_session, dict):
token_allowed = self._external_resume_status_allows_token(external_session.get("status"))
token_allowed = external_session_status_allows_resume(
external_session.get("status")
)
agent_type = str(
external_session.get("agent_type")
or task.assigned_external_agent
or ""
).strip()
assigned_agent_type = str(task.assigned_external_agent or "").strip()
token_candidates = [
str(external_session.get("resume_session_id") or "").strip(),
str(external_session.get("provider_session_id") or "").strip(),
@@ -5997,18 +6403,30 @@ class OPCEngine:
(
candidate
for candidate in token_candidates
if self._external_resume_token_is_provider_token(
if is_provider_session_token(
candidate,
task=task,
agent_type=agent_type,
project_id=str(task.project_id or "default"),
)
),
"",
)
if token and agent_type and token_allowed:
if (
token
and agent_type
and token_allowed
and agent_type == assigned_agent_type
):
task.metadata["external_resume_session_id"] = token
task.metadata["external_resume_agent_type"] = agent_type
task.metadata["external_resume_session_scope_id"] = task_session_scope_id(task)
task.metadata["external_resume_checkpoint_session_updated_at"] = str(
external_session.get("updated_at", "") or ""
).strip()
task.metadata["external_resume_checkpoint_session_status"] = str(
external_session.get("status", "") or ""
).strip()
task.metadata.pop("external_resume_fallback", None)
elif task.assigned_external_agent:
task.metadata.pop("external_resume_session_id", None)
task.metadata.pop("external_resume_agent_type", None)
@@ -6017,7 +6435,6 @@ class OPCEngine:
task.execution_lock = False
task.execution_locked_at = None
task.result = None
task_snapshot = task_snapshot_by_id.get(task.id, {})
work_item_snapshot = work_item_snapshot_by_id.get(work_item_id, {})
task_work_item_snapshot = task_snapshot.get("work_item", {})
phase_value = str(work_item_snapshot.get("phase", "") or "").strip()
@@ -6121,13 +6538,11 @@ class OPCEngine:
role_session_id = str(task.metadata.get("delegation_role_session_id", "") or "").strip()
if role_session_id and callable(update_role_session):
adapter_state = adapter_state_by_role.get(role_session_id)
try:
await update_role_session(
role_session_id,
focused_work_item_id="",
status="idle",
adapter_session_state=dict(adapter_state) if isinstance(adapter_state, dict) else None,
metadata_updates={
"last_resume_checkpoint_type": str(payload.get("checkpoint_type", "") or ""),
"last_resume_requested_at": datetime.now().isoformat(),
@@ -6260,6 +6675,108 @@ class OPCEngine:
return None
return await fallback(agent_type, project_id, task_id=task.id)
async def _load_best_external_resume_session_for_task(
self,
task: Task,
) -> Any | None:
"""Prefer a real provider thread over monitoring placeholder rows.
A live run initially owns a synthetic ``agent:project:task`` row. The
provider may publish its real thread id milliseconds later with the
same (or an older) timestamp. Recency alone is therefore not a valid
resume-token selector.
"""
if not self.store or not task.id:
return None
agent_type = str(task.assigned_external_agent or "").strip()
if not agent_type:
return None
list_sessions = getattr(self.store, "list_external_sessions", None)
if not callable(list_sessions):
return None
try:
sessions = await list_sessions(
project_id=task.project_id or self.project_id or "default",
task_id=task.id,
limit=100,
)
except TypeError:
return None
except Exception:
logger.opt(exception=True).debug(
"failed to list external resume-session candidates"
)
return None
selected, _token = select_best_external_resume_session(
sessions,
agent_type=agent_type,
project_id=str(task.project_id or self.project_id or "default"),
)
return selected
async def _checkpoint_external_resume_token_was_terminalized(
self,
task: Task,
*,
agent_type: str,
token: str,
checkpoint_updated_at: str,
checkpoint_status: str,
) -> bool:
"""Let a newer durable terminal row veto a checkpoint's working token."""
if not self.store or not task.id or not checkpoint_updated_at:
return False
list_sessions = getattr(self.store, "list_external_sessions", None)
if not callable(list_sessions):
return False
try:
sessions = await list_sessions(
project_id=task.project_id or self.project_id or "default",
task_id=task.id,
limit=100,
)
except Exception:
logger.opt(exception=True).debug(
"failed to verify checkpoint external resume token status"
)
return False
matching = [
session
for session in sessions
if str(getattr(session, "agent_type", "") or "").strip() == agent_type
and external_session_matches_provider_token(session, token)
]
if not matching:
return False
def _timestamp(value: Any) -> float:
candidate = value
if isinstance(candidate, str):
try:
candidate = datetime.fromisoformat(candidate)
except ValueError:
return 0.0
try:
return float(candidate.timestamp())
except Exception:
return 0.0
latest = max(
matching,
key=lambda session: _timestamp(getattr(session, "updated_at", None)),
)
latest_status = str(getattr(latest, "status", "") or "").strip().lower()
if external_session_status_allows_resume(latest_status):
return False
latest_timestamp = _timestamp(getattr(latest, "updated_at", None))
checkpoint_timestamp = _timestamp(checkpoint_updated_at)
return latest_timestamp > checkpoint_timestamp or (
latest_timestamp >= checkpoint_timestamp
and latest_status != str(checkpoint_status or "").strip().lower()
)
@staticmethod
def _clone_external_adapter(adapter: Any) -> Any:
config = getattr(adapter, "config", None)
@@ -6340,25 +6857,58 @@ class OPCEngine:
or metadata_agent_type != run_adapter.agent_type
)
)
synthetic_prefix = f"{run_adapter.agent_type}:{task.project_id}:"
project_id = str(task.project_id or self.project_id or "default").strip() or "default"
session_token = (
metadata_session_token
if metadata_session_token
and metadata_agent_type == run_adapter.agent_type
and not metadata_session_token.startswith(synthetic_prefix)
and is_provider_session_token(
metadata_session_token,
agent_type=run_adapter.agent_type,
project_id=project_id,
)
else ""
)
latest_session = await self._load_latest_external_session_for_task(task)
if session_token and await self._checkpoint_external_resume_token_was_terminalized(
task,
agent_type=run_adapter.agent_type,
token=session_token,
checkpoint_updated_at=str(
task.metadata.get("external_resume_checkpoint_session_updated_at", "")
or ""
).strip(),
checkpoint_status=str(
task.metadata.get("external_resume_checkpoint_session_status", "")
or ""
).strip(),
):
task.metadata = dict(task.metadata or {})
task.metadata.pop("external_resume_session_id", None)
task.metadata.pop("external_resume_agent_type", None)
task.metadata.pop("external_resume_session_scope_id", None)
task.metadata["external_resume_fallback"] = "context_replay_provider_terminal"
cloned_config = (
run_adapter.config.model_copy(deep=True)
if hasattr(run_adapter.config, "model_copy")
else run_adapter.config
)
if hasattr(cloned_config, "session_mode"):
cloned_config.session_mode = "new"
if hasattr(cloned_config, "session_id"):
cloned_config.session_id = ""
return run_adapter.__class__(config=cloned_config), resume_metadata
latest_session = (
await self._load_best_external_resume_session_for_task(task)
or await self._load_latest_external_session_for_task(task)
)
if latest_session and str(getattr(latest_session, "agent_type", "") or "").strip() != run_adapter.agent_type:
latest_session = None
if not session_token:
metadata = dict(getattr(latest_session, "metadata", {}) or {}) if latest_session else {}
session_token = str(metadata.get("resume_session_id") or metadata.get("provider_session_id") or "").strip()
if not session_token:
persisted_session_id = str(getattr(latest_session, "session_id", "") or "").strip() if latest_session else ""
synthetic_prefix = f"{run_adapter.agent_type}:{task.project_id}:"
if persisted_session_id and not persisted_session_id.startswith(synthetic_prefix):
session_token = persisted_session_id
session_token = provider_token_from_external_session(
latest_session,
agent_type=run_adapter.agent_type,
project_id=project_id,
)
if not session_token and not latest_session and metadata_token_is_unusable:
return run_adapter, resume_metadata
if not session_token:
@@ -8379,6 +8929,18 @@ class OPCEngine:
bool(task.metadata.get("execution_agent_locked"))
and str(task.metadata.get("selected_execution_agent", "") or "").strip() == agent_name
)
or (
str(
dict(task.metadata.get("agent_selection", {}) or {}).get(
"selection_source",
"",
)
or ""
).strip()
== "company_runtime_resume_checkpoint"
and str(task.assigned_external_agent or "").strip()
== agent_name
)
)
metadata = {
**metadata,
@@ -10818,6 +11380,19 @@ class OPCEngine:
tasks,
checkpoint_session_id=parent_session_id,
)
notify_kanban_changed = getattr(
self.company_executor,
"_notify_kanban_changed",
None,
)
if callable(notify_kanban_changed):
# The registry attempt above belongs to this successful
# checkpoint handoff, and resume preparation has now
# cleared its durable holds. Publish through the existing
# canonical snapshot path so UI control state becomes
# running/stoppable without introducing a second
# liveness signal in the WS layer.
await notify_kanban_changed()
except asyncio.CancelledError as exc:
if driver_ownership is not None:
driver_ownership.release()
opc/layer2_organization/company_mode.py
+6 -3
View File
@@ -5356,10 +5356,13 @@ class CompanyWorkItemExecutor:
target_status_or_phase=Phase.RUNNING,
reason="pre_execution_claim",
)
if task.metadata.get("force_native_execution"):
task.assigned_external_agent = None
elif self.agent_selector:
if self.agent_selector:
# The selector also owns checkpoint attempt pins. Forced-native
# work must pass through it so a resumed native pin is validated
# and consumed instead of leaking into a later dispatch.
await self.agent_selector(task, role)
elif task.metadata.get("force_native_execution"):
task.assigned_external_agent = None
else:
if not task.assigned_external_agent:
strategy = task.metadata.get("work_item_execution_strategy", "auto")
opc/layer3_agent/external_broker.py
+363 -19
View File
@@ -32,6 +32,13 @@ from opc.layer2_organization.work_item_runtime import is_work_item_runtime_metad
from opc.layer2_organization.work_item_identity import projection_id_for_task, turn_type_for_task
from opc.layer2_organization.work_item_links import linked_work_item_id_for_task, set_linked_work_item_id
from opc.layer3_agent.adapters.base import ExternalAgentAdapter
from opc.layer3_agent.external_session_identity import (
external_session_allows_resume,
external_session_matches_provider_token,
is_provider_session_token,
provider_token_from_external_session,
select_best_external_resume_session,
)
from opc.layer3_agent.preflight import (
assert_external_agent_write_contract,
ExternalAgentPreflightError,
@@ -51,15 +58,6 @@ from opc.layer4_tools.collaboration_rpc import (
)
def _external_session_allows_resume(session: ExternalSession | None) -> bool:
if session is None:
return False
status = str(getattr(session, "status", "") or "").strip().lower()
if status in {"failed", "cancelled", "denied", "rejected", "hard_timeout", "idle_timeout", "startup_timeout"}:
return False
return True
def _collaboration_role_cfg(org_engine: Any | None, role_id: str) -> Any | None:
if org_engine is None or not role_id:
return None
@@ -104,6 +102,78 @@ class ExternalAgentBroker:
def _normalize_external_agent_choice(value: Any) -> str:
return re.sub(r"[\s\-]+", "_", str(value or "").strip()).strip("_").lower()
async def _best_resume_external_session(
self,
*,
adapter: ExternalAgentAdapter,
task: Task,
role_session_id: str,
) -> ExternalSession | None:
list_sessions = getattr(self.store, "list_external_sessions", None)
if not callable(list_sessions):
return None
project_id = str(task.project_id or "default").strip() or "default"
kwargs: dict[str, Any] = {
"project_id": project_id,
"limit": 100,
}
if role_session_id:
kwargs["opc_session_id"] = role_session_id
else:
kwargs["task_id"] = task.id
try:
sessions = await list_sessions(**kwargs)
except TypeError:
return None
except Exception:
logger.opt(exception=True).debug(
"External resume restore: candidate listing failed"
)
return None
selected, _token = select_best_external_resume_session(
sessions,
agent_type=adapter.agent_type,
project_id=project_id,
)
return selected
async def _provider_stream_token_allows_resume(
self,
*,
adapter: ExternalAgentAdapter,
task: Task,
role_session_id: str,
token: str,
) -> bool:
"""Cross-check an early stream token against its latest run status."""
list_sessions = getattr(self.store, "list_external_sessions", None)
if not callable(list_sessions):
return False
project_id = str(task.project_id or "default").strip() or "default"
kwargs: dict[str, Any] = {"project_id": project_id, "limit": 100}
if role_session_id:
kwargs["opc_session_id"] = role_session_id
else:
kwargs["task_id"] = task.id
try:
sessions = await list_sessions(**kwargs)
except Exception:
logger.opt(exception=True).debug(
"External resume restore: provider-stream status lookup failed"
)
return False
for session in sessions:
if str(getattr(session, "agent_type", "") or "").strip() != adapter.agent_type:
continue
if external_session_matches_provider_token(session, token):
return (
external_session_allows_resume(session)
and str(getattr(session, "status", "") or "").strip().lower()
in {"done", "suspended"}
)
return False
@classmethod
def _task_explicitly_selected_external_agent(cls, task: Task, agent_type: str) -> bool:
selected_agent = cls._normalize_external_agent_choice(agent_type)
@@ -402,6 +472,8 @@ class ExternalAgentBroker:
config = getattr(adapter, "config", None)
if config is None:
return
if str(getattr(config, "session_mode", "") or "").strip().lower() == "new":
return
# Respect already-configured resume state.
if str(getattr(config, "session_mode", "") or "").strip().lower() == "resume" and getattr(config, "session_id", ""):
return
@@ -440,10 +512,53 @@ class ExternalAgentBroker:
or entry.get("provider_session_id")
or ""
).strip()
if not is_provider_session_token(
session_token,
agent_type=adapter.agent_type,
project_id=project_id,
):
session_token = ""
if (
session_token
and str(entry.get("source", "") or "").strip()
== "provider_stream"
and not await self._provider_stream_token_allows_resume(
adapter=adapter,
task=task,
role_session_id=role_session_id,
token=session_token,
)
):
session_token = ""
clear_role_state = getattr(
store,
"update_role_session_adapter_state",
None,
)
if callable(clear_role_state):
try:
await clear_role_state(
role_session_id,
adapter.agent_type,
None,
)
except Exception:
logger.opt(exception=True).debug(
"External resume restore: stale provider-stream token clear failed"
)
# Do not immediately rediscover the same unfinalized or
# failed stream row through the compatibility fallback.
return
prior = None
if not session_token:
prior = await self._best_resume_external_session(
adapter=adapter,
task=task,
role_session_id=role_session_id,
)
if role_session_id:
if prior is None:
try:
prior = await store.get_external_session(
adapter.agent_type,
@@ -469,7 +584,7 @@ class ExternalAgentBroker:
return
if prior is None:
return
if not _external_session_allows_resume(prior):
if not external_session_allows_resume(prior):
if on_progress:
await on_progress(
f"[External resume] {adapter.agent_type} skipped prior "
@@ -477,12 +592,11 @@ class ExternalAgentBroker:
)
return
session_token = str(
(prior.metadata or {}).get("resume_session_id")
or (prior.metadata or {}).get("provider_session_id")
or prior.session_id
or ""
).strip()
session_token = provider_token_from_external_session(
prior,
agent_type=adapter.agent_type,
project_id=project_id,
)
can_resume_without_session_id = bool(
adapter.can_resume_without_session_id()
if hasattr(adapter, "can_resume_without_session_id")
@@ -507,6 +621,86 @@ class ExternalAgentBroker:
f"[External resume] {adapter.agent_type} restored prior session → {label}"
)
async def _persist_discovered_provider_session(
self,
*,
adapter: ExternalAgentAdapter,
task: Task,
workspace_path: str,
runtime_session_id: str,
metadata: dict[str, Any],
provider_session_id: str,
status: str,
extra: dict[str, Any],
) -> bool:
"""Persist a provider thread as soon as it appears on the stream.
Waiting for process exit loses the token when Stop cancels the broker.
The external-session row is written first, then the canonical per-role
adapter state, so a concurrent suspend checkpoint can capture either
durable source.
"""
project_id = str(task.project_id or "default").strip() or "default"
token = str(provider_session_id or "").strip()
if not is_provider_session_token(
token,
agent_type=adapter.agent_type,
project_id=project_id,
):
return False
metadata["resume_session_id"] = token
metadata["provider_session_id"] = token
task.metadata = dict(task.metadata or {})
task.metadata["external_resume_session_id"] = token
task.metadata["external_resume_agent_type"] = adapter.agent_type
task.metadata["external_resume_session_scope_id"] = task_session_scope_id(task)
discovered_at = datetime.now().isoformat()
await self._save_runtime_session(
adapter=adapter,
task=task,
workspace_path=workspace_path,
session_id=runtime_session_id,
status=status,
metadata=metadata,
extra={
**dict(extra or {}),
"resume_session_id": token,
"provider_session_id": token,
"provider_session_discovered_at": discovered_at,
},
)
role_session_id = str(
task.metadata.get("delegation_role_session_id", "") or ""
).strip()
update_role_state = getattr(
self.store,
"update_role_session_adapter_state",
None,
)
if role_session_id and callable(update_role_state):
try:
await update_role_state(
role_session_id,
adapter.agent_type,
{
"resume_session_id": token,
"provider_session_id": token,
"agent_type": adapter.agent_type,
"updated_at": discovered_at,
"last_task_id": str(task.id or ""),
"last_project_id": project_id,
"workspace_path": workspace_path,
"source": "provider_stream",
"status": "working",
},
)
except Exception:
logger.opt(exception=True).debug(
"Provider stream token role-state write failed"
)
return True
async def _run_interactive(
self,
adapter: ExternalAgentAdapter,
@@ -741,7 +935,9 @@ class ExternalAgentBroker:
"timeout_reason": "",
"fatal_reason": "",
"process_cleanup": {},
"provider_session_id": "",
}
provider_session_lock = asyncio.Lock()
stream_line_counts: dict[str, int] = {}
trace_path = self._external_trace_path(adapter, task, started_at)
@@ -782,6 +978,37 @@ class ExternalAgentBroker:
stream_name=stream_name,
text=text,
)
if text.strip():
try:
discovered_provider_session_id = str(
adapter.extract_resume_session_id(text) or ""
).strip()
except Exception:
discovered_provider_session_id = ""
if discovered_provider_session_id:
async with provider_session_lock:
if not state["provider_session_id"]:
persisted = await self._persist_discovered_provider_session(
adapter=adapter,
task=task,
workspace_path=workspace_path,
runtime_session_id=session_id,
metadata=metadata,
provider_session_id=discovered_provider_session_id,
status="working",
extra={
"pid": proc.pid,
"started_at": started_at.isoformat(),
"last_activity_at": datetime.now().isoformat(),
"activity_count": state["activity_count"] + 1,
"last_output": text.strip(),
"stream": stream_name,
},
)
if persisted:
state["provider_session_id"] = (
discovered_provider_session_id
)
try:
fatal_reason = adapter.detect_runtime_failure(text, stream_name, metadata)
except TypeError:
@@ -1024,6 +1251,7 @@ class ExternalAgentBroker:
heartbeat_task = asyncio.create_task(_heartbeat())
idle_task = asyncio.create_task(_watch_idle())
inbox_task = asyncio.create_task(_poll_inbox())
cancellation_status_persisted = False
try:
try:
@@ -1113,11 +1341,9 @@ class ExternalAgentBroker:
"return_code": proc.returncode,
},
)
cancellation_status_persisted = True
raise
finally:
if proc.returncode is None:
state["process_cleanup"] = await self._terminate_process(proc)
async def _cancel_and_await(task_obj: asyncio.Task[Any]) -> None:
task_obj.cancel()
with contextlib.suppress(asyncio.CancelledError):
@@ -1137,6 +1363,9 @@ class ExternalAgentBroker:
return
raise
async def _finish_process_cleanup() -> None:
if proc.returncode is None:
state["process_cleanup"] = await self._terminate_process(proc)
for task_obj in (heartbeat_task, idle_task, inbox_task):
await _cancel_and_await(task_obj)
for task_obj in (stdout_task, stderr_task):
@@ -1147,6 +1376,49 @@ class ExternalAgentBroker:
if collab_rpc_server is not None:
await collab_rpc_server.close()
adapter._process = None # noqa: SLF001
cleanup_task = asyncio.create_task(_finish_process_cleanup())
try:
await asyncio.shield(cleanup_task)
except asyncio.CancelledError:
# Cancellation can arrive after proc.wait() completed but
# while stdout/adapter cleanup is still draining. Finish the
# cleanup in its own task and make the provider row terminal
# before propagating cancellation; otherwise a stale
# checkpoint can retain a false `working` capability.
with contextlib.suppress(Exception, asyncio.CancelledError):
await cleanup_task
if not cancellation_status_persisted:
cleanup_terminal_status = (
"done" if proc.returncode == 0 else "failed"
)
terminal_save = asyncio.create_task(self._save_runtime_session(
adapter=adapter,
task=task,
workspace_path=workspace_path,
session_id=session_id,
status=cleanup_terminal_status,
metadata=metadata,
extra={
"pid": proc.pid,
"started_at": started_at.isoformat(),
"last_activity_at": state["last_activity_at"].isoformat(),
"activity_count": state["activity_count"],
"last_output": state["last_output"],
"return_code": proc.returncode,
"failure_reason": (
""
if cleanup_terminal_status == "done"
else f"{adapter.agent_type} exited with code {proc.returncode}"
),
},
))
try:
await asyncio.shield(terminal_save)
except asyncio.CancelledError:
with contextlib.suppress(Exception, asyncio.CancelledError):
await terminal_save
raise
output = "".join(stdout_chunks)
errors = "".join(stderr_chunks)
normalized_output = adapter.normalize_result_output(output)
@@ -1197,6 +1469,43 @@ class ExternalAgentBroker:
raw_output=output,
base_artifacts=artifacts,
)
terminal_status = (
"done"
if not state["timed_out"]
and not state["fatal_reason"]
and return_code == 0
else "failed"
)
terminal_save = asyncio.create_task(self._save_runtime_session(
adapter=adapter,
task=task,
workspace_path=workspace_path,
session_id=session_id,
status=terminal_status,
metadata=metadata,
extra={
**artifacts,
"return_code": return_code,
"failure_reason": (
""
if terminal_status == "done"
else str(
state["timeout_reason"]
or state["fatal_reason"]
or f"{adapter.agent_type} exited with code {return_code}"
)
),
},
))
try:
await asyncio.shield(terminal_save)
except asyncio.CancelledError:
# Once the subprocess has exited, its terminal status must win
# over the early provider-stream `working` capability even if the
# parent coroutine is cancelled in this narrow handoff window.
with contextlib.suppress(Exception):
await terminal_save
raise
if state["timed_out"]:
return TaskResult(
status=TaskStatus.FAILED,
@@ -2107,6 +2416,11 @@ class ExternalAgentBroker:
or metadata.get("resume_session_id")
or ""
).strip(),
"provider_session_id": str(
extra.get("provider_session_id")
or metadata.get("provider_session_id")
or ""
).strip(),
**extra,
},
updated_at=datetime.now(),
@@ -2232,3 +2546,33 @@ class ExternalAgentBroker:
f"PR6 role adapter-state write failed "
f"sid={role_session_id} agent={adapter.agent_type}",
)
elif (
role_session_id
and result.status != TaskStatus.DONE
and hasattr(self.store, "get_role_session_adapter_state")
and hasattr(self.store, "update_role_session_adapter_state")
):
# A stream token is durable early so Stop can retain it. A normal
# terminal failure, however, must not leave that attempt's token
# pinned for a later unrelated turn.
try:
current = await self.store.get_role_session_adapter_state(
role_session_id,
adapter.agent_type,
)
if (
isinstance(current, dict)
and str(current.get("last_task_id", "") or "").strip()
== str(task.id or "").strip()
and str(current.get("source", "") or "").strip()
== "provider_stream"
):
await self.store.update_role_session_adapter_state(
role_session_id,
adapter.agent_type,
None,
)
except Exception:
logger.opt(exception=True).debug(
"Failed to clear provider-stream role state after terminal failure"
)
opc/layer3_agent/external_session_identity.py
+138
View File
@@ -0,0 +1,138 @@
"""Pure external provider-session identity helpers.
Monitoring rows exist before an external CLI reports its real resumable
thread/session id. A synthetic ``agent:project:task`` id is useful for local
observability, but is never a provider resume capability.
"""
from __future__ import annotations
from typing import Any, Iterable
NON_RESUMABLE_EXTERNAL_SESSION_STATUSES: frozenset[str] = frozenset({
"failed",
"cancelled",
"denied",
"rejected",
"hard_timeout",
"idle_timeout",
"startup_timeout",
})
def external_session_status_allows_resume(status: Any) -> bool:
status = str(status or "").strip().lower()
return status not in NON_RESUMABLE_EXTERNAL_SESSION_STATUSES
def external_session_allows_resume(session: Any | None) -> bool:
return session is not None and external_session_status_allows_resume(
getattr(session, "status", "")
)
def is_provider_session_token(
token: Any,
*,
agent_type: str,
project_id: str,
) -> bool:
value = str(token or "").strip()
if not value:
return False
normalized_agent = str(agent_type or "").strip()
normalized_project = str(project_id or "default").strip() or "default"
if normalized_agent and value.startswith(
f"{normalized_agent}:{normalized_project}:"
):
return False
if normalized_agent and value.startswith(f"{normalized_agent}:"):
if len(value.split(":")) >= 3:
return False
return True
def provider_token_from_external_session(
session: Any | None,
*,
agent_type: str,
project_id: str,
) -> str:
if not external_session_allows_resume(session):
return ""
metadata = dict(getattr(session, "metadata", {}) or {})
for candidate in (
metadata.get("resume_session_id"),
metadata.get("provider_session_id"),
getattr(session, "session_id", ""),
):
token = str(candidate or "").strip()
if is_provider_session_token(
token,
agent_type=agent_type,
project_id=project_id,
):
return token
return ""
def external_session_matches_provider_token(
session: Any | None,
token: Any,
) -> bool:
"""Return whether a persisted row represents ``token`` regardless of status."""
if session is None:
return False
expected = str(token or "").strip()
if not expected:
return False
metadata = dict(getattr(session, "metadata", {}) or {})
return expected in {
str(candidate or "").strip()
for candidate in (
metadata.get("resume_session_id"),
metadata.get("provider_session_id"),
getattr(session, "session_id", ""),
)
if str(candidate or "").strip()
}
def select_best_external_resume_session(
sessions: Iterable[Any],
*,
agent_type: str,
project_id: str,
) -> tuple[Any | None, str]:
"""Select the newest valid provider capability, ignoring placeholders."""
valid: list[tuple[Any, str]] = []
normalized_agent = str(agent_type or "").strip()
for session in list(sessions or []):
if (
str(getattr(session, "agent_type", "") or "").strip()
!= normalized_agent
):
continue
token = provider_token_from_external_session(
session,
agent_type=normalized_agent,
project_id=project_id,
)
if token:
valid.append((session, token))
if not valid:
return None, ""
def _sort_key(item: tuple[Any, str]) -> tuple[float, str]:
session, token = item
updated_at = getattr(session, "updated_at", None)
try:
timestamp = float(updated_at.timestamp())
except Exception:
timestamp = 0.0
return timestamp, token
return max(valid, key=_sort_key)
opc/plugins/office_ui/snapshot_builder.py
+22 -17
View File
@@ -2630,23 +2630,16 @@ async def _build_company_runtime_control_by_task(
]
checkpoint = identity.checkpoint
def _task_status_value(task: Any) -> str:
status = getattr(task, "status", "")
if hasattr(status, "value"):
return str(status.value or "").strip().lower()
return str(status or "").strip().lower().removeprefix("taskstatus.")
non_terminal_group = [
task for task in group
if _task_status_value(task) not in {"done", "failed", "cancelled"}
]
# Persisted RUNNING is only a projection. The controller-local
# execution registry is the sole proof that this process still owns a
# coroutine capable of monitoring and persisting the run.
# coroutine capable of monitoring and persisting the run. Driver
# ownership can deliberately sit on a terminal work-item envelope
# while the resumed scheduler is still active, so Task.status must not
# filter this lookup.
runtime_is_live = getattr(engine, "_task_runtime_is_live", None)
has_running_task = False
if callable(runtime_is_live):
for task in non_terminal_group:
for task in group:
live_result = runtime_is_live(task)
if inspect.isawaitable(live_result):
live_result = await live_result
@@ -2657,19 +2650,31 @@ async def _build_company_runtime_control_by_task(
str((getattr(task, "metadata", {}) or {}).get("company_runtime_stop_state", "") or "").strip()
in {"suspending", "suspended", "resuming_after_suspending"}
and bool(str((getattr(task, "metadata", {}) or {}).get("company_runtime_stop_marked_at", "") or "").strip())
for task in non_terminal_group
for task in group
)
any_held_suspended = any(
str((getattr(task, "metadata", {}) or {}).get("dispatch_hold", "") or "").strip()
== "company_runtime_suspended"
for task in non_terminal_group
for task in group
)
any_resuming = any(
str((getattr(task, "metadata", {}) or {}).get("company_runtime_stop_state", "") or "").strip() == "resuming"
for task in non_terminal_group
for task in group
)
checkpoint_status = str(getattr(checkpoint, "status", "") or "").strip().lower() if checkpoint is not None else ""
if any_resuming or checkpoint_status == "resuming":
# ``resuming`` is the durable checkpoint claim for the whole resumed
# execution, not merely a short UI transition. Once the controller
# registry proves that execution ownership exists, the runtime is
# stoppable and must project as running until that ownership ends.
if checkpoint_status == "pending":
state = "suspended"
elif checkpoint_status == "resuming" and (
any_held_suspended or any_stop_in_progress
):
state = "suspending"
elif checkpoint_status == "resuming" and has_running_task:
state = "running"
elif any_resuming or checkpoint_status == "resuming":
state = "resuming"
elif checkpoint is not None:
state = "suspended"
@@ -2678,7 +2683,7 @@ async def _build_company_runtime_control_by_task(
and any(
str((getattr(task, "metadata", {}) or {}).get("company_runtime_stop_state", "") or "").strip()
in {"suspending", "resuming_after_suspending"}
for task in non_terminal_group
for task in group
)
):
state = "suspending"
opc/plugins/office_ui/tests/test_snapshot_builder_company_kanban.py
+127 -1
View File
@@ -7,7 +7,7 @@ from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
from opc.core.models import DelegationRun, DelegationWorkItem, Phase
from opc.core.models import DelegationRun, DelegationWorkItem, ExecutionCheckpoint, Phase
from opc.database.store import OPCStore
from opc.plugins.office_ui.snapshot_builder import (
_build_company_runtime_control_by_task,
@@ -145,6 +145,132 @@ class CompanyKanbanProjectionTests(unittest.IsolatedAsyncioTestCase):
self.assertFalse(control["parent-task"]["can_stop"])
engine._task_runtime_is_live.assert_awaited_once_with(parent_task)
async def test_registry_live_resume_checkpoint_projects_running_and_stoppable(self) -> None:
created_at = datetime.now(timezone.utc)
parent_task = SimpleNamespace(
id="parent-task",
session_id="session-root",
parent_session_id="",
title="Root runtime",
status="running",
created_at=created_at,
metadata={
"exec_mode": "company",
"company_profile": "corporate",
},
)
terminal_driver_task = SimpleNamespace(
id="terminal-driver-task",
session_id="driver-session",
parent_session_id="session-root",
title="Terminal driver envelope",
status="done",
created_at=created_at,
metadata={
"mode": "company",
"work_item_runtime": True,
"work_item_projection_id": "driver",
},
)
checkpoint = ExecutionCheckpoint(
checkpoint_id="resume-checkpoint",
project_id="proj1",
session_id="session-root",
checkpoint_type="company_runtime_suspended",
status="resuming",
task_id="terminal-driver-task",
payload={"parent_session_id": "session-root"},
)
store = MagicMock()
store.get_execution_checkpoints = AsyncMock(return_value=[checkpoint])
engine = SimpleNamespace(
store=store,
_task_runtime_is_live=AsyncMock(
side_effect=lambda task: task.id == "terminal-driver-task"
),
)
control = await _build_company_runtime_control_by_task(
engine,
[parent_task, terminal_driver_task],
"proj1",
)
self.assertEqual(
control["parent-task"]["runtime_control_state"],
"running",
)
self.assertTrue(control["parent-task"]["can_stop"])
self.assertFalse(control["parent-task"]["can_resume"])
async def test_stop_hold_and_pending_checkpoint_override_live_resume_projection(self) -> None:
created_at = datetime.now(timezone.utc)
parent_task = SimpleNamespace(
id="parent-task",
session_id="session-root",
parent_session_id="",
title="Root runtime",
status="running",
created_at=created_at,
metadata={
"exec_mode": "company",
"company_profile": "corporate",
},
)
child_task = SimpleNamespace(
id="child-task",
session_id="child-session",
parent_session_id="session-root",
title="Live child",
status="done",
created_at=created_at,
metadata={
"mode": "company",
"work_item_runtime": True,
"work_item_projection_id": "child",
"dispatch_hold": "company_runtime_suspended",
},
)
checkpoint = ExecutionCheckpoint(
checkpoint_id="resume-checkpoint",
project_id="proj1",
session_id="session-root",
checkpoint_type="company_runtime_suspended",
status="resuming",
task_id="child-task",
payload={"parent_session_id": "session-root"},
)
store = MagicMock()
store.get_execution_checkpoints = AsyncMock(return_value=[checkpoint])
engine = SimpleNamespace(
store=store,
_task_runtime_is_live=AsyncMock(return_value=True),
)
suspending = await _build_company_runtime_control_by_task(
engine,
[parent_task, child_task],
"proj1",
)
self.assertEqual(
suspending["parent-task"]["runtime_control_state"],
"suspending",
)
self.assertFalse(suspending["parent-task"]["can_stop"])
checkpoint.status = "pending"
suspended = await _build_company_runtime_control_by_task(
engine,
[parent_task, child_task],
"proj1",
)
self.assertEqual(
suspended["parent-task"]["runtime_control_state"],
"suspended",
)
self.assertFalse(suspended["parent-task"]["can_stop"])
self.assertTrue(suspended["parent-task"]["can_resume"])
async def test_runtime_control_treats_dispatch_hold_as_suspending_without_checkpoint(self) -> None:
created_at = datetime.now(timezone.utc)
parent_task = SimpleNamespace(
opc/plugins/office_ui/ws_handler.py
+7 -1
View File
@@ -7985,7 +7985,13 @@ class WSHandler:
async with lock:
try:
try:
await self._set_company_runtime_control(target, state="resuming")
await self._set_company_runtime_control(
target,
state="resuming",
checkpoint_id=str(
getattr(checkpoint, "checkpoint_id", "") or ""
).strip(),
)
except Exception:
logger.opt(exception=True).debug("failed to broadcast company suspend reply routing state")
tests/test_company_runtime_identity.py
+5
View File
@@ -250,6 +250,11 @@ def test_work_item_chat_resume_uses_canonical_ui_anchor_as_engine_origin() -> No
assert call.kwargs["session_id"] == "runtime-session"
assert call.kwargs["origin_task_id"] == "ui-anchor"
assert handler._session_to_task["runtime-session"] == "ui-anchor"
handler._set_company_runtime_control.assert_awaited_once_with(
target,
state="resuming",
checkpoint_id="checkpoint-1",
)
handler.on_kanban_changed.assert_awaited_once_with(engine=run_engine)
asyncio.run(scenario())
tests/test_company_runtime_suspend_resume.py
+630
View File
@@ -1,9 +1,13 @@
from __future__ import annotations
import asyncio
import tempfile
import unittest
from datetime import datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock
from opc.core.models import (
CompanyMemberSession,
@@ -13,6 +17,7 @@ from opc.core.models import (
ExternalSession,
Phase,
Task,
TaskResult,
TaskStatus,
)
from opc.database.store import OPCStore
@@ -87,6 +92,15 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
},
)
)
await store.save_delegation_role_session(
DelegationRoleSession(
role_session_id=role_session_id,
run_id="run-1",
project_id="proj1",
role_id="executor",
seat_id="seat-1",
)
)
await store.save_delegation_work_item(
DelegationWorkItem(
work_item_id=work_item_id,
@@ -262,6 +276,603 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(refreshed_item.metadata.get("dispatch_hold"), "")
self.assertEqual(refreshed_item.claimed_by_role_runtime_session_id, "")
async def test_second_stop_during_resumed_execution_restores_pending_checkpoint(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
engine = self._engine(store)
first_stop = await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
self.assertIsNotNone(first_stop)
execution_started = asyncio.Event()
class BlockingCompanyExecutor:
def __init__(self) -> None:
self._notify_kanban_changed = AsyncMock()
async def execute(
self,
_plan: CompanyWorkItemRuntimePlan,
_tasks: list[Task],
) -> str:
execution_started.set()
await asyncio.Event().wait()
return "unreachable"
executor = BlockingCompanyExecutor()
engine.company_executor = executor # type: ignore[assignment]
resume_task = asyncio.create_task(engine._maybe_resume_checkpoint(
"continue",
"sess-parent",
reply_metadata={"ui_force_resume": True},
))
await asyncio.wait_for(execution_started.wait(), timeout=1)
executor._notify_kanban_changed.assert_awaited_once_with()
self.assertTrue(engine._active_task_run_registry.is_active("proj1", task.id))
second_stop = await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
self.assertIsNotNone(second_stop)
self.assertEqual(
second_stop["checkpoint_id"],
first_stop["checkpoint_id"],
)
resume_task.cancel()
with self.assertRaises(asyncio.CancelledError):
await resume_task
pending = await store.get_execution_checkpoints(
project_id="proj1",
session_id="sess-parent",
checkpoint_types=["company_runtime_suspended"],
statuses=["pending"],
)
refreshed_task = await store.get_task(task.id)
refreshed_item = await store.get_delegation_work_item("work-item-1")
self.assertEqual(len(pending), 1)
self.assertEqual(pending[0].checkpoint_id, first_stop["checkpoint_id"])
self.assertEqual(pending[0].payload.get("resume_state"), "interrupted")
assert refreshed_task is not None
assert refreshed_item is not None
self.assertEqual(
refreshed_task.metadata.get("dispatch_hold"),
"company_runtime_suspended",
)
self.assertEqual(
refreshed_item.metadata.get("dispatch_hold"),
"company_runtime_suspended",
)
self.assertFalse(engine._active_task_run_registry.is_active("proj1", task.id))
async def test_resume_attempt_pins_unlocked_auto_external_agent_without_permanent_lock(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
task.assigned_external_agent = "opencode"
task.metadata.update({
"delegation_seat_id": "seat-1",
"employee_assignment": {"employee_id": "employee-executor"},
"selected_execution_agent": "codex",
"selected_execution_agent_source": "fallback_rules",
"execution_agent_locked": False,
"preferred_external_agent": "codex",
"agent_selection": {
"selected": "opencode",
"selection_source": "llm",
},
})
await store.save_task(task)
engine = self._engine(store)
engine.org_engine = SimpleNamespace()
engine._available_external_agents = lambda: ["opencode", "codex"]
adaptive_selector = AsyncMock(return_value=(
"codex",
{
"selected": "codex",
"selection_source": "fallback_rules",
},
))
engine._select_task_execution_agent_via_llm = adaptive_selector
await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
checkpoint = (
await store.get_pending_checkpoints(
project_id="proj1",
session_id="sess-parent",
checkpoint_types=["company_runtime_suspended"],
)
)[0]
checkpoint_identity = checkpoint.payload["task_snapshots"][0][
"execution_identity"
]
self.assertEqual(checkpoint_identity["selected_execution_agent"], "opencode")
self.assertEqual(checkpoint_identity["assigned_external_agent"], "opencode")
self.assertEqual(checkpoint_identity["agent_selection_source"], "llm")
captured: dict[str, Any] = {}
class SelectingExecutor:
async def execute(self, plan: CompanyWorkItemRuntimePlan, tasks: list[Task]) -> str:
resumed = tasks[0]
captured["selected"] = await engine._assign_task_execution_agent(
resumed,
role=SimpleNamespace(),
)
captured["task"] = resumed
return "runtime resumed"
engine.company_executor = SelectingExecutor()
await engine._maybe_resume_checkpoint(
"continue",
"sess-parent",
reply_metadata={"ui_force_resume": True},
)
resumed_task = captured["task"]
self.assertEqual(captured["selected"], "opencode")
self.assertEqual(resumed_task.assigned_external_agent, "opencode")
self.assertEqual(resumed_task.assigned_to, "executor")
self.assertEqual(resumed_task.metadata["delegation_seat_id"], "seat-1")
self.assertEqual(
resumed_task.metadata["employee_assignment"]["employee_id"],
"employee-executor",
)
self.assertFalse(resumed_task.metadata["execution_agent_locked"])
self.assertNotIn(
"_company_runtime_resume_execution_agent_pin",
resumed_task.metadata,
)
self.assertEqual(
resumed_task.metadata["agent_selection"]["selection_source"],
"company_runtime_resume_checkpoint",
)
adaptive_selector.assert_not_awaited()
# A later ordinary dispatch has no checkpoint pin and may adapt again.
selected_after_resume = await engine._assign_task_execution_agent(
resumed_task,
role=SimpleNamespace(),
)
self.assertEqual(selected_after_resume, "codex")
adaptive_selector.assert_awaited_once()
async def test_resume_attempt_pins_native_without_running_adaptive_selector(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
task.assigned_external_agent = None
task.metadata.update({
"selected_execution_agent": "native",
"selected_execution_agent_source": "fallback_rules",
"execution_agent_locked": False,
})
await store.save_task(task)
engine = self._engine(store)
engine.org_engine = SimpleNamespace()
engine._available_external_agents = lambda: ["opencode", "codex"]
adaptive_selector = AsyncMock(return_value=(
"opencode",
{"selected": "opencode", "selection_source": "fallback_rules"},
))
engine._select_task_execution_agent_via_llm = adaptive_selector
await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
captured: dict[str, Any] = {}
class SelectingExecutor:
async def execute(self, plan: CompanyWorkItemRuntimePlan, tasks: list[Task]) -> str:
captured["selected"] = await engine._assign_task_execution_agent(
tasks[0],
role=SimpleNamespace(),
)
captured["task"] = tasks[0]
return "runtime resumed"
engine.company_executor = SelectingExecutor()
await engine._maybe_resume_checkpoint(
"continue",
"sess-parent",
reply_metadata={"ui_force_resume": True},
)
self.assertIsNone(captured["selected"])
self.assertIsNone(captured["task"].assigned_external_agent)
self.assertEqual(
captured["task"].metadata["selected_execution_agent"],
"native",
)
adaptive_selector.assert_not_awaited()
async def test_resume_identity_mismatch_fails_closed_and_returns_checkpoint_to_pending(self) -> None:
mutations = {
"role_id": lambda item: setattr(item, "role_id", "replacement-role"),
"seat_id": lambda item: setattr(item, "seat_id", "replacement-seat"),
"role_runtime_session_id": lambda item: setattr(
item,
"role_runtime_session_id",
"replacement-role-session",
),
"employee_id": lambda item: item.metadata.update({
"employee_assignment": {"employee_id": "replacement-employee"},
}),
}
for field_name, mutate in mutations.items():
with self.subTest(field_name=field_name):
store = await self._store()
_, task = await self._seed_runtime(store)
task.metadata.update({
"delegation_seat_id": "seat-1",
"employee_assignment": {"employee_id": "employee-executor"},
"selected_execution_agent": "codex",
"selected_execution_agent_source": "fallback_rules",
"execution_agent_locked": False,
})
await store.save_task(task)
work_item = await store.get_delegation_work_item("work-item-1")
assert work_item is not None
work_item.metadata = {
**dict(work_item.metadata or {}),
"employee_assignment": {"employee_id": "employee-executor"},
}
await store.save_delegation_work_item(work_item)
engine = self._engine(store)
await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
mutate(work_item)
await store.save_delegation_work_item(work_item)
executor = self._CapturingCompanyExecutor()
engine.company_executor = executor
with self.assertRaisesRegex(RuntimeError, field_name):
await engine._maybe_resume_checkpoint(
"continue",
"sess-parent",
reply_metadata={"ui_force_resume": True},
)
pending = await store.get_execution_checkpoints(
project_id="proj1",
session_id="sess-parent",
checkpoint_types=["company_runtime_suspended"],
statuses=["pending"],
)
resuming = await store.get_execution_checkpoints(
project_id="proj1",
session_id="sess-parent",
checkpoint_types=["company_runtime_suspended"],
statuses=["resuming"],
)
self.assertEqual(len(pending), 1)
self.assertEqual(resuming, [])
self.assertEqual(
pending[0].payload.get("resume_state"),
"failed_before_handoff",
)
self.assertEqual(executor.calls, [])
async def test_resume_repairs_stale_task_projection_from_work_item_and_role_session(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
task.metadata.update({
"delegation_seat_id": "seat-1",
"employee_assignment": {"employee_id": "employee-executor"},
"selected_execution_agent": "codex",
})
await store.save_task(task)
work_item = await store.get_delegation_work_item("work-item-1")
role_session = await store.get_delegation_role_session("role-runtime-1")
assert work_item is not None and role_session is not None
work_item.metadata = {
**dict(work_item.metadata or {}),
"employee_assignment": {"employee_id": "employee-executor"},
}
role_session.employee_id = "employee-executor"
await store.save_delegation_work_item(work_item)
await store.save_delegation_role_session(role_session)
engine = self._engine(store)
await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
stale_task = await store.get_task(task.id)
assert stale_task is not None
stale_task.assigned_to = "stale-role"
stale_task.metadata.update({
"delegation_seat_id": "stale-seat",
"delegation_role_session_id": "stale-role-session",
"employee_assignment": {"employee_id": "stale-employee"},
})
await store.save_task(stale_task)
captured: dict[str, Task] = {}
class CapturingExecutor:
async def execute(self, plan: CompanyWorkItemRuntimePlan, tasks: list[Task]) -> str:
captured["task"] = tasks[0]
return "resumed"
engine.company_executor = CapturingExecutor()
await engine._maybe_resume_checkpoint(
"continue",
"sess-parent",
reply_metadata={"ui_force_resume": True},
)
repaired = captured["task"]
self.assertEqual(repaired.assigned_to, "executor")
self.assertEqual(repaired.metadata["delegation_seat_id"], "seat-1")
self.assertEqual(
repaired.metadata["delegation_role_session_id"],
"role-runtime-1",
)
self.assertEqual(
repaired.metadata["employee_assignment"]["employee_id"],
"employee-executor",
)
async def test_resume_rejects_role_runtime_session_identity_mismatch(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
engine = self._engine(store)
await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
role_session = await store.get_delegation_role_session("role-runtime-1")
assert role_session is not None
role_session.role_id = "replacement-role"
await store.save_delegation_role_session(role_session)
engine.company_executor = self._CapturingCompanyExecutor()
with self.assertRaisesRegex(RuntimeError, "role session role_id"):
await engine._maybe_resume_checkpoint(
"continue",
"sess-parent",
reply_metadata={"ui_force_resume": True},
)
async def test_force_native_real_work_item_path_consumes_resume_pin(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
task.assigned_external_agent = None
task.metadata.update({
"force_native_execution": True,
"selected_execution_agent": "native",
"agent_selection": {
"selected": "native",
"selection_source": "forced_native",
},
})
await store.save_task(task)
engine = self._engine(store)
await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
checkpoint = (
await store.get_pending_checkpoints(
project_id="proj1",
session_id="sess-parent",
checkpoint_types=["company_runtime_suspended"],
)
)[0]
stored_task = await store.get_task(task.id)
assert stored_task is not None
resumed = (
await engine._prepare_company_runtime_tasks_for_resume(
[stored_task],
checkpoint.payload,
resume_task_ids={task.id},
)
)[0]
self.assertIn(
"_company_runtime_resume_execution_agent_pin",
resumed.metadata,
)
role = SimpleNamespace(role_id="executor", preferred_external_agent="codex")
class OrgEngine:
@staticmethod
def get_role_for_work_item(role_id: str, tags: list[str]) -> Any:
return role
engine.org_engine = OrgEngine()
executor = CompanyWorkItemExecutor(
org_engine=engine.org_engine,
communication=None,
approval_engine=SimpleNamespace(),
memory=None,
execute_task=AsyncMock(return_value=TaskResult(
status=TaskStatus.FAILED,
content="stop after selector",
)),
save_task=store.save_task,
store=store,
agent_selector=engine._assign_task_execution_agent,
)
result = await executor._run_work_item(resumed, {"execution": resumed})
self.assertEqual(result.status, TaskStatus.FAILED)
self.assertNotIn(
"_company_runtime_resume_execution_agent_pin",
resumed.metadata,
)
self.assertEqual(
resumed.metadata["agent_selection"]["selection_source"],
"company_runtime_resume_checkpoint",
)
self.assertIsNone(resumed.assigned_external_agent)
async def test_resume_does_not_overwrite_newer_durable_role_adapter_state(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
await store.update_role_session_adapter_state(
"role-runtime-1",
"codex",
{"resume_session_id": "thread-before-stop", "updated_at": "2026-01-01"},
)
engine = self._engine(store)
await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
await store.update_role_session_adapter_state(
"role-runtime-1",
"codex",
{"resume_session_id": "thread-after-checkpoint", "updated_at": "2026-07-14"},
)
engine.company_executor = self._CapturingCompanyExecutor()
await engine._maybe_resume_checkpoint(
"continue",
"sess-parent",
reply_metadata={"ui_force_resume": True},
)
state = await store.get_role_session_adapter_state(
"role-runtime-1",
"codex",
)
assert state is not None
self.assertEqual(state["resume_session_id"], "thread-after-checkpoint")
async def test_resume_fixed_backend_failure_never_tries_alternate_or_native(self) -> None:
engine = OPCEngine(project_id="proj1")
attempted_agents: list[str] = []
class Adapter:
def __init__(self, name: str) -> None:
self.agent_type = name
self.config = SimpleNamespace(
session_mode="auto",
run_mode="batch",
)
def supports_interactive(self) -> bool:
return False
def build_invocation(
self,
task: Task,
workspace_path: str | None = None,
) -> tuple[list[str], dict[str, Any]]:
return [self.agent_type], {
"agent": self.agent_type,
"command": self.agent_type,
}
codex = Adapter("codex")
opencode = Adapter("opencode")
class Registry:
def get_ordered_available(self) -> list[tuple[str, Adapter]]:
return [("opencode", opencode), ("codex", codex)]
def get(self, name: str) -> Adapter | None:
return {"codex": codex, "opencode": opencode}.get(name)
class Broker:
async def run(
self,
*,
adapter: Adapter,
task: Task,
workspace_path: str,
on_progress: Any = None,
prepared_task: Task | None = None,
) -> TaskResult:
attempted_agents.append(adapter.agent_type)
return TaskResult(
status=TaskStatus.FAILED,
content=f"{adapter.agent_type} failed",
artifacts={},
)
engine.adapter_registry = Registry()
engine.external_broker = Broker()
engine._resolve_external_workspace = lambda task: "/tmp/workspace"
engine._build_external_agent_task = AsyncMock(side_effect=lambda task: task)
engine._configure_external_adapter_for_task = AsyncMock(
side_effect=lambda task, adapter: (adapter, {}),
)
engine._emit_external_agent_audit = AsyncMock()
engine._run_native_agent = AsyncMock(return_value=TaskResult(
status=TaskStatus.DONE,
content="native fallback",
))
task = Task(
id="resume-fixed-backend",
title="Resume fixed backend",
project_id="proj1",
assigned_external_agent="codex",
metadata={
"target_output_dir": "/tmp/workspace",
"agent_selection": {
"selection_source": "company_runtime_resume_checkpoint",
},
},
)
result = await engine._run_task_once(task)
self.assertEqual(result.status, TaskStatus.FAILED)
self.assertEqual(attempted_agents, ["codex"])
engine._run_native_agent.assert_not_awaited()
async def test_suspend_prefers_provider_token_over_newer_synthetic_monitor_row(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
await store.save_external_session(ExternalSession(
agent_type="codex",
project_id="proj1",
session_id="codex:proj1:execution-task",
opc_session_id="role-runtime-1",
task_id=task.id,
workspace_path="/tmp/opc-test",
run_mode="interactive",
status="working",
metadata={},
updated_at=datetime.now() + timedelta(seconds=1),
))
engine = self._engine(store)
await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
checkpoint = (
await store.get_pending_checkpoints(
project_id="proj1",
session_id="sess-parent",
checkpoint_types=["company_runtime_suspended"],
)
)[0]
external = checkpoint.payload["external_sessions"][task.id]
self.assertEqual(external["session_id"], "provider-session-1")
self.assertEqual(external["resume_session_id"], "provider-session-1")
async def test_text_after_stop_routes_to_final_decider_instead_of_plain_resume(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
@@ -388,6 +999,7 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
run_id="run-1",
role_id="ceo",
seat_id="seat-ceo",
role_runtime_session_id="role-ceo",
title="CEO delivery",
kind="deliver",
projection_id="ceo-deliver",
@@ -403,6 +1015,7 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
run_id="run-1",
role_id="engineer",
seat_id="seat-engineer",
role_runtime_session_id="role-engineer",
title="Worker execution",
kind="execute",
projection_id="worker-execute",
@@ -415,6 +1028,18 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
)
await store.save_delegation_work_item(ceo_item)
await store.save_delegation_work_item(worker_item)
await store.save_delegation_role_session(DelegationRoleSession(
role_session_id="role-ceo",
run_id="run-1",
role_id="ceo",
seat_id="seat-ceo",
))
await store.save_delegation_role_session(DelegationRoleSession(
role_session_id="role-engineer",
run_id="run-1",
role_id="engineer",
seat_id="seat-engineer",
))
common_metadata = {
"company_profile": "corporate",
@@ -1455,6 +2080,11 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
)
resumed_task = captured["tasks"][0]
self.assertEqual(resumed_task.assigned_external_agent, "codex")
self.assertEqual(
resumed_task.metadata["selected_execution_agent"],
"codex",
)
self.assertNotIn("external_resume_session_id", resumed_task.metadata)
self.assertEqual(resumed_task.metadata["external_resume_fallback"], "context_replay")
tests/test_engine_runtime_recovery.py
+79 -1
View File
@@ -802,6 +802,24 @@ async def test_resume_candidates_use_nonterminal_work_item_over_terminal_task(
"phase": Phase.RUNNING.value,
}
],
"task_snapshots": [
{
"task_id": task.id,
"status": TaskStatus.DONE.value,
"assigned_to": "executor",
"assigned_external_agent": "",
"selected_execution_agent": "native",
"work_item_id": work_item.work_item_id,
"work_item": {
"work_item_id": work_item.work_item_id,
"phase": Phase.RUNNING.value,
"role_id": "executor",
"seat_id": "seat::executor",
"role_runtime_session_id": "",
"metadata": {},
},
}
],
},
resume_task_ids={task.id},
)
@@ -1062,7 +1080,9 @@ async def test_successful_checkpoint_handoff_reopens_cancelled_ui_anchor(
await store.save_execution_checkpoint(checkpoint)
engine = OPCEngine(project_id="project-a")
engine.store = store
engine.company_executor = SimpleNamespace()
engine.company_executor = SimpleNamespace(
_notify_kanban_changed=AsyncMock(),
)
engine._prepare_company_runtime_tasks_for_resume = AsyncMock(
return_value=[runtime_task]
)
@@ -1087,6 +1107,7 @@ async def test_successful_checkpoint_handoff_reopens_cancelled_ui_anchor(
handed_off_tasks, driver_ownership = handed_off
assert handed_off_tasks == [runtime_task]
assert driver_ownership is not None
engine.company_executor._notify_kanban_changed.assert_awaited_once_with()
driver_ownership.release()
assert still_cancelled is not None
assert still_cancelled.status == TaskStatus.CANCELLED
@@ -1110,6 +1131,63 @@ async def test_successful_checkpoint_handoff_reopens_cancelled_ui_anchor(
await store.close()
@_async_test
async def test_failed_checkpoint_handoff_does_not_publish_running_snapshot(
tmp_path: Path,
) -> None:
store = OPCStore(tmp_path / "tasks.db")
await store.initialize()
try:
runtime_task = _runtime_task(
task_id="runtime-task",
status=TaskStatus.BLOCKED,
)
await store.save_task(runtime_task)
checkpoint = ExecutionCheckpoint(
project_id="project-a",
session_id="root-session",
checkpoint_type="company_runtime_interrupted",
task_id=runtime_task.id,
payload={
"task_ids": [runtime_task.id],
"parent_session_id": "root-session",
},
)
await store.save_execution_checkpoint(checkpoint)
engine = OPCEngine(project_id="project-a")
engine.store = store
engine.company_executor = SimpleNamespace(
_notify_kanban_changed=AsyncMock(),
)
engine._prepare_company_runtime_tasks_for_resume = AsyncMock(
side_effect=RuntimeError("prepare failed"),
)
with pytest.raises(RuntimeError, match="prepare failed"):
await engine._handoff_company_suspend_checkpoint(
checkpoint,
payload=dict(checkpoint.payload),
parent_session_id="root-session",
tasks=[runtime_task],
)
engine.company_executor._notify_kanban_changed.assert_not_awaited()
assert not engine._active_task_run_registry.is_active(
"project-a",
runtime_task.id,
)
pending = await store.get_execution_checkpoints(
project_id="project-a",
session_id="root-session",
checkpoint_types=["company_runtime_interrupted"],
statuses=["pending"],
)
assert len(pending) == 1
assert pending[0].payload.get("resume_state") == "failed_before_handoff"
finally:
await store.close()
@_async_test
async def test_executor_failure_restores_pending_checkpoint_and_durable_holds(
tmp_path: Path,
tests/test_external_agent_monitoring.py
+354
View File
@@ -19,6 +19,7 @@ from opc.core.models import (
AgentStatus,
ApprovalAction,
ApprovalDecision,
DelegationRoleSession,
DelegationWorkItem,
ExecutionMode,
Phase,
@@ -32,6 +33,8 @@ from opc.engine import OPCEngine
from opc.layer1_perception.context_assembler import ContextAssembler, ExternalContextLayers
from opc.layer2_organization import comms as file_comms
from opc.layer2_organization.prompt_contract import make_prompt_contract
from opc.layer2_organization.company_mode import serialize_company_work_item_runtime_plan
from opc.layer2_organization.org_work_item_planner import CompanyWorkItemRuntimePlan
from opc.layer2_organization.work_item_links import set_linked_work_item_id
from opc.layer3_agent.adapters.claude_code import ClaudeCodeAdapter
from opc.layer3_agent.adapters.codex_adapter import CodexAdapter
@@ -548,6 +551,357 @@ class ExternalAgentMonitoringTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(session.metadata.get("resume_session_id"), "ses_1")
await store.close()
async def test_live_provider_thread_is_durable_before_stop_checkpoint_and_reused(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
store = OPCStore(Path(tmpdir) / "tasks.db")
await store.initialize()
role_session_id = "role-runtime::run-live::executor"
parent_session_id = "company-live-session"
await store.save_delegation_role_session(DelegationRoleSession(
role_session_id=role_session_id,
run_id="run-live",
role_id="executor",
seat_id="seat-live",
))
await store.save_task(Task(
id="ui-live",
title="Company chat",
session_id=parent_session_id,
project_id="proj1",
status=TaskStatus.IDLE,
metadata={"exec_mode": "company", "company_profile": "corporate"},
))
work_item = DelegationWorkItem(
work_item_id="work-live",
run_id="run-live",
role_id="executor",
seat_id="seat-live",
role_runtime_session_id=role_session_id,
title="Live work",
projection_id="live",
phase=Phase.RUNNING,
claimed_by_role_runtime_session_id=role_session_id,
claimed_by_seat_id="seat-live",
metadata={"work_item_projection_id": "live"},
)
await store.save_delegation_work_item(work_item)
plan = CompanyWorkItemRuntimePlan(
profile="corporate",
metadata={
"execution_model": "multi_team_org",
"runtime_model": "multi_team_org",
},
)
task = Task(
id="live-provider-task",
title="Live provider task",
session_id="live-child-session",
parent_session_id=parent_session_id,
project_id="proj1",
assigned_to="executor",
assigned_external_agent="script_agent",
status=TaskStatus.RUNNING,
metadata={
"work_item_runtime": True,
"work_item_projection_id": "live",
"delegation_run_id": "run-live",
"delegation_role_session_id": role_session_id,
"delegation_seat_id": "seat-live",
"selected_execution_agent": "script_agent",
"selected_execution_agent_source": "fallback_rules",
"company_profile": "corporate",
"execution_model": "multi_team_org",
"runtime_model": "multi_team_org",
"company_work_item_plan": serialize_company_work_item_runtime_plan(plan),
},
)
set_linked_work_item_id(task, work_item.work_item_id)
await store.save_task(task)
await store.link_work_item_runtime_task(work_item.work_item_id, task.id)
broker = ExternalAgentBroker(store, _ApprovalStub())
adapter = _ScriptAdapter(
"import json,sys,time\n"
"print(json.dumps({'sessionID': 'provider-live-thread'}))\n"
"sys.stdout.flush()\n"
"time.sleep(30)\n"
)
run_task = asyncio.create_task(broker.run(adapter, task, tmpdir))
role_state = None
for _ in range(200):
role_state = await store.get_role_session_adapter_state(
role_session_id,
"script_agent",
)
if role_state and role_state.get("resume_session_id"):
break
await asyncio.sleep(0.01)
self.assertIsNotNone(role_state)
assert role_state is not None
self.assertEqual(
role_state["resume_session_id"],
"provider-live-thread",
)
self.assertFalse(run_task.done())
engine = OPCEngine()
engine.project_id = "proj1"
engine.store = store
await engine.suspend_company_runtime(
origin_task_id="ui-live",
session_id=parent_session_id,
reason="user_stop",
)
checkpoint = (
await store.get_pending_checkpoints(
project_id="proj1",
session_id=parent_session_id,
checkpoint_types=["company_runtime_suspended"],
)
)[0]
captured = checkpoint.payload["external_sessions"][task.id]
self.assertEqual(
captured["resume_session_id"],
"provider-live-thread",
)
self.assertEqual(
captured["provider_session_id"],
"provider-live-thread",
)
run_task.cancel()
with self.assertRaises(asyncio.CancelledError):
await run_task
resumed_task = await store.get_task(task.id)
assert resumed_task is not None
resume_adapter = _ScriptAdapter("print('unused')")
resume_adapter.config.resume_session_flag = "--resume"
await broker._restore_session_resume_from_store(
resume_adapter,
resumed_task,
)
self.assertEqual(resume_adapter.config.session_mode, "resume")
self.assertEqual(
resume_adapter.config.session_id,
"provider-live-thread",
)
await store.close()
async def test_codex_thread_started_stream_restores_real_resume_argv(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
store = OPCStore(Path(tmpdir) / "tasks.db")
await store.initialize()
role_session_id = "role-runtime::codex-live::executor"
await store.save_delegation_role_session(DelegationRoleSession(
role_session_id=role_session_id,
run_id="codex-live",
project_id="proj1",
role_id="executor",
seat_id="seat-codex-live",
))
task = Task(
id="codex-live-task",
title="Codex live task",
description="Keep the same Codex thread.",
project_id="proj1",
session_id="codex-live-child",
parent_session_id="codex-live-parent",
assigned_to="executor",
assigned_external_agent="codex",
status=TaskStatus.RUNNING,
metadata={
"work_item_runtime": True,
"delegation_role_session_id": role_session_id,
"delegation_seat_id": "seat-codex-live",
},
)
await store.save_task(task)
class ScriptedCodexAdapter(CodexAdapter):
async def start_process(
self,
cmd: list[str],
workspace_path: str,
extra_env: dict[str, str] | None = None,
task: Task | None = None,
launch_metadata: dict[str, object] | None = None,
) -> asyncio.subprocess.Process:
return await asyncio.create_subprocess_exec(
sys.executable,
"-c",
(
"import json,sys,time\n"
"print(json.dumps({'type':'thread.started','thread_id':'thread-real-codex'}))\n"
"sys.stdout.flush()\n"
"time.sleep(30)\n"
),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=workspace_path,
)
broker = ExternalAgentBroker(store, _ApprovalStub())
live_adapter = ScriptedCodexAdapter()
live_adapter.config.run_mode = "exec"
run_task = asyncio.create_task(
broker.run(live_adapter, task, tmpdir)
)
state = None
for _ in range(200):
state = await store.get_role_session_adapter_state(
role_session_id,
"codex",
)
if state and state.get("resume_session_id"):
break
await asyncio.sleep(0.01)
assert state is not None
self.assertEqual(state["resume_session_id"], "thread-real-codex")
run_task.cancel()
with self.assertRaises(asyncio.CancelledError):
await run_task
resume_adapter = CodexAdapter()
resumed_task = await store.get_task(task.id)
assert resumed_task is not None
await broker._restore_session_resume_from_store(
resume_adapter,
resumed_task,
)
self.assertEqual(resume_adapter.config.session_mode, "resume")
self.assertEqual(resume_adapter.config.session_id, "thread-real-codex")
cmd, _metadata = resume_adapter.build_invocation(
resumed_task,
workspace_path=tmpdir,
)
self.assertEqual(cmd[1:3], ["exec", "resume"])
self.assertIn("thread-real-codex", cmd)
self.assertEqual(cmd[-1], "-")
await store.close()
async def test_cleanup_cancellation_terminalizes_checkpoint_provider_token(self) -> None:
"""A Stop checkpoint's working token cannot outlive a failed process."""
with tempfile.TemporaryDirectory() as tmpdir:
store = OPCStore(Path(tmpdir) / "tasks.db")
await store.initialize()
role_session_id = "role-runtime::cleanup-race::executor"
await store.save_delegation_role_session(DelegationRoleSession(
role_session_id=role_session_id,
run_id="cleanup-race",
project_id="proj1",
role_id="executor",
seat_id="seat-cleanup-race",
))
task = Task(
id="cleanup-race-task",
title="Cleanup cancellation race",
description="Do not resume a provider thread that later failed.",
project_id="proj1",
session_id="cleanup-race-child",
parent_session_id="cleanup-race-parent",
assigned_to="executor",
assigned_external_agent="codex",
status=TaskStatus.RUNNING,
metadata={"delegation_role_session_id": role_session_id},
)
await store.save_task(task)
cleanup_started = asyncio.Event()
allow_cleanup = asyncio.Event()
class CleanupRaceCodexAdapter(CodexAdapter):
async def start_process(
self,
cmd: list[str],
workspace_path: str,
extra_env: dict[str, str] | None = None,
task: Task | None = None,
launch_metadata: dict[str, object] | None = None,
) -> asyncio.subprocess.Process:
return await asyncio.create_subprocess_exec(
sys.executable,
"-c",
(
"import json,sys,time\n"
"print(json.dumps({'type':'thread.started','thread_id':'thread-cleanup-race'}))\n"
"sys.stdout.flush()\n"
"time.sleep(.2)\n"
"sys.exit(7)\n"
),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=workspace_path,
)
async def cleanup_process(self, proc: asyncio.subprocess.Process) -> None:
cleanup_started.set()
await allow_cleanup.wait()
await super().cleanup_process(proc)
broker = ExternalAgentBroker(store, _ApprovalStub())
live_adapter = CleanupRaceCodexAdapter()
run_task = asyncio.create_task(broker.run(live_adapter, task, tmpdir))
checkpoint_session = None
for _ in range(200):
checkpoint_session = await store.get_external_session(
"codex", "proj1", task_id=task.id
)
if (
checkpoint_session is not None
and checkpoint_session.status == "working"
and checkpoint_session.metadata.get("resume_session_id")
== "thread-cleanup-race"
):
break
await asyncio.sleep(0.01)
assert checkpoint_session is not None
self.assertEqual(checkpoint_session.status, "working")
await asyncio.wait_for(cleanup_started.wait(), timeout=5)
run_task.cancel()
await asyncio.sleep(0)
allow_cleanup.set()
with self.assertRaises(asyncio.CancelledError):
await run_task
terminal_session = await store.get_external_session(
"codex", "proj1", task_id=task.id
)
assert terminal_session is not None
self.assertEqual(terminal_session.status, "failed")
self.assertGreater(
terminal_session.updated_at,
checkpoint_session.updated_at,
)
task.metadata.update({
"work_item_runtime": True,
"external_resume_session_id": "thread-cleanup-race",
"external_resume_agent_type": "codex",
"external_resume_session_scope_id": "cleanup-race-parent",
"external_resume_checkpoint_session_updated_at": (
checkpoint_session.updated_at.isoformat()
),
"external_resume_checkpoint_session_status": "working",
})
engine = OPCEngine()
engine.project_id = "proj1"
engine.store = store
resume_adapter, _resume_metadata = (
await engine._configure_external_adapter_for_task(
task,
CodexAdapter(),
)
)
self.assertEqual(resume_adapter.config.session_mode, "new")
self.assertEqual(resume_adapter.config.session_id, "")
self.assertEqual(
task.metadata.get("external_resume_fallback"),
"context_replay_provider_terminal",
)
await store.close()
async def test_silent_external_agent_times_out_with_reason(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
store = _SessionStoreStub()
tests/test_external_session_continuity.py
+72
View File
@@ -28,6 +28,7 @@ from opc.core.models import (
ApprovalAction,
ApprovalDecision,
DelegationRoleSession,
ExternalSession,
RiskLevel,
Task,
TaskResult,
@@ -358,6 +359,77 @@ class BrokerRestorePrefersRoleStateTests(unittest.IsolatedAsyncioTestCase):
self.assertNotEqual(adapter.config.session_id, "thread-codex-only")
async def test_restore_never_treats_synthetic_monitor_identity_as_provider_token(self) -> None:
synthetic_id = "codex:proj1:task-new"
await self.store.update_role_session_adapter_state(
self.role_session_id,
"codex",
{
"resume_session_id": synthetic_id,
"provider_session_id": synthetic_id,
},
)
await self.store.save_external_session(
ExternalSession(
agent_type="codex",
project_id="proj1",
session_id=synthetic_id,
opc_session_id=self.role_session_id,
task_id="task-new",
workspace_path="/tmp/ws",
run_mode="exec",
status="working",
metadata={},
)
)
adapter = _MiniAdapter(agent_type="codex", can_resume_blank=False)
task = self._task()
await self.broker._restore_session_resume_from_store(adapter, task)
self.assertNotEqual(adapter.config.session_mode, "resume")
self.assertEqual(adapter.config.session_id, "")
self.assertNotIn("external_resume_session_id", task.metadata)
async def test_restore_rejects_failed_early_provider_stream_token(self) -> None:
token = "thread-failed"
await self.store.update_role_session_adapter_state(
self.role_session_id,
"codex",
{
"resume_session_id": token,
"provider_session_id": token,
"last_task_id": "task-new",
"source": "provider_stream",
"status": "working",
},
)
await self.store.save_external_session(ExternalSession(
agent_type="codex",
project_id="proj1",
session_id=token,
opc_session_id=self.role_session_id,
task_id="task-new",
workspace_path="/tmp/ws",
run_mode="exec",
status="failed",
metadata={
"resume_session_id": token,
"provider_session_id": token,
},
))
adapter = _MiniAdapter(agent_type="codex", can_resume_blank=False)
task = self._task()
await self.broker._restore_session_resume_from_store(adapter, task)
self.assertNotEqual(adapter.config.session_mode, "resume")
self.assertEqual(adapter.config.session_id, "")
self.assertIsNone(await self.store.get_role_session_adapter_state(
self.role_session_id,
"codex",
))
if __name__ == "__main__":
unittest.main()