Gitinbox/OpenOPC
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: preserve company resume control and agent identity

Browse Source
This commit is contained in:
LZH-YS1998
2026-07-14 16:59:24 +08:00
parent b8202bbe9e
commit 297295a4aa
12 changed files with 2459 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_company_runtime_identity.py
+5
View File
@@ -250,6 +250,11 @@ def test_work_item_chat_resume_uses_canonical_ui_anchor_as_engine_origin() -> No
assert call.kwargs["session_id"] == "runtime-session"
assert call.kwargs["origin_task_id"] == "ui-anchor"
assert handler._session_to_task["runtime-session"] == "ui-anchor"
handler._set_company_runtime_control.assert_awaited_once_with(
target,
state="resuming",
checkpoint_id="checkpoint-1",
)
handler.on_kanban_changed.assert_awaited_once_with(engine=run_engine)
asyncio.run(scenario())
tests/test_company_runtime_suspend_resume.py
+630
View File
@@ -1,9 +1,13 @@
from __future__ import annotations
import asyncio
import tempfile
import unittest
from datetime import datetime, timedelta
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock
from opc.core.models import (
CompanyMemberSession,
@@ -13,6 +17,7 @@ from opc.core.models import (
ExternalSession,
Phase,
Task,
TaskResult,
TaskStatus,
)
from opc.database.store import OPCStore
@@ -87,6 +92,15 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
},
)
)
await store.save_delegation_role_session(
DelegationRoleSession(
role_session_id=role_session_id,
run_id="run-1",
project_id="proj1",
role_id="executor",
seat_id="seat-1",
)
)
await store.save_delegation_work_item(
DelegationWorkItem(
work_item_id=work_item_id,
@@ -262,6 +276,603 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(refreshed_item.metadata.get("dispatch_hold"), "")
self.assertEqual(refreshed_item.claimed_by_role_runtime_session_id, "")
async def test_second_stop_during_resumed_execution_restores_pending_checkpoint(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
engine = self._engine(store)
first_stop = await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
self.assertIsNotNone(first_stop)
execution_started = asyncio.Event()
class BlockingCompanyExecutor:
def __init__(self) -> None:
self._notify_kanban_changed = AsyncMock()
async def execute(
self,
_plan: CompanyWorkItemRuntimePlan,
_tasks: list[Task],
) -> str:
execution_started.set()
await asyncio.Event().wait()
return "unreachable"
executor = BlockingCompanyExecutor()
engine.company_executor = executor # type: ignore[assignment]
resume_task = asyncio.create_task(engine._maybe_resume_checkpoint(
"continue",
"sess-parent",
reply_metadata={"ui_force_resume": True},
))
await asyncio.wait_for(execution_started.wait(), timeout=1)
executor._notify_kanban_changed.assert_awaited_once_with()
self.assertTrue(engine._active_task_run_registry.is_active("proj1", task.id))
second_stop = await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
self.assertIsNotNone(second_stop)
self.assertEqual(
second_stop["checkpoint_id"],
first_stop["checkpoint_id"],
)
resume_task.cancel()
with self.assertRaises(asyncio.CancelledError):
await resume_task
pending = await store.get_execution_checkpoints(
project_id="proj1",
session_id="sess-parent",
checkpoint_types=["company_runtime_suspended"],
statuses=["pending"],
)
refreshed_task = await store.get_task(task.id)
refreshed_item = await store.get_delegation_work_item("work-item-1")
self.assertEqual(len(pending), 1)
self.assertEqual(pending[0].checkpoint_id, first_stop["checkpoint_id"])
self.assertEqual(pending[0].payload.get("resume_state"), "interrupted")
assert refreshed_task is not None
assert refreshed_item is not None
self.assertEqual(
refreshed_task.metadata.get("dispatch_hold"),
"company_runtime_suspended",
)
self.assertEqual(
refreshed_item.metadata.get("dispatch_hold"),
"company_runtime_suspended",
)
self.assertFalse(engine._active_task_run_registry.is_active("proj1", task.id))
async def test_resume_attempt_pins_unlocked_auto_external_agent_without_permanent_lock(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
task.assigned_external_agent = "opencode"
task.metadata.update({
"delegation_seat_id": "seat-1",
"employee_assignment": {"employee_id": "employee-executor"},
"selected_execution_agent": "codex",
"selected_execution_agent_source": "fallback_rules",
"execution_agent_locked": False,
"preferred_external_agent": "codex",
"agent_selection": {
"selected": "opencode",
"selection_source": "llm",
},
})
await store.save_task(task)
engine = self._engine(store)
engine.org_engine = SimpleNamespace()
engine._available_external_agents = lambda: ["opencode", "codex"]
adaptive_selector = AsyncMock(return_value=(
"codex",
{
"selected": "codex",
"selection_source": "fallback_rules",
},
))
engine._select_task_execution_agent_via_llm = adaptive_selector
await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
checkpoint = (
await store.get_pending_checkpoints(
project_id="proj1",
session_id="sess-parent",
checkpoint_types=["company_runtime_suspended"],
)
)[0]
checkpoint_identity = checkpoint.payload["task_snapshots"][0][
"execution_identity"
]
self.assertEqual(checkpoint_identity["selected_execution_agent"], "opencode")
self.assertEqual(checkpoint_identity["assigned_external_agent"], "opencode")
self.assertEqual(checkpoint_identity["agent_selection_source"], "llm")
captured: dict[str, Any] = {}
class SelectingExecutor:
async def execute(self, plan: CompanyWorkItemRuntimePlan, tasks: list[Task]) -> str:
resumed = tasks[0]
captured["selected"] = await engine._assign_task_execution_agent(
resumed,
role=SimpleNamespace(),
)
captured["task"] = resumed
return "runtime resumed"
engine.company_executor = SelectingExecutor()
await engine._maybe_resume_checkpoint(
"continue",
"sess-parent",
reply_metadata={"ui_force_resume": True},
)
resumed_task = captured["task"]
self.assertEqual(captured["selected"], "opencode")
self.assertEqual(resumed_task.assigned_external_agent, "opencode")
self.assertEqual(resumed_task.assigned_to, "executor")
self.assertEqual(resumed_task.metadata["delegation_seat_id"], "seat-1")
self.assertEqual(
resumed_task.metadata["employee_assignment"]["employee_id"],
"employee-executor",
)
self.assertFalse(resumed_task.metadata["execution_agent_locked"])
self.assertNotIn(
"_company_runtime_resume_execution_agent_pin",
resumed_task.metadata,
)
self.assertEqual(
resumed_task.metadata["agent_selection"]["selection_source"],
"company_runtime_resume_checkpoint",
)
adaptive_selector.assert_not_awaited()
# A later ordinary dispatch has no checkpoint pin and may adapt again.
selected_after_resume = await engine._assign_task_execution_agent(
resumed_task,
role=SimpleNamespace(),
)
self.assertEqual(selected_after_resume, "codex")
adaptive_selector.assert_awaited_once()
async def test_resume_attempt_pins_native_without_running_adaptive_selector(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
task.assigned_external_agent = None
task.metadata.update({
"selected_execution_agent": "native",
"selected_execution_agent_source": "fallback_rules",
"execution_agent_locked": False,
})
await store.save_task(task)
engine = self._engine(store)
engine.org_engine = SimpleNamespace()
engine._available_external_agents = lambda: ["opencode", "codex"]
adaptive_selector = AsyncMock(return_value=(
"opencode",
{"selected": "opencode", "selection_source": "fallback_rules"},
))
engine._select_task_execution_agent_via_llm = adaptive_selector
await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
captured: dict[str, Any] = {}
class SelectingExecutor:
async def execute(self, plan: CompanyWorkItemRuntimePlan, tasks: list[Task]) -> str:
captured["selected"] = await engine._assign_task_execution_agent(
tasks[0],
role=SimpleNamespace(),
)
captured["task"] = tasks[0]
return "runtime resumed"
engine.company_executor = SelectingExecutor()
await engine._maybe_resume_checkpoint(
"continue",
"sess-parent",
reply_metadata={"ui_force_resume": True},
)
self.assertIsNone(captured["selected"])
self.assertIsNone(captured["task"].assigned_external_agent)
self.assertEqual(
captured["task"].metadata["selected_execution_agent"],
"native",
)
adaptive_selector.assert_not_awaited()
async def test_resume_identity_mismatch_fails_closed_and_returns_checkpoint_to_pending(self) -> None:
mutations = {
"role_id": lambda item: setattr(item, "role_id", "replacement-role"),
"seat_id": lambda item: setattr(item, "seat_id", "replacement-seat"),
"role_runtime_session_id": lambda item: setattr(
item,
"role_runtime_session_id",
"replacement-role-session",
),
"employee_id": lambda item: item.metadata.update({
"employee_assignment": {"employee_id": "replacement-employee"},
}),
}
for field_name, mutate in mutations.items():
with self.subTest(field_name=field_name):
store = await self._store()
_, task = await self._seed_runtime(store)
task.metadata.update({
"delegation_seat_id": "seat-1",
"employee_assignment": {"employee_id": "employee-executor"},
"selected_execution_agent": "codex",
"selected_execution_agent_source": "fallback_rules",
"execution_agent_locked": False,
})
await store.save_task(task)
work_item = await store.get_delegation_work_item("work-item-1")
assert work_item is not None
work_item.metadata = {
**dict(work_item.metadata or {}),
"employee_assignment": {"employee_id": "employee-executor"},
}
await store.save_delegation_work_item(work_item)
engine = self._engine(store)
await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
mutate(work_item)
await store.save_delegation_work_item(work_item)
executor = self._CapturingCompanyExecutor()
engine.company_executor = executor
with self.assertRaisesRegex(RuntimeError, field_name):
await engine._maybe_resume_checkpoint(
"continue",
"sess-parent",
reply_metadata={"ui_force_resume": True},
)
pending = await store.get_execution_checkpoints(
project_id="proj1",
session_id="sess-parent",
checkpoint_types=["company_runtime_suspended"],
statuses=["pending"],
)
resuming = await store.get_execution_checkpoints(
project_id="proj1",
session_id="sess-parent",
checkpoint_types=["company_runtime_suspended"],
statuses=["resuming"],
)
self.assertEqual(len(pending), 1)
self.assertEqual(resuming, [])
self.assertEqual(
pending[0].payload.get("resume_state"),
"failed_before_handoff",
)
self.assertEqual(executor.calls, [])
async def test_resume_repairs_stale_task_projection_from_work_item_and_role_session(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
task.metadata.update({
"delegation_seat_id": "seat-1",
"employee_assignment": {"employee_id": "employee-executor"},
"selected_execution_agent": "codex",
})
await store.save_task(task)
work_item = await store.get_delegation_work_item("work-item-1")
role_session = await store.get_delegation_role_session("role-runtime-1")
assert work_item is not None and role_session is not None
work_item.metadata = {
**dict(work_item.metadata or {}),
"employee_assignment": {"employee_id": "employee-executor"},
}
role_session.employee_id = "employee-executor"
await store.save_delegation_work_item(work_item)
await store.save_delegation_role_session(role_session)
engine = self._engine(store)
await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
stale_task = await store.get_task(task.id)
assert stale_task is not None
stale_task.assigned_to = "stale-role"
stale_task.metadata.update({
"delegation_seat_id": "stale-seat",
"delegation_role_session_id": "stale-role-session",
"employee_assignment": {"employee_id": "stale-employee"},
})
await store.save_task(stale_task)
captured: dict[str, Task] = {}
class CapturingExecutor:
async def execute(self, plan: CompanyWorkItemRuntimePlan, tasks: list[Task]) -> str:
captured["task"] = tasks[0]
return "resumed"
engine.company_executor = CapturingExecutor()
await engine._maybe_resume_checkpoint(
"continue",
"sess-parent",
reply_metadata={"ui_force_resume": True},
)
repaired = captured["task"]
self.assertEqual(repaired.assigned_to, "executor")
self.assertEqual(repaired.metadata["delegation_seat_id"], "seat-1")
self.assertEqual(
repaired.metadata["delegation_role_session_id"],
"role-runtime-1",
)
self.assertEqual(
repaired.metadata["employee_assignment"]["employee_id"],
"employee-executor",
)
async def test_resume_rejects_role_runtime_session_identity_mismatch(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
engine = self._engine(store)
await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
role_session = await store.get_delegation_role_session("role-runtime-1")
assert role_session is not None
role_session.role_id = "replacement-role"
await store.save_delegation_role_session(role_session)
engine.company_executor = self._CapturingCompanyExecutor()
with self.assertRaisesRegex(RuntimeError, "role session role_id"):
await engine._maybe_resume_checkpoint(
"continue",
"sess-parent",
reply_metadata={"ui_force_resume": True},
)
async def test_force_native_real_work_item_path_consumes_resume_pin(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
task.assigned_external_agent = None
task.metadata.update({
"force_native_execution": True,
"selected_execution_agent": "native",
"agent_selection": {
"selected": "native",
"selection_source": "forced_native",
},
})
await store.save_task(task)
engine = self._engine(store)
await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
checkpoint = (
await store.get_pending_checkpoints(
project_id="proj1",
session_id="sess-parent",
checkpoint_types=["company_runtime_suspended"],
)
)[0]
stored_task = await store.get_task(task.id)
assert stored_task is not None
resumed = (
await engine._prepare_company_runtime_tasks_for_resume(
[stored_task],
checkpoint.payload,
resume_task_ids={task.id},
)
)[0]
self.assertIn(
"_company_runtime_resume_execution_agent_pin",
resumed.metadata,
)
role = SimpleNamespace(role_id="executor", preferred_external_agent="codex")
class OrgEngine:
@staticmethod
def get_role_for_work_item(role_id: str, tags: list[str]) -> Any:
return role
engine.org_engine = OrgEngine()
executor = CompanyWorkItemExecutor(
org_engine=engine.org_engine,
communication=None,
approval_engine=SimpleNamespace(),
memory=None,
execute_task=AsyncMock(return_value=TaskResult(
status=TaskStatus.FAILED,
content="stop after selector",
)),
save_task=store.save_task,
store=store,
agent_selector=engine._assign_task_execution_agent,
)
result = await executor._run_work_item(resumed, {"execution": resumed})
self.assertEqual(result.status, TaskStatus.FAILED)
self.assertNotIn(
"_company_runtime_resume_execution_agent_pin",
resumed.metadata,
)
self.assertEqual(
resumed.metadata["agent_selection"]["selection_source"],
"company_runtime_resume_checkpoint",
)
self.assertIsNone(resumed.assigned_external_agent)
async def test_resume_does_not_overwrite_newer_durable_role_adapter_state(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
await store.update_role_session_adapter_state(
"role-runtime-1",
"codex",
{"resume_session_id": "thread-before-stop", "updated_at": "2026-01-01"},
)
engine = self._engine(store)
await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
await store.update_role_session_adapter_state(
"role-runtime-1",
"codex",
{"resume_session_id": "thread-after-checkpoint", "updated_at": "2026-07-14"},
)
engine.company_executor = self._CapturingCompanyExecutor()
await engine._maybe_resume_checkpoint(
"continue",
"sess-parent",
reply_metadata={"ui_force_resume": True},
)
state = await store.get_role_session_adapter_state(
"role-runtime-1",
"codex",
)
assert state is not None
self.assertEqual(state["resume_session_id"], "thread-after-checkpoint")
async def test_resume_fixed_backend_failure_never_tries_alternate_or_native(self) -> None:
engine = OPCEngine(project_id="proj1")
attempted_agents: list[str] = []
class Adapter:
def __init__(self, name: str) -> None:
self.agent_type = name
self.config = SimpleNamespace(
session_mode="auto",
run_mode="batch",
)
def supports_interactive(self) -> bool:
return False
def build_invocation(
self,
task: Task,
workspace_path: str | None = None,
) -> tuple[list[str], dict[str, Any]]:
return [self.agent_type], {
"agent": self.agent_type,
"command": self.agent_type,
}
codex = Adapter("codex")
opencode = Adapter("opencode")
class Registry:
def get_ordered_available(self) -> list[tuple[str, Adapter]]:
return [("opencode", opencode), ("codex", codex)]
def get(self, name: str) -> Adapter | None:
return {"codex": codex, "opencode": opencode}.get(name)
class Broker:
async def run(
self,
*,
adapter: Adapter,
task: Task,
workspace_path: str,
on_progress: Any = None,
prepared_task: Task | None = None,
) -> TaskResult:
attempted_agents.append(adapter.agent_type)
return TaskResult(
status=TaskStatus.FAILED,
content=f"{adapter.agent_type} failed",
artifacts={},
)
engine.adapter_registry = Registry()
engine.external_broker = Broker()
engine._resolve_external_workspace = lambda task: "/tmp/workspace"
engine._build_external_agent_task = AsyncMock(side_effect=lambda task: task)
engine._configure_external_adapter_for_task = AsyncMock(
side_effect=lambda task, adapter: (adapter, {}),
)
engine._emit_external_agent_audit = AsyncMock()
engine._run_native_agent = AsyncMock(return_value=TaskResult(
status=TaskStatus.DONE,
content="native fallback",
))
task = Task(
id="resume-fixed-backend",
title="Resume fixed backend",
project_id="proj1",
assigned_external_agent="codex",
metadata={
"target_output_dir": "/tmp/workspace",
"agent_selection": {
"selection_source": "company_runtime_resume_checkpoint",
},
},
)
result = await engine._run_task_once(task)
self.assertEqual(result.status, TaskStatus.FAILED)
self.assertEqual(attempted_agents, ["codex"])
engine._run_native_agent.assert_not_awaited()
async def test_suspend_prefers_provider_token_over_newer_synthetic_monitor_row(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
await store.save_external_session(ExternalSession(
agent_type="codex",
project_id="proj1",
session_id="codex:proj1:execution-task",
opc_session_id="role-runtime-1",
task_id=task.id,
workspace_path="/tmp/opc-test",
run_mode="interactive",
status="working",
metadata={},
updated_at=datetime.now() + timedelta(seconds=1),
))
engine = self._engine(store)
await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
checkpoint = (
await store.get_pending_checkpoints(
project_id="proj1",
session_id="sess-parent",
checkpoint_types=["company_runtime_suspended"],
)
)[0]
external = checkpoint.payload["external_sessions"][task.id]
self.assertEqual(external["session_id"], "provider-session-1")
self.assertEqual(external["resume_session_id"], "provider-session-1")
async def test_text_after_stop_routes_to_final_decider_instead_of_plain_resume(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
@@ -388,6 +999,7 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
run_id="run-1",
role_id="ceo",
seat_id="seat-ceo",
role_runtime_session_id="role-ceo",
title="CEO delivery",
kind="deliver",
projection_id="ceo-deliver",
@@ -403,6 +1015,7 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
run_id="run-1",
role_id="engineer",
seat_id="seat-engineer",
role_runtime_session_id="role-engineer",
title="Worker execution",
kind="execute",
projection_id="worker-execute",
@@ -415,6 +1028,18 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
)
await store.save_delegation_work_item(ceo_item)
await store.save_delegation_work_item(worker_item)
await store.save_delegation_role_session(DelegationRoleSession(
role_session_id="role-ceo",
run_id="run-1",
role_id="ceo",
seat_id="seat-ceo",
))
await store.save_delegation_role_session(DelegationRoleSession(
role_session_id="role-engineer",
run_id="run-1",
role_id="engineer",
seat_id="seat-engineer",
))
common_metadata = {
"company_profile": "corporate",
@@ -1455,6 +2080,11 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
)
resumed_task = captured["tasks"][0]
self.assertEqual(resumed_task.assigned_external_agent, "codex")
self.assertEqual(
resumed_task.metadata["selected_execution_agent"],
"codex",
)
self.assertNotIn("external_resume_session_id", resumed_task.metadata)
self.assertEqual(resumed_task.metadata["external_resume_fallback"], "context_replay")
tests/test_engine_runtime_recovery.py
+79 -1
View File
@@ -802,6 +802,24 @@ async def test_resume_candidates_use_nonterminal_work_item_over_terminal_task(
"phase": Phase.RUNNING.value,
}
],
"task_snapshots": [
{
"task_id": task.id,
"status": TaskStatus.DONE.value,
"assigned_to": "executor",
"assigned_external_agent": "",
"selected_execution_agent": "native",
"work_item_id": work_item.work_item_id,
"work_item": {
"work_item_id": work_item.work_item_id,
"phase": Phase.RUNNING.value,
"role_id": "executor",
"seat_id": "seat::executor",
"role_runtime_session_id": "",
"metadata": {},
},
}
],
},
resume_task_ids={task.id},
)
@@ -1062,7 +1080,9 @@ async def test_successful_checkpoint_handoff_reopens_cancelled_ui_anchor(
await store.save_execution_checkpoint(checkpoint)
engine = OPCEngine(project_id="project-a")
engine.store = store
engine.company_executor = SimpleNamespace()
engine.company_executor = SimpleNamespace(
_notify_kanban_changed=AsyncMock(),
)
engine._prepare_company_runtime_tasks_for_resume = AsyncMock(
return_value=[runtime_task]
)
@@ -1087,6 +1107,7 @@ async def test_successful_checkpoint_handoff_reopens_cancelled_ui_anchor(
handed_off_tasks, driver_ownership = handed_off
assert handed_off_tasks == [runtime_task]
assert driver_ownership is not None
engine.company_executor._notify_kanban_changed.assert_awaited_once_with()
driver_ownership.release()
assert still_cancelled is not None
assert still_cancelled.status == TaskStatus.CANCELLED
@@ -1110,6 +1131,63 @@ async def test_successful_checkpoint_handoff_reopens_cancelled_ui_anchor(
await store.close()
@_async_test
async def test_failed_checkpoint_handoff_does_not_publish_running_snapshot(
tmp_path: Path,
) -> None:
store = OPCStore(tmp_path / "tasks.db")
await store.initialize()
try:
runtime_task = _runtime_task(
task_id="runtime-task",
status=TaskStatus.BLOCKED,
)
await store.save_task(runtime_task)
checkpoint = ExecutionCheckpoint(
project_id="project-a",
session_id="root-session",
checkpoint_type="company_runtime_interrupted",
task_id=runtime_task.id,
payload={
"task_ids": [runtime_task.id],
"parent_session_id": "root-session",
},
)
await store.save_execution_checkpoint(checkpoint)
engine = OPCEngine(project_id="project-a")
engine.store = store
engine.company_executor = SimpleNamespace(
_notify_kanban_changed=AsyncMock(),
)
engine._prepare_company_runtime_tasks_for_resume = AsyncMock(
side_effect=RuntimeError("prepare failed"),
)
with pytest.raises(RuntimeError, match="prepare failed"):
await engine._handoff_company_suspend_checkpoint(
checkpoint,
payload=dict(checkpoint.payload),
parent_session_id="root-session",
tasks=[runtime_task],
)
engine.company_executor._notify_kanban_changed.assert_not_awaited()
assert not engine._active_task_run_registry.is_active(
"project-a",
runtime_task.id,
)
pending = await store.get_execution_checkpoints(
project_id="project-a",
session_id="root-session",
checkpoint_types=["company_runtime_interrupted"],
statuses=["pending"],
)
assert len(pending) == 1
assert pending[0].payload.get("resume_state") == "failed_before_handoff"
finally:
await store.close()
@_async_test
async def test_executor_failure_restores_pending_checkpoint_and_durable_holds(
tmp_path: Path,
tests/test_external_agent_monitoring.py
+354
View File
@@ -19,6 +19,7 @@ from opc.core.models import (
AgentStatus,
ApprovalAction,
ApprovalDecision,
DelegationRoleSession,
DelegationWorkItem,
ExecutionMode,
Phase,
@@ -32,6 +33,8 @@ from opc.engine import OPCEngine
from opc.layer1_perception.context_assembler import ContextAssembler, ExternalContextLayers
from opc.layer2_organization import comms as file_comms
from opc.layer2_organization.prompt_contract import make_prompt_contract
from opc.layer2_organization.company_mode import serialize_company_work_item_runtime_plan
from opc.layer2_organization.org_work_item_planner import CompanyWorkItemRuntimePlan
from opc.layer2_organization.work_item_links import set_linked_work_item_id
from opc.layer3_agent.adapters.claude_code import ClaudeCodeAdapter
from opc.layer3_agent.adapters.codex_adapter import CodexAdapter
@@ -548,6 +551,357 @@ class ExternalAgentMonitoringTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(session.metadata.get("resume_session_id"), "ses_1")
await store.close()
async def test_live_provider_thread_is_durable_before_stop_checkpoint_and_reused(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
store = OPCStore(Path(tmpdir) / "tasks.db")
await store.initialize()
role_session_id = "role-runtime::run-live::executor"
parent_session_id = "company-live-session"
await store.save_delegation_role_session(DelegationRoleSession(
role_session_id=role_session_id,
run_id="run-live",
role_id="executor",
seat_id="seat-live",
))
await store.save_task(Task(
id="ui-live",
title="Company chat",
session_id=parent_session_id,
project_id="proj1",
status=TaskStatus.IDLE,
metadata={"exec_mode": "company", "company_profile": "corporate"},
))
work_item = DelegationWorkItem(
work_item_id="work-live",
run_id="run-live",
role_id="executor",
seat_id="seat-live",
role_runtime_session_id=role_session_id,
title="Live work",
projection_id="live",
phase=Phase.RUNNING,
claimed_by_role_runtime_session_id=role_session_id,
claimed_by_seat_id="seat-live",
metadata={"work_item_projection_id": "live"},
)
await store.save_delegation_work_item(work_item)
plan = CompanyWorkItemRuntimePlan(
profile="corporate",
metadata={
"execution_model": "multi_team_org",
"runtime_model": "multi_team_org",
},
)
task = Task(
id="live-provider-task",
title="Live provider task",
session_id="live-child-session",
parent_session_id=parent_session_id,
project_id="proj1",
assigned_to="executor",
assigned_external_agent="script_agent",
status=TaskStatus.RUNNING,
metadata={
"work_item_runtime": True,
"work_item_projection_id": "live",
"delegation_run_id": "run-live",
"delegation_role_session_id": role_session_id,
"delegation_seat_id": "seat-live",
"selected_execution_agent": "script_agent",
"selected_execution_agent_source": "fallback_rules",
"company_profile": "corporate",
"execution_model": "multi_team_org",
"runtime_model": "multi_team_org",
"company_work_item_plan": serialize_company_work_item_runtime_plan(plan),
},
)
set_linked_work_item_id(task, work_item.work_item_id)
await store.save_task(task)
await store.link_work_item_runtime_task(work_item.work_item_id, task.id)
broker = ExternalAgentBroker(store, _ApprovalStub())
adapter = _ScriptAdapter(
"import json,sys,time\n"
"print(json.dumps({'sessionID': 'provider-live-thread'}))\n"
"sys.stdout.flush()\n"
"time.sleep(30)\n"
)
run_task = asyncio.create_task(broker.run(adapter, task, tmpdir))
role_state = None
for _ in range(200):
role_state = await store.get_role_session_adapter_state(
role_session_id,
"script_agent",
)
if role_state and role_state.get("resume_session_id"):
break
await asyncio.sleep(0.01)
self.assertIsNotNone(role_state)
assert role_state is not None
self.assertEqual(
role_state["resume_session_id"],
"provider-live-thread",
)
self.assertFalse(run_task.done())
engine = OPCEngine()
engine.project_id = "proj1"
engine.store = store
await engine.suspend_company_runtime(
origin_task_id="ui-live",
session_id=parent_session_id,
reason="user_stop",
)
checkpoint = (
await store.get_pending_checkpoints(
project_id="proj1",
session_id=parent_session_id,
checkpoint_types=["company_runtime_suspended"],
)
)[0]
captured = checkpoint.payload["external_sessions"][task.id]
self.assertEqual(
captured["resume_session_id"],
"provider-live-thread",
)
self.assertEqual(
captured["provider_session_id"],
"provider-live-thread",
)
run_task.cancel()
with self.assertRaises(asyncio.CancelledError):
await run_task
resumed_task = await store.get_task(task.id)
assert resumed_task is not None
resume_adapter = _ScriptAdapter("print('unused')")
resume_adapter.config.resume_session_flag = "--resume"
await broker._restore_session_resume_from_store(
resume_adapter,
resumed_task,
)
self.assertEqual(resume_adapter.config.session_mode, "resume")
self.assertEqual(
resume_adapter.config.session_id,
"provider-live-thread",
)
await store.close()
async def test_codex_thread_started_stream_restores_real_resume_argv(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
store = OPCStore(Path(tmpdir) / "tasks.db")
await store.initialize()
role_session_id = "role-runtime::codex-live::executor"
await store.save_delegation_role_session(DelegationRoleSession(
role_session_id=role_session_id,
run_id="codex-live",
project_id="proj1",
role_id="executor",
seat_id="seat-codex-live",
))
task = Task(
id="codex-live-task",
title="Codex live task",
description="Keep the same Codex thread.",
project_id="proj1",
session_id="codex-live-child",
parent_session_id="codex-live-parent",
assigned_to="executor",
assigned_external_agent="codex",
status=TaskStatus.RUNNING,
metadata={
"work_item_runtime": True,
"delegation_role_session_id": role_session_id,
"delegation_seat_id": "seat-codex-live",
},
)
await store.save_task(task)
class ScriptedCodexAdapter(CodexAdapter):
async def start_process(
self,
cmd: list[str],
workspace_path: str,
extra_env: dict[str, str] | None = None,
task: Task | None = None,
launch_metadata: dict[str, object] | None = None,
) -> asyncio.subprocess.Process:
return await asyncio.create_subprocess_exec(
sys.executable,
"-c",
(
"import json,sys,time\n"
"print(json.dumps({'type':'thread.started','thread_id':'thread-real-codex'}))\n"
"sys.stdout.flush()\n"
"time.sleep(30)\n"
),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=workspace_path,
)
broker = ExternalAgentBroker(store, _ApprovalStub())
live_adapter = ScriptedCodexAdapter()
live_adapter.config.run_mode = "exec"
run_task = asyncio.create_task(
broker.run(live_adapter, task, tmpdir)
)
state = None
for _ in range(200):
state = await store.get_role_session_adapter_state(
role_session_id,
"codex",
)
if state and state.get("resume_session_id"):
break
await asyncio.sleep(0.01)
assert state is not None
self.assertEqual(state["resume_session_id"], "thread-real-codex")
run_task.cancel()
with self.assertRaises(asyncio.CancelledError):
await run_task
resume_adapter = CodexAdapter()
resumed_task = await store.get_task(task.id)
assert resumed_task is not None
await broker._restore_session_resume_from_store(
resume_adapter,
resumed_task,
)
self.assertEqual(resume_adapter.config.session_mode, "resume")
self.assertEqual(resume_adapter.config.session_id, "thread-real-codex")
cmd, _metadata = resume_adapter.build_invocation(
resumed_task,
workspace_path=tmpdir,
)
self.assertEqual(cmd[1:3], ["exec", "resume"])
self.assertIn("thread-real-codex", cmd)
self.assertEqual(cmd[-1], "-")
await store.close()
async def test_cleanup_cancellation_terminalizes_checkpoint_provider_token(self) -> None:
"""A Stop checkpoint's working token cannot outlive a failed process."""
with tempfile.TemporaryDirectory() as tmpdir:
store = OPCStore(Path(tmpdir) / "tasks.db")
await store.initialize()
role_session_id = "role-runtime::cleanup-race::executor"
await store.save_delegation_role_session(DelegationRoleSession(
role_session_id=role_session_id,
run_id="cleanup-race",
project_id="proj1",
role_id="executor",
seat_id="seat-cleanup-race",
))
task = Task(
id="cleanup-race-task",
title="Cleanup cancellation race",
description="Do not resume a provider thread that later failed.",
project_id="proj1",
session_id="cleanup-race-child",
parent_session_id="cleanup-race-parent",
assigned_to="executor",
assigned_external_agent="codex",
status=TaskStatus.RUNNING,
metadata={"delegation_role_session_id": role_session_id},
)
await store.save_task(task)
cleanup_started = asyncio.Event()
allow_cleanup = asyncio.Event()
class CleanupRaceCodexAdapter(CodexAdapter):
async def start_process(
self,
cmd: list[str],
workspace_path: str,
extra_env: dict[str, str] | None = None,
task: Task | None = None,
launch_metadata: dict[str, object] | None = None,
) -> asyncio.subprocess.Process:
return await asyncio.create_subprocess_exec(
sys.executable,
"-c",
(
"import json,sys,time\n"
"print(json.dumps({'type':'thread.started','thread_id':'thread-cleanup-race'}))\n"
"sys.stdout.flush()\n"
"time.sleep(.2)\n"
"sys.exit(7)\n"
),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=workspace_path,
)
async def cleanup_process(self, proc: asyncio.subprocess.Process) -> None:
cleanup_started.set()
await allow_cleanup.wait()
await super().cleanup_process(proc)
broker = ExternalAgentBroker(store, _ApprovalStub())
live_adapter = CleanupRaceCodexAdapter()
run_task = asyncio.create_task(broker.run(live_adapter, task, tmpdir))
checkpoint_session = None
for _ in range(200):
checkpoint_session = await store.get_external_session(
"codex", "proj1", task_id=task.id
)
if (
checkpoint_session is not None
and checkpoint_session.status == "working"
and checkpoint_session.metadata.get("resume_session_id")
== "thread-cleanup-race"
):
break
await asyncio.sleep(0.01)
assert checkpoint_session is not None
self.assertEqual(checkpoint_session.status, "working")
await asyncio.wait_for(cleanup_started.wait(), timeout=5)
run_task.cancel()
await asyncio.sleep(0)
allow_cleanup.set()
with self.assertRaises(asyncio.CancelledError):
await run_task
terminal_session = await store.get_external_session(
"codex", "proj1", task_id=task.id
)
assert terminal_session is not None
self.assertEqual(terminal_session.status, "failed")
self.assertGreater(
terminal_session.updated_at,
checkpoint_session.updated_at,
)
task.metadata.update({
"work_item_runtime": True,
"external_resume_session_id": "thread-cleanup-race",
"external_resume_agent_type": "codex",
"external_resume_session_scope_id": "cleanup-race-parent",
"external_resume_checkpoint_session_updated_at": (
checkpoint_session.updated_at.isoformat()
),
"external_resume_checkpoint_session_status": "working",
})
engine = OPCEngine()
engine.project_id = "proj1"
engine.store = store
resume_adapter, _resume_metadata = (
await engine._configure_external_adapter_for_task(
task,
CodexAdapter(),
)
)
self.assertEqual(resume_adapter.config.session_mode, "new")
self.assertEqual(resume_adapter.config.session_id, "")
self.assertEqual(
task.metadata.get("external_resume_fallback"),
"context_replay_provider_terminal",
)
await store.close()
async def test_silent_external_agent_times_out_with_reason(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
store = _SessionStoreStub()
tests/test_external_session_continuity.py
+72
View File
@@ -28,6 +28,7 @@ from opc.core.models import (
ApprovalAction,
ApprovalDecision,
DelegationRoleSession,
ExternalSession,
RiskLevel,
Task,
TaskResult,
@@ -358,6 +359,77 @@ class BrokerRestorePrefersRoleStateTests(unittest.IsolatedAsyncioTestCase):
self.assertNotEqual(adapter.config.session_id, "thread-codex-only")
async def test_restore_never_treats_synthetic_monitor_identity_as_provider_token(self) -> None:
synthetic_id = "codex:proj1:task-new"
await self.store.update_role_session_adapter_state(
self.role_session_id,
"codex",
{
"resume_session_id": synthetic_id,
"provider_session_id": synthetic_id,
},
)
await self.store.save_external_session(
ExternalSession(
agent_type="codex",
project_id="proj1",
session_id=synthetic_id,
opc_session_id=self.role_session_id,
task_id="task-new",
workspace_path="/tmp/ws",
run_mode="exec",
status="working",
metadata={},
)
)
adapter = _MiniAdapter(agent_type="codex", can_resume_blank=False)
task = self._task()
await self.broker._restore_session_resume_from_store(adapter, task)
self.assertNotEqual(adapter.config.session_mode, "resume")
self.assertEqual(adapter.config.session_id, "")
self.assertNotIn("external_resume_session_id", task.metadata)
async def test_restore_rejects_failed_early_provider_stream_token(self) -> None:
token = "thread-failed"
await self.store.update_role_session_adapter_state(
self.role_session_id,
"codex",
{
"resume_session_id": token,
"provider_session_id": token,
"last_task_id": "task-new",
"source": "provider_stream",
"status": "working",
},
)
await self.store.save_external_session(ExternalSession(
agent_type="codex",
project_id="proj1",
session_id=token,
opc_session_id=self.role_session_id,
task_id="task-new",
workspace_path="/tmp/ws",
run_mode="exec",
status="failed",
metadata={
"resume_session_id": token,
"provider_session_id": token,
},
))
adapter = _MiniAdapter(agent_type="codex", can_resume_blank=False)
task = self._task()
await self.broker._restore_session_resume_from_store(adapter, task)
self.assertNotEqual(adapter.config.session_mode, "resume")
self.assertEqual(adapter.config.session_id, "")
self.assertIsNone(await self.store.get_role_session_adapter_state(
self.role_session_id,
"codex",
))
if __name__ == "__main__":
unittest.main()