fix: unify company runtime recovery lifecycle

This commit is contained in:
LZH-YS1998
2026-07-14 14:35:43 +08:00
parent 5e02364eb4
commit b8202bbe9e
56 changed files with 8753 additions and 3542 deletions
+264 -50
View File
@@ -1116,6 +1116,34 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(calls, ["first", "second"])
self.assertIn("Queued #1", console.export_text())
def test_chat_turn_controller_prepares_checkpoint_before_cancelling_active_turn(self) -> None:
state, _engine = self._make_state()
async def _run() -> list[str]:
order: list[str] = []
started = asyncio.Event()
async def active_turn() -> None:
started.set()
try:
await asyncio.Event().wait()
finally:
order.append("cancelled")
async def prepare() -> list[dict[str, Any]]:
order.append("prepared")
return []
state.engine.prepare_active_company_runtimes_for_shutdown = prepare
controller = ChatTurnController(state)
controller.active_task = asyncio.create_task(active_turn())
await started.wait()
await controller.shutdown()
return order
self.assertEqual(asyncio.run(_run()), ["prepared", "cancelled"])
def test_busy_slash_policy_allows_readonly_and_blocks_mutating_commands(self) -> None:
self.assertEqual(_busy_slash_policy("kanban", []), BusyCommandPolicy.IMMEDIATE_READONLY)
self.assertEqual(_busy_slash_policy("logs", ["task-1"]), BusyCommandPolicy.IMMEDIATE_READONLY)
@@ -1168,6 +1196,17 @@ class CliSlashCommandTests(unittest.TestCase):
"org_id": "quantum_harbor",
"preferred_agent": "codex",
})
store.checkpoints = [
SimpleNamespace(
checkpoint_id="cp-org-interrupted",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id="task-1",
session_id="sess-1",
updated_at=datetime(2026, 5, 17, 12, 0),
payload={},
)
]
state, engine = self._make_state(store=store)
state.mode = "company"
state.company_profile = "corporate"
@@ -1183,7 +1222,59 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(engine.calls[-1]["org_id"], "quantum_harbor")
self.assertIsNone(engine.calls[-1]["company_profile"])
self.assertEqual(engine.calls[-1]["preferred_agent"], "codex")
self.assertEqual(engine.calls[-1]["message_metadata"], {"ui_force_resume": True})
self.assertEqual(engine.calls[-1]["message_metadata"], {
"ui_force_resume": True,
"response_to_checkpoint_id": "cp-org-interrupted",
"response_to_checkpoint_type": "company_runtime_interrupted",
})
def test_company_continue_requires_a_durable_runtime_checkpoint(self) -> None:
console = Console(record=True, force_terminal=False, width=120)
store = self._Store()
store.tasks[0].metadata.update({
"exec_mode": "company",
"company_profile": "corporate",
})
state, engine = self._make_state(store=store)
async def _run() -> None:
await _handle_chat_slash_command(state, "/continue")
with patch("opc.cli.app.console", console):
asyncio.run(_run())
self.assertEqual(engine.calls, [])
self.assertIn("No suspended or interrupted company runtime", console.export_text())
def test_continue_slash_routes_to_durable_runtime_checkpoint(self) -> None:
store = self._Store()
store.checkpoints = [
SimpleNamespace(
checkpoint_id="cp-interrupted",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id="task-1",
session_id="sess-1",
updated_at=datetime(2026, 5, 17, 12, 0),
payload={},
)
]
state, engine = self._make_state(store=store)
state.mode = "company"
state.runtime_control_state = "suspended"
async def _run() -> None:
await _handle_chat_slash_command(state, "/continue")
asyncio.run(_run())
self.assertEqual(engine.calls[-1]["session_id"], "sess-1")
self.assertEqual(engine.calls[-1]["message_metadata"], {
"ui_force_resume": True,
"response_to_checkpoint_id": "cp-interrupted",
"response_to_checkpoint_type": "company_runtime_interrupted",
})
self.assertEqual(state.runtime_control_checkpoint_id, "cp-interrupted")
def test_plain_message_after_stop_routes_to_suspend_checkpoint(self) -> None:
store = self._Store()
@@ -1209,7 +1300,127 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(engine.calls[-1]["message_metadata"]["response_to_checkpoint_id"], "cp-suspend")
self.assertEqual(engine.calls[-1]["message_metadata"]["response_to_checkpoint_type"], "company_runtime_suspended")
self.assertEqual(state.runtime_control_state, "running")
self.assertEqual(state.runtime_control_state, "suspended")
def test_plain_message_from_company_child_uses_root_runtime_checkpoint(self) -> None:
store = self._Store()
now = datetime(2026, 5, 17, 12, 0)
store.tasks[0].metadata.update({
"exec_mode": "company",
"mode": "company",
"company_profile": "corporate",
})
store.tasks[0].parent_session_id = ""
store.tasks.append(SimpleNamespace(
id="worker-child",
title="Worker",
description="Worker turn",
status=TaskStatus.BLOCKED,
priority=3,
assigned_to="worker",
session_id="sess-1:role:worker",
parent_session_id="sess-1",
project_id="demo",
created_at=now,
tags=[],
result={},
linked_work_item_id="worker-item",
metadata={
"exec_mode": "company",
"work_item_runtime": True,
"work_item_projection_id": "worker",
"company_runtime_root_session_id": "sess-1",
},
context_snapshot={},
))
store.checkpoints = [SimpleNamespace(
checkpoint_id="cp-child-interrupted",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id="task-1",
session_id="sess-1",
project_id="demo",
updated_at=now,
payload={"parent_session_id": "sess-1"},
)]
state, engine = self._make_state(store=store)
state.session_id = "sess-1:role:worker"
# The selected child channel may have left the CLI's ambient mode in
# task mode. Runtime config, not ambient state, owns resumed execution.
state.mode = "task"
asyncio.run(_process_interactive_chat_message(state, "revise and continue"))
call = engine.calls[-1]
self.assertEqual(call["session_id"], "sess-1")
self.assertEqual(call["origin_task_id"], "task-1")
self.assertEqual(call["mode"], "company")
self.assertEqual(call["company_profile"], "corporate")
self.assertEqual(call["message_metadata"], {
"response_to_checkpoint_id": "cp-child-interrupted",
"response_to_checkpoint_type": "company_runtime_interrupted",
})
def test_plain_message_during_resuming_checkpoint_fails_closed(self) -> None:
console = Console(record=True, force_terminal=False, width=140)
store = self._Store()
store.tasks[0].metadata.update({
"exec_mode": "company",
"mode": "company",
"company_profile": "corporate",
})
store.checkpoints = [SimpleNamespace(
checkpoint_id="cp-resuming",
checkpoint_type="company_runtime_suspended",
status="resuming",
task_id="task-1",
session_id="sess-1",
project_id="demo",
updated_at=datetime(2026, 5, 17, 12, 0),
payload={"parent_session_id": "sess-1"},
)]
state, engine = self._make_state(store=store)
state.mode = "company"
with patch("opc.cli.app.console", console):
asyncio.run(_process_interactive_chat_message(state, "continue again"))
self.assertEqual(engine.calls, [])
self.assertIn("checkpoint is resuming", console.export_text())
def test_explicit_cli_runtime_checkpoint_mismatch_fails_closed(self) -> None:
console = Console(record=True, force_terminal=False, width=140)
store = self._Store()
store.tasks[0].metadata.update({
"exec_mode": "company",
"mode": "company",
"company_profile": "corporate",
})
store.checkpoints = [SimpleNamespace(
checkpoint_id="cp-current",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id="task-1",
session_id="sess-1",
project_id="demo",
updated_at=datetime(2026, 5, 17, 12, 0),
payload={"parent_session_id": "sess-1"},
)]
state, engine = self._make_state(store=store)
state.mode = "company"
with patch("opc.cli.app.console", console):
asyncio.run(_process_interactive_chat_message(
state,
"continue stale",
message_metadata={
"response_to_checkpoint_id": "cp-stale",
"response_to_checkpoint_type": "company_runtime_interrupted",
},
))
self.assertEqual(engine.calls, [])
self.assertIn("checkpoint identity mismatch", console.export_text())
def test_session_resume_still_switches_session(self) -> None:
state, engine = self._make_state(store=self._Store())
@@ -1287,8 +1498,6 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(output["payload"]["checkpoint_id"], "cp-stop")
self.assertEqual(output["cancelled"], ["task-company"])
self.assertEqual(output["suspend_calls"][0]["session_id"], "sess-company")
self.assertEqual(output["task"].metadata["dispatch_hold"], "company_runtime_suspended")
self.assertEqual(output["task"].metadata["company_runtime_stop_state"], "suspended")
def test_session_service_continue_uses_force_resume_metadata(self) -> None:
class Store:
@@ -1305,10 +1514,17 @@ class CliSlashCommandTests(unittest.TestCase):
metadata={
"exec_mode": "company",
"company_profile": "corporate",
"dispatch_hold": "company_runtime_suspended",
"company_runtime_stop_state": "suspended",
},
)
self.checkpoint = SimpleNamespace(
checkpoint_id="cp-company",
checkpoint_type="company_runtime_suspended",
status="pending",
project_id="demo",
session_id="sess-company",
task_id="task-company",
payload={},
)
async def get_task(self, task_id: str):
return self.task if task_id == self.task.id else None
@@ -1319,6 +1535,9 @@ class CliSlashCommandTests(unittest.TestCase):
async def get_session(self, session_id: str):
return SimpleNamespace(session_id=session_id, project_id="demo")
async def get_execution_checkpoints(self, **_kwargs):
return [self.checkpoint]
async def save_task(self, task):
self.task = task
@@ -1331,7 +1550,12 @@ class CliSlashCommandTests(unittest.TestCase):
engine = SimpleNamespace(project_id="demo", store=Store(), process_message=process_message)
context = OfficeServiceContext(engine=engine, agent_store=None, chat_store=None, event_adapter=None)
result = await SessionService(context).continue_run(project_id="demo", target="sess-company")
result = await SessionService(context).continue_run(
project_id="demo",
target="sess-company",
runtime_session_id="sess-company",
checkpoint_id="cp-company",
)
self.assertEqual(result.payload["response"], "resumed")
return calls
@@ -1340,7 +1564,11 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(calls[-1]["content"], "Resume the existing runtime.")
self.assertEqual(calls[-1]["session_id"], "sess-company")
self.assertEqual(calls[-1]["mode"], "company")
self.assertEqual(calls[-1]["message_metadata"], {"ui_force_resume": True})
self.assertEqual(calls[-1]["message_metadata"], {
"ui_force_resume": True,
"response_to_checkpoint_id": "cp-company",
"response_to_checkpoint_type": "company_runtime_suspended",
})
def test_session_service_continue_preserves_custom_org_id(self) -> None:
class Store:
@@ -1359,10 +1587,17 @@ class CliSlashCommandTests(unittest.TestCase):
"company_profile": "custom",
"org_id": "quantum_harbor",
"preferred_agent": "codex",
"dispatch_hold": "company_runtime_suspended",
"company_runtime_stop_state": "suspended",
},
)
self.checkpoint = SimpleNamespace(
checkpoint_id="cp-org",
checkpoint_type="company_runtime_interrupted",
status="pending",
project_id="demo",
session_id="sess-org",
task_id="task-org",
payload={},
)
async def get_task(self, task_id: str):
return self.task if task_id == self.task.id else None
@@ -1373,6 +1608,9 @@ class CliSlashCommandTests(unittest.TestCase):
async def get_session(self, session_id: str):
return SimpleNamespace(session_id=session_id, project_id="demo")
async def get_execution_checkpoints(self, **_kwargs):
return [self.checkpoint]
async def save_task(self, task):
self.task = task
@@ -1385,7 +1623,12 @@ class CliSlashCommandTests(unittest.TestCase):
engine = SimpleNamespace(project_id="demo", store=Store(), process_message=process_message)
context = OfficeServiceContext(engine=engine, agent_store=None, chat_store=None, event_adapter=None)
result = await SessionService(context).continue_run(project_id="demo", target="sess-org")
result = await SessionService(context).continue_run(
project_id="demo",
target="sess-org",
runtime_session_id="sess-org",
checkpoint_id="cp-org",
)
self.assertEqual(result.payload["response"], "resumed")
return calls
@@ -1394,7 +1637,11 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(calls[-1]["mode"], "org")
self.assertEqual(calls[-1]["org_id"], "quantum_harbor")
self.assertIsNone(calls[-1]["company_profile"])
self.assertEqual(calls[-1]["message_metadata"], {"ui_force_resume": True})
self.assertEqual(calls[-1]["message_metadata"], {
"ui_force_resume": True,
"response_to_checkpoint_id": "cp-org",
"response_to_checkpoint_type": "company_runtime_interrupted",
})
def test_queue_slash_lists_and_drops_queued_prompts(self) -> None:
console = Console(record=True, force_terminal=False, width=160)
@@ -2648,49 +2895,16 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertIn("Pending Checkpoints", rendered)
self.assertIn("cp-1", rendered)
def test_recover_slash_lists_and_resumes_interrupted_runtime(self) -> None:
def test_legacy_recover_slash_is_not_registered(self) -> None:
console = Console(record=True, force_terminal=False, width=200)
state, _engine = self._make_state(store=self._Store())
resume_calls: list[str] = []
class _FakeRecoveryManager:
async def get_status(self):
return SimpleNamespace(
interrupted=[
SimpleNamespace(
parent_task_id="parent-task",
parent_session_id="parent-session",
title="Interrupted Runtime",
profile="corporate",
interrupted_at="2026-05-03T12:00:00",
work_items=[
SimpleNamespace(projection_id="wi-1", interrupted=True),
SimpleNamespace(projection_id="wi-2", interrupted=False),
],
)
],
active_recoveries=[],
)
async def resume(self, parent_task_id: str):
resume_calls.append(parent_task_id)
if parent_task_id == "parent-task":
return {"ok": True, "resumed_work_item_projection_ids": ["wi-1"]}
return {"ok": False, "error": "not_found"}
async def _run() -> None:
await _handle_chat_slash_command(state, "/recover")
await _handle_chat_slash_command(state, "/recover resume parent-task")
await _handle_chat_slash_command(state, "/recover resume cp-1")
with patch("opc.cli.app.console", console), patch("opc.cli.app._get_chat_recovery_manager", return_value=_FakeRecoveryManager()):
asyncio.run(_run())
with patch("opc.cli.app.console", console):
asyncio.run(_handle_chat_slash_command(state, "/recover"))
rendered = console.export_text()
self.assertIn("Interrupted Runtime", rendered)
self.assertIn("Recovery started for parent-task", rendered)
self.assertIn("Checkpoint cp-1 is not resumed directly", rendered)
self.assertEqual(resume_calls, ["parent-task", "cp-1"])
self.assertIn("Unknown command: /recover", rendered)
self.assertNotIn("Interrupted Company Runtimes", rendered)
def test_logs_slash_renders_task_and_session_runtime_details(self) -> None:
console = Console(record=True, force_terminal=False, width=220)