fix: unify company runtime recovery lifecycle

This commit is contained in:
LZH-YS1998
2026-07-14 14:35:43 +08:00
parent 5e02364eb4
commit b8202bbe9e
56 changed files with 8753 additions and 3542 deletions
+549
View File
@@ -0,0 +1,549 @@
from __future__ import annotations
import asyncio
from functools import wraps
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from opc.core.active_task_runs import (
ActiveTaskRunAdmissionClosed,
ActiveTaskRunRegistry,
)
from opc.core.models import CompanyMemberSession, Task, TaskResult, TaskStatus
from opc.engine import OPCEngine
from opc.layer2_organization.company_mode import CompanyWorkItemExecutor
from opc.layer2_organization.company_runtime_identity import is_company_runtime_task
from opc.layer2_organization.org_work_item_planner import CompanyWorkItemRuntimePlan
def _async_test(func):
@wraps(func)
def runner(*args, **kwargs):
return asyncio.run(func(*args, **kwargs))
return runner
def test_overlapping_attempts_remain_active_until_last_attempt_exits() -> None:
registry = ActiveTaskRunRegistry()
first = registry.register("project-a", "task-1")
second = registry.register("project-a", "task-1")
assert first != second
assert registry.attempt_count("project-a", "task-1") == 2
assert registry.is_active("project-a", "task-1")
assert registry.unregister("project-a", "task-1", first)
assert registry.is_active("project-a", "task-1")
assert registry.unregister("project-a", "task-1", second)
assert not registry.is_active("project-a", "task-1")
def test_registry_isolates_projects_with_equal_task_ids() -> None:
registry = ActiveTaskRunRegistry()
token = registry.register("project-a", "task-1")
assert registry.active_task_ids("project-a") == {"task-1"}
assert registry.active_task_ids("project-b") == set()
assert not registry.is_active("project-b", "task-1")
assert registry.unregister("project-a", "task-1", token)
def test_plain_child_task_is_not_classified_as_company_runtime_scope() -> None:
task = Task(
id="plain-task",
title="Plain task",
project_id="project-a",
parent_session_id="parent-session",
metadata={"mode": "task", "parent_session_id": "parent-session"},
)
assert not is_company_runtime_task(task)
def test_closing_admission_preserves_existing_attempts_and_rejects_new_ones() -> None:
registry = ActiveTaskRunRegistry()
token = registry.register("project-a", "task-1")
registry.close_admission()
assert registry.admission_closed
assert registry.active_task_ids("project-a") == {"task-1"}
assert registry.is_active("project-a", "task-1")
with pytest.raises(ActiveTaskRunAdmissionClosed):
registry.register("project-a", "task-2")
assert registry.unregister("project-a", "task-1", token)
def test_closed_admission_allows_only_nested_live_driver_attempts() -> None:
registry = ActiveTaskRunRegistry()
driver_token = registry.register("project-a", "driver-task")
with registry.bind_driver_attempt(driver_token):
registry.close_admission()
nested_token = registry.register("project-a", "claimed-child")
assert registry.is_active("project-a", "claimed-child")
registry.unregister("project-a", "claimed-child", nested_token)
with pytest.raises(ActiveTaskRunAdmissionClosed):
registry.register("project-a", "new-ingress")
registry.unregister("project-a", "driver-task", driver_token)
@_async_test
async def test_shutdown_barrier_allows_only_reserved_handoff_to_register() -> None:
registry = ActiveTaskRunRegistry()
handoff_token = registry.reserve_handoff()
with registry.bind_handoff(handoff_token):
barrier = asyncio.create_task(
registry.close_admission_and_wait_for_handoffs()
)
await asyncio.sleep(0)
assert not barrier.done()
attempt_token = registry.register("project-a", "task-1")
registry.release_handoff(handoff_token)
await asyncio.wait_for(barrier, timeout=0.1)
# The handoff wait ends at real coroutine registration, not at the end of
# that execution attempt.
assert registry.is_active("project-a", "task-1")
assert registry.pending_handoff_count == 0
with pytest.raises(ActiveTaskRunAdmissionClosed):
registry.register("project-a", "late-task")
assert registry.unregister("project-a", "task-1", attempt_token)
@_async_test
async def test_shutdown_barrier_drains_request_that_exits_before_registration() -> None:
registry = ActiveTaskRunRegistry()
handoff_token = registry.reserve_handoff()
barrier = asyncio.create_task(registry.close_admission_and_wait_for_handoffs())
await asyncio.sleep(0)
assert not barrier.done()
assert registry.release_handoff(handoff_token)
await asyncio.wait_for(barrier, timeout=0.1)
assert registry.pending_handoff_count == 0
assert registry.active_task_ids("project-a") == set()
@_async_test
async def test_revoked_handoff_cannot_block_shutdown_or_register_late() -> None:
registry = ActiveTaskRunRegistry()
handoff_token = registry.reserve_handoff()
with registry.bind_handoff(handoff_token):
assert registry.retain_current_handoff() == handoff_token
registry.close_admission()
assert registry.revoke_handoff(handoff_token)
await asyncio.wait_for(
registry.close_admission_and_wait_for_handoffs(),
timeout=0.1,
)
with pytest.raises(ActiveTaskRunAdmissionClosed):
registry.register("project-a", "late-task")
assert registry.pending_handoff_count == 0
assert not registry.release_handoff(handoff_token)
@_async_test
async def test_engine_turns_closed_admission_into_infrastructure_cancellation() -> None:
registry = ActiveTaskRunRegistry()
registry.close_admission()
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
engine._run_task_once = AsyncMock()
task = Task(
id="late-task",
title="Late task",
project_id="project-a",
status=TaskStatus.PENDING,
)
with pytest.raises(asyncio.CancelledError):
await engine._execute_task(task)
engine._run_task_once.assert_not_awaited()
assert registry.active_task_ids("project-a") == set()
@_async_test
async def test_task_liveness_uses_registry_only() -> None:
registry = ActiveTaskRunRegistry()
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
engine.store = SimpleNamespace(get_latest_external_session_for_task=AsyncMock())
task = Task(
id="task-1",
title="Live task",
project_id="project-a",
status=TaskStatus.RUNNING,
)
assert not await engine._task_runtime_is_live(task)
engine.store.get_latest_external_session_for_task.assert_not_awaited()
token = registry.register("project-a", task.id)
assert await engine._task_runtime_is_live(task)
registry.unregister("project-a", task.id, token)
@_async_test
async def test_project_delegate_receives_controller_registry() -> None:
registry = ActiveTaskRunRegistry()
root = OPCEngine(project_id="project-a", active_task_run_registry=registry)
root._initialized = True
captured: dict[str, object] = {}
class FakeDelegate:
def __init__(self, **kwargs: object) -> None:
captured.update(kwargs)
self.store = None
async def initialize(self) -> None:
return None
with patch("opc.engine.OPCEngine", FakeDelegate):
delegate = await root._get_project_delegate("project-b")
assert delegate is root._project_engine_delegates["project-b"]
assert captured["active_task_run_registry"] is registry
assert captured["owns_active_task_run_registry"] is False
@_async_test
async def test_shutdown_cancellation_does_not_write_business_cancelled() -> None:
registry = ActiveTaskRunRegistry()
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
engine._shutting_down = True
engine.store = SimpleNamespace(
is_ready=True,
get_task=AsyncMock(),
save_task=AsyncMock(),
)
engine._run_task_once = AsyncMock(side_effect=asyncio.CancelledError)
task = Task(
id="task-1",
title="Interrupted task",
project_id="project-a",
status=TaskStatus.RUNNING,
)
with pytest.raises(asyncio.CancelledError):
await engine._execute_task(task)
engine.store.get_task.assert_not_awaited()
engine.store.save_task.assert_not_awaited()
assert not registry.is_active("project-a", task.id)
@_async_test
async def test_company_cancellation_never_synthesizes_hold_without_checkpoint() -> None:
engine = OPCEngine(project_id="project-a")
engine.store = SimpleNamespace(
is_ready=True,
get_task=AsyncMock(),
save_task=AsyncMock(),
)
engine._run_task_once = AsyncMock(side_effect=asyncio.CancelledError)
task = Task(
id="company-task",
title="Company task",
project_id="project-a",
status=TaskStatus.RUNNING,
metadata={"work_item_runtime": True},
)
with pytest.raises(asyncio.CancelledError):
await engine._execute_task(task)
engine.store.get_task.assert_not_awaited()
engine.store.save_task.assert_not_awaited()
assert "company_runtime_suspended_at" not in task.metadata
assert "last_stop_reason" not in task.metadata
@_async_test
async def test_suspended_checkpoint_discards_racing_task_completion() -> None:
engine = OPCEngine(project_id="project-a")
engine._shutting_down = False
task = Task(
id="company-task",
title="Company task",
project_id="project-a",
status=TaskStatus.RUNNING,
metadata={"work_item_runtime": True},
)
suspended = Task(
id=task.id,
title=task.title,
project_id=task.project_id,
status=TaskStatus.BLOCKED,
metadata={
"work_item_runtime": True,
"dispatch_hold": "company_runtime_suspended",
"company_runtime_stop_state": "suspended",
},
)
engine.store = SimpleNamespace(
get_task=AsyncMock(return_value=suspended),
save_task=AsyncMock(),
)
engine._run_task_once = AsyncMock(
return_value=TaskResult(status=TaskStatus.DONE, content="done", artifacts={})
)
engine._apply_runtime_state_to_task = MagicMock()
with pytest.raises(asyncio.CancelledError):
await engine._execute_task(task)
engine.store.save_task.assert_not_awaited()
engine._apply_runtime_state_to_task.assert_not_called()
@_async_test
async def test_attempt_stays_active_until_result_persistence_finishes() -> None:
registry = ActiveTaskRunRegistry()
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
save_started = asyncio.Event()
allow_save = asyncio.Event()
async def blocked_save(_task: Task) -> None:
save_started.set()
await allow_save.wait()
engine.store = SimpleNamespace(
get_task=AsyncMock(return_value=None),
save_task=blocked_save,
)
engine._run_task_once = AsyncMock(
return_value=TaskResult(status=TaskStatus.IDLE, content="done", artifacts={})
)
engine._apply_runtime_state_to_task = MagicMock()
task = Task(
id="persisting-task",
title="Persisting task",
project_id="project-a",
status=TaskStatus.RUNNING,
)
execution = asyncio.create_task(engine._execute_task(task))
await save_started.wait()
assert registry.is_active("project-a", task.id)
allow_save.set()
await execution
assert not registry.is_active("project-a", task.id)
@_async_test
async def test_claimed_work_item_ownership_covers_post_execution_finalize_gap() -> None:
registry = ActiveTaskRunRegistry()
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
engine.store = SimpleNamespace(
get_task=AsyncMock(return_value=None),
save_task=AsyncMock(),
)
engine._run_task_once = AsyncMock(
return_value=TaskResult(status=TaskStatus.IDLE, content="done", artifacts={})
)
engine._apply_runtime_state_to_task = MagicMock()
task = Task(
id="finalizing-work-item",
title="Finalizing work item",
project_id="project-a",
parent_session_id="runtime-session",
status=TaskStatus.RUNNING,
metadata={"work_item_runtime": True},
)
inner_finished = asyncio.Event()
allow_finalize = asyncio.Event()
executor = object.__new__(CompanyWorkItemExecutor)
executor.active_task_run_registry = registry
async def run_claimed(*_args: object, **_kwargs: object) -> TaskResult:
result = await engine._execute_task(task)
inner_finished.set()
await allow_finalize.wait()
return result
executor._run_claimed_work_item = run_claimed
owned = executor._create_claimed_work_item_task(
CompanyMemberSession(
role_id="executor",
seat_id="seat::executor",
member_session_id="role-session",
),
task,
{},
)
await inner_finished.wait()
assert registry.attempt_count("project-a", task.id) == 1
allow_finalize.set()
await owned
assert not registry.is_active("project-a", task.id)
@_async_test
async def test_work_item_claim_and_spawn_share_stop_scope_lock() -> None:
registry = ActiveTaskRunRegistry()
executor = object.__new__(CompanyWorkItemExecutor)
executor.active_task_run_registry = registry
claim_entered = asyncio.Event()
allow_claim = asyncio.Event()
allow_child_exit = asyncio.Event()
stop_acquired = asyncio.Event()
task = Task(
id="claimed-task",
title="Claimed task",
project_id="project-a",
session_id="role-session",
parent_session_id="runtime-session",
metadata={"work_item_runtime": True},
)
member_session = CompanyMemberSession(
role_id="executor",
seat_id="seat::executor",
member_session_id="role-session",
)
async def claim_runnable_tasks(
_tasks: list[Task],
*,
work_items: list[object],
) -> list[tuple[CompanyMemberSession, Task]]:
del work_items
claim_entered.set()
await allow_claim.wait()
return [(member_session, task)]
async def run_claimed(*_args: object, **_kwargs: object) -> None:
await allow_child_exit.wait()
executor.runtime = SimpleNamespace(
claim_runnable_tasks=claim_runnable_tasks,
)
executor._run_claimed_work_item = run_claimed
active: dict[asyncio.Task, tuple[CompanyMemberSession, Task]] = {}
scheduled = asyncio.create_task(
executor._claim_and_create_work_item_tasks([task], [], active)
)
await claim_entered.wait()
async def stop_scope() -> None:
async with registry.scope_lock("project-a", "runtime-session"):
assert registry.is_active("project-a", task.id)
stop_acquired.set()
stopping = asyncio.create_task(stop_scope())
await asyncio.sleep(0)
assert not stop_acquired.is_set()
allow_claim.set()
await scheduled
await stopping
assert len(active) == 1
allow_child_exit.set()
await asyncio.gather(*active)
assert not registry.is_active("project-a", task.id)
@_async_test
async def test_company_executor_driver_ownership_covers_idle_scheduler_window() -> None:
registry = ActiveTaskRunRegistry()
entered = asyncio.Event()
allow_exit = asyncio.Event()
executor = object.__new__(CompanyWorkItemExecutor)
executor.active_task_run_registry = registry
async def idle_scheduler(
_plan: CompanyWorkItemRuntimePlan,
_tasks: list[Task],
) -> str:
entered.set()
await allow_exit.wait()
return "done"
executor._execute_multi_team_org = idle_scheduler
task = Task(
id="driver-task",
title="Driver task",
project_id="project-a",
parent_session_id="runtime-session",
metadata={"work_item_runtime": True},
)
execution = asyncio.create_task(
executor.execute(CompanyWorkItemRuntimePlan(), [task])
)
await entered.wait()
assert registry.is_active("project-a", task.id)
allow_exit.set()
assert await execution == "done"
assert not registry.is_active("project-a", task.id)
@_async_test
async def test_borrowed_engine_shutdown_keeps_controller_registry_open() -> None:
registry = ActiveTaskRunRegistry()
root_token = registry.register("project-a", "root-attempt")
borrowed = OPCEngine(
project_id="project-a",
active_task_run_registry=registry,
owns_active_task_run_registry=False,
)
await borrowed.shutdown()
assert not registry.admission_closed
assert registry.is_active("project-a", "root-attempt")
next_token = registry.register("project-a", "next-attempt")
registry.unregister("project-a", "next-attempt", next_token)
registry.unregister("project-a", "root-attempt", root_token)
@_async_test
async def test_shutdown_preparation_includes_project_delegates() -> None:
engine = OPCEngine(project_id="project-a")
delegate_prepare = AsyncMock(
return_value=[{"session_id": "delegate-session", "checkpoint_id": "checkpoint-1"}]
)
engine._project_engine_delegates["project-b"] = SimpleNamespace(
prepare_active_company_runtimes_for_shutdown=delegate_prepare,
)
prepared = await engine.prepare_active_company_runtimes_for_shutdown()
assert prepared == [{"session_id": "delegate-session", "checkpoint_id": "checkpoint-1"}]
delegate_prepare.assert_awaited_once()
@_async_test
async def test_engine_shutdown_prepares_before_closing_subsystems() -> None:
engine = OPCEngine(project_id="project-a")
engine.prepare_active_company_runtimes_for_shutdown = AsyncMock(return_value=[])
await engine.shutdown()
engine.prepare_active_company_runtimes_for_shutdown.assert_awaited_once()
@_async_test
async def test_engine_shutdown_does_not_close_store_when_durable_prepare_fails() -> None:
engine = OPCEngine(project_id="project-a")
engine.prepare_active_company_runtimes_for_shutdown = AsyncMock(
side_effect=RuntimeError("checkpoint failed")
)
engine.store = SimpleNamespace(close=AsyncMock())
engine.message_bus.stop = MagicMock()
with pytest.raises(RuntimeError, match="checkpoint failed"):
await engine.shutdown()
engine.message_bus.stop.assert_not_called()
engine.store.close.assert_not_awaited()
+264 -50
View File
@@ -1116,6 +1116,34 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(calls, ["first", "second"])
self.assertIn("Queued #1", console.export_text())
def test_chat_turn_controller_prepares_checkpoint_before_cancelling_active_turn(self) -> None:
state, _engine = self._make_state()
async def _run() -> list[str]:
order: list[str] = []
started = asyncio.Event()
async def active_turn() -> None:
started.set()
try:
await asyncio.Event().wait()
finally:
order.append("cancelled")
async def prepare() -> list[dict[str, Any]]:
order.append("prepared")
return []
state.engine.prepare_active_company_runtimes_for_shutdown = prepare
controller = ChatTurnController(state)
controller.active_task = asyncio.create_task(active_turn())
await started.wait()
await controller.shutdown()
return order
self.assertEqual(asyncio.run(_run()), ["prepared", "cancelled"])
def test_busy_slash_policy_allows_readonly_and_blocks_mutating_commands(self) -> None:
self.assertEqual(_busy_slash_policy("kanban", []), BusyCommandPolicy.IMMEDIATE_READONLY)
self.assertEqual(_busy_slash_policy("logs", ["task-1"]), BusyCommandPolicy.IMMEDIATE_READONLY)
@@ -1168,6 +1196,17 @@ class CliSlashCommandTests(unittest.TestCase):
"org_id": "quantum_harbor",
"preferred_agent": "codex",
})
store.checkpoints = [
SimpleNamespace(
checkpoint_id="cp-org-interrupted",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id="task-1",
session_id="sess-1",
updated_at=datetime(2026, 5, 17, 12, 0),
payload={},
)
]
state, engine = self._make_state(store=store)
state.mode = "company"
state.company_profile = "corporate"
@@ -1183,7 +1222,59 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(engine.calls[-1]["org_id"], "quantum_harbor")
self.assertIsNone(engine.calls[-1]["company_profile"])
self.assertEqual(engine.calls[-1]["preferred_agent"], "codex")
self.assertEqual(engine.calls[-1]["message_metadata"], {"ui_force_resume": True})
self.assertEqual(engine.calls[-1]["message_metadata"], {
"ui_force_resume": True,
"response_to_checkpoint_id": "cp-org-interrupted",
"response_to_checkpoint_type": "company_runtime_interrupted",
})
def test_company_continue_requires_a_durable_runtime_checkpoint(self) -> None:
console = Console(record=True, force_terminal=False, width=120)
store = self._Store()
store.tasks[0].metadata.update({
"exec_mode": "company",
"company_profile": "corporate",
})
state, engine = self._make_state(store=store)
async def _run() -> None:
await _handle_chat_slash_command(state, "/continue")
with patch("opc.cli.app.console", console):
asyncio.run(_run())
self.assertEqual(engine.calls, [])
self.assertIn("No suspended or interrupted company runtime", console.export_text())
def test_continue_slash_routes_to_durable_runtime_checkpoint(self) -> None:
store = self._Store()
store.checkpoints = [
SimpleNamespace(
checkpoint_id="cp-interrupted",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id="task-1",
session_id="sess-1",
updated_at=datetime(2026, 5, 17, 12, 0),
payload={},
)
]
state, engine = self._make_state(store=store)
state.mode = "company"
state.runtime_control_state = "suspended"
async def _run() -> None:
await _handle_chat_slash_command(state, "/continue")
asyncio.run(_run())
self.assertEqual(engine.calls[-1]["session_id"], "sess-1")
self.assertEqual(engine.calls[-1]["message_metadata"], {
"ui_force_resume": True,
"response_to_checkpoint_id": "cp-interrupted",
"response_to_checkpoint_type": "company_runtime_interrupted",
})
self.assertEqual(state.runtime_control_checkpoint_id, "cp-interrupted")
def test_plain_message_after_stop_routes_to_suspend_checkpoint(self) -> None:
store = self._Store()
@@ -1209,7 +1300,127 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(engine.calls[-1]["message_metadata"]["response_to_checkpoint_id"], "cp-suspend")
self.assertEqual(engine.calls[-1]["message_metadata"]["response_to_checkpoint_type"], "company_runtime_suspended")
self.assertEqual(state.runtime_control_state, "running")
self.assertEqual(state.runtime_control_state, "suspended")
def test_plain_message_from_company_child_uses_root_runtime_checkpoint(self) -> None:
store = self._Store()
now = datetime(2026, 5, 17, 12, 0)
store.tasks[0].metadata.update({
"exec_mode": "company",
"mode": "company",
"company_profile": "corporate",
})
store.tasks[0].parent_session_id = ""
store.tasks.append(SimpleNamespace(
id="worker-child",
title="Worker",
description="Worker turn",
status=TaskStatus.BLOCKED,
priority=3,
assigned_to="worker",
session_id="sess-1:role:worker",
parent_session_id="sess-1",
project_id="demo",
created_at=now,
tags=[],
result={},
linked_work_item_id="worker-item",
metadata={
"exec_mode": "company",
"work_item_runtime": True,
"work_item_projection_id": "worker",
"company_runtime_root_session_id": "sess-1",
},
context_snapshot={},
))
store.checkpoints = [SimpleNamespace(
checkpoint_id="cp-child-interrupted",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id="task-1",
session_id="sess-1",
project_id="demo",
updated_at=now,
payload={"parent_session_id": "sess-1"},
)]
state, engine = self._make_state(store=store)
state.session_id = "sess-1:role:worker"
# The selected child channel may have left the CLI's ambient mode in
# task mode. Runtime config, not ambient state, owns resumed execution.
state.mode = "task"
asyncio.run(_process_interactive_chat_message(state, "revise and continue"))
call = engine.calls[-1]
self.assertEqual(call["session_id"], "sess-1")
self.assertEqual(call["origin_task_id"], "task-1")
self.assertEqual(call["mode"], "company")
self.assertEqual(call["company_profile"], "corporate")
self.assertEqual(call["message_metadata"], {
"response_to_checkpoint_id": "cp-child-interrupted",
"response_to_checkpoint_type": "company_runtime_interrupted",
})
def test_plain_message_during_resuming_checkpoint_fails_closed(self) -> None:
console = Console(record=True, force_terminal=False, width=140)
store = self._Store()
store.tasks[0].metadata.update({
"exec_mode": "company",
"mode": "company",
"company_profile": "corporate",
})
store.checkpoints = [SimpleNamespace(
checkpoint_id="cp-resuming",
checkpoint_type="company_runtime_suspended",
status="resuming",
task_id="task-1",
session_id="sess-1",
project_id="demo",
updated_at=datetime(2026, 5, 17, 12, 0),
payload={"parent_session_id": "sess-1"},
)]
state, engine = self._make_state(store=store)
state.mode = "company"
with patch("opc.cli.app.console", console):
asyncio.run(_process_interactive_chat_message(state, "continue again"))
self.assertEqual(engine.calls, [])
self.assertIn("checkpoint is resuming", console.export_text())
def test_explicit_cli_runtime_checkpoint_mismatch_fails_closed(self) -> None:
console = Console(record=True, force_terminal=False, width=140)
store = self._Store()
store.tasks[0].metadata.update({
"exec_mode": "company",
"mode": "company",
"company_profile": "corporate",
})
store.checkpoints = [SimpleNamespace(
checkpoint_id="cp-current",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id="task-1",
session_id="sess-1",
project_id="demo",
updated_at=datetime(2026, 5, 17, 12, 0),
payload={"parent_session_id": "sess-1"},
)]
state, engine = self._make_state(store=store)
state.mode = "company"
with patch("opc.cli.app.console", console):
asyncio.run(_process_interactive_chat_message(
state,
"continue stale",
message_metadata={
"response_to_checkpoint_id": "cp-stale",
"response_to_checkpoint_type": "company_runtime_interrupted",
},
))
self.assertEqual(engine.calls, [])
self.assertIn("checkpoint identity mismatch", console.export_text())
def test_session_resume_still_switches_session(self) -> None:
state, engine = self._make_state(store=self._Store())
@@ -1287,8 +1498,6 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(output["payload"]["checkpoint_id"], "cp-stop")
self.assertEqual(output["cancelled"], ["task-company"])
self.assertEqual(output["suspend_calls"][0]["session_id"], "sess-company")
self.assertEqual(output["task"].metadata["dispatch_hold"], "company_runtime_suspended")
self.assertEqual(output["task"].metadata["company_runtime_stop_state"], "suspended")
def test_session_service_continue_uses_force_resume_metadata(self) -> None:
class Store:
@@ -1305,10 +1514,17 @@ class CliSlashCommandTests(unittest.TestCase):
metadata={
"exec_mode": "company",
"company_profile": "corporate",
"dispatch_hold": "company_runtime_suspended",
"company_runtime_stop_state": "suspended",
},
)
self.checkpoint = SimpleNamespace(
checkpoint_id="cp-company",
checkpoint_type="company_runtime_suspended",
status="pending",
project_id="demo",
session_id="sess-company",
task_id="task-company",
payload={},
)
async def get_task(self, task_id: str):
return self.task if task_id == self.task.id else None
@@ -1319,6 +1535,9 @@ class CliSlashCommandTests(unittest.TestCase):
async def get_session(self, session_id: str):
return SimpleNamespace(session_id=session_id, project_id="demo")
async def get_execution_checkpoints(self, **_kwargs):
return [self.checkpoint]
async def save_task(self, task):
self.task = task
@@ -1331,7 +1550,12 @@ class CliSlashCommandTests(unittest.TestCase):
engine = SimpleNamespace(project_id="demo", store=Store(), process_message=process_message)
context = OfficeServiceContext(engine=engine, agent_store=None, chat_store=None, event_adapter=None)
result = await SessionService(context).continue_run(project_id="demo", target="sess-company")
result = await SessionService(context).continue_run(
project_id="demo",
target="sess-company",
runtime_session_id="sess-company",
checkpoint_id="cp-company",
)
self.assertEqual(result.payload["response"], "resumed")
return calls
@@ -1340,7 +1564,11 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(calls[-1]["content"], "Resume the existing runtime.")
self.assertEqual(calls[-1]["session_id"], "sess-company")
self.assertEqual(calls[-1]["mode"], "company")
self.assertEqual(calls[-1]["message_metadata"], {"ui_force_resume": True})
self.assertEqual(calls[-1]["message_metadata"], {
"ui_force_resume": True,
"response_to_checkpoint_id": "cp-company",
"response_to_checkpoint_type": "company_runtime_suspended",
})
def test_session_service_continue_preserves_custom_org_id(self) -> None:
class Store:
@@ -1359,10 +1587,17 @@ class CliSlashCommandTests(unittest.TestCase):
"company_profile": "custom",
"org_id": "quantum_harbor",
"preferred_agent": "codex",
"dispatch_hold": "company_runtime_suspended",
"company_runtime_stop_state": "suspended",
},
)
self.checkpoint = SimpleNamespace(
checkpoint_id="cp-org",
checkpoint_type="company_runtime_interrupted",
status="pending",
project_id="demo",
session_id="sess-org",
task_id="task-org",
payload={},
)
async def get_task(self, task_id: str):
return self.task if task_id == self.task.id else None
@@ -1373,6 +1608,9 @@ class CliSlashCommandTests(unittest.TestCase):
async def get_session(self, session_id: str):
return SimpleNamespace(session_id=session_id, project_id="demo")
async def get_execution_checkpoints(self, **_kwargs):
return [self.checkpoint]
async def save_task(self, task):
self.task = task
@@ -1385,7 +1623,12 @@ class CliSlashCommandTests(unittest.TestCase):
engine = SimpleNamespace(project_id="demo", store=Store(), process_message=process_message)
context = OfficeServiceContext(engine=engine, agent_store=None, chat_store=None, event_adapter=None)
result = await SessionService(context).continue_run(project_id="demo", target="sess-org")
result = await SessionService(context).continue_run(
project_id="demo",
target="sess-org",
runtime_session_id="sess-org",
checkpoint_id="cp-org",
)
self.assertEqual(result.payload["response"], "resumed")
return calls
@@ -1394,7 +1637,11 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(calls[-1]["mode"], "org")
self.assertEqual(calls[-1]["org_id"], "quantum_harbor")
self.assertIsNone(calls[-1]["company_profile"])
self.assertEqual(calls[-1]["message_metadata"], {"ui_force_resume": True})
self.assertEqual(calls[-1]["message_metadata"], {
"ui_force_resume": True,
"response_to_checkpoint_id": "cp-org",
"response_to_checkpoint_type": "company_runtime_interrupted",
})
def test_queue_slash_lists_and_drops_queued_prompts(self) -> None:
console = Console(record=True, force_terminal=False, width=160)
@@ -2648,49 +2895,16 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertIn("Pending Checkpoints", rendered)
self.assertIn("cp-1", rendered)
def test_recover_slash_lists_and_resumes_interrupted_runtime(self) -> None:
def test_legacy_recover_slash_is_not_registered(self) -> None:
console = Console(record=True, force_terminal=False, width=200)
state, _engine = self._make_state(store=self._Store())
resume_calls: list[str] = []
class _FakeRecoveryManager:
async def get_status(self):
return SimpleNamespace(
interrupted=[
SimpleNamespace(
parent_task_id="parent-task",
parent_session_id="parent-session",
title="Interrupted Runtime",
profile="corporate",
interrupted_at="2026-05-03T12:00:00",
work_items=[
SimpleNamespace(projection_id="wi-1", interrupted=True),
SimpleNamespace(projection_id="wi-2", interrupted=False),
],
)
],
active_recoveries=[],
)
async def resume(self, parent_task_id: str):
resume_calls.append(parent_task_id)
if parent_task_id == "parent-task":
return {"ok": True, "resumed_work_item_projection_ids": ["wi-1"]}
return {"ok": False, "error": "not_found"}
async def _run() -> None:
await _handle_chat_slash_command(state, "/recover")
await _handle_chat_slash_command(state, "/recover resume parent-task")
await _handle_chat_slash_command(state, "/recover resume cp-1")
with patch("opc.cli.app.console", console), patch("opc.cli.app._get_chat_recovery_manager", return_value=_FakeRecoveryManager()):
asyncio.run(_run())
with patch("opc.cli.app.console", console):
asyncio.run(_handle_chat_slash_command(state, "/recover"))
rendered = console.export_text()
self.assertIn("Interrupted Runtime", rendered)
self.assertIn("Recovery started for parent-task", rendered)
self.assertIn("Checkpoint cp-1 is not resumed directly", rendered)
self.assertEqual(resume_calls, ["parent-task", "cp-1"])
self.assertIn("Unknown command: /recover", rendered)
self.assertNotIn("Interrupted Company Runtimes", rendered)
def test_logs_slash_renders_task_and_session_runtime_details(self) -> None:
console = Console(record=True, force_terminal=False, width=220)
+76
View File
@@ -527,6 +527,25 @@ class CompanyCollaborationTests(unittest.IsolatedAsyncioTestCase):
await runtime.bootstrap([task])
runtime.enqueue_runnable_work_items([work_item], task_by_work_item_id={"work-item-1": task})
async def claim_work_item(*_args: object, **kwargs: object) -> DelegationWorkItem:
role_session_id = str(kwargs["role_runtime_session_id"])
work_item.phase = Phase.RUNNING
work_item.role_runtime_session_id = role_session_id
work_item.claimed_by_role_runtime_session_id = role_session_id
work_item.claimed_by_seat_id = str(kwargs.get("seat_id", ""))
work_item.metadata = {
**dict(work_item.metadata or {}),
"claimed_by_role_session_id": role_session_id,
"claimed_task_id": str(kwargs["task_id"]),
}
return work_item
runtime.store = SimpleNamespace(
is_ready=True,
claim_delegation_work_item_if_dispatchable=claim_work_item,
save_delegation_role_session=AsyncMock(),
)
claims = await runtime.claim_runnable_tasks([task], work_items=[work_item])
self.assertEqual(len(claims), 1)
@@ -534,6 +553,61 @@ class CompanyCollaborationTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(claimed_task.id, task.id)
self.assertEqual(claimed_session.member_session_id, "role-session::proj1::root-a::executor::backend-architect")
async def test_company_runtime_does_not_spawn_after_atomic_claim_loses_to_hold(self) -> None:
runtime = CompanyRuntime(
org_engine=DummyOrgEngine(),
communication=DummyRuntimeCommunication(),
)
task = Task(
id="held-work-item-task",
title="Held Work Item",
session_id="root-a",
parent_session_id="root-a",
assigned_to="executor",
status=TaskStatus.PENDING,
project_id="proj1",
metadata={
"work_item_projection_id": "held_execution",
"work_item_role_id": "executor",
"employee_assignment": {
"employee_id": "backend-architect",
"role_id": "executor",
},
},
)
set_linked_work_item_id(task, "held-work-item")
work_item = DelegationWorkItem(
work_item_id="held-work-item",
run_id="run-1",
cell_id="cell-1",
role_id="executor",
projection_id="held_execution",
phase=Phase.READY,
)
held_after_race = DelegationWorkItem(
**{
**work_item.__dict__,
"metadata": {"dispatch_hold": "company_runtime_suspended"},
}
)
claim = AsyncMock(return_value=None)
runtime.store = SimpleNamespace(
is_ready=True,
claim_delegation_work_item_if_dispatchable=claim,
get_delegation_work_item=AsyncMock(return_value=held_after_race),
)
await runtime.bootstrap([task])
runtime.enqueue_runnable_work_items(
[work_item],
task_by_work_item_id={work_item.work_item_id: task},
)
claims = await runtime.claim_runnable_tasks([task], work_items=[work_item])
self.assertEqual(claims, [])
self.assertNotIn(work_item.work_item_id, runtime._claimed_work_item_ids)
claim.assert_awaited_once()
async def test_company_runtime_does_not_double_claim_same_work_item(self) -> None:
runtime = CompanyRuntime(
org_engine=DummyOrgEngine(),
@@ -2722,9 +2796,11 @@ class CompanyCollaborationTests(unittest.IsolatedAsyncioTestCase):
engine.project_id = "proj1"
engine.store = store
engine.company_executor = DummyExecutor()
attempt_token = engine._active_task_run_registry.register("proj1", running_task.id)
response = await engine._maybe_resume_existing_company_runtime("缁х画", "sess-parent-live")
refreshed = await store.get_task(running_task.id)
engine._active_task_run_registry.unregister("proj1", running_task.id, attempt_token)
self.assertIn("already in progress", response)
self.assertEqual(refreshed.status, TaskStatus.RUNNING)
@@ -80,7 +80,6 @@ class SharedRoleSessionExecutionTests(unittest.IsolatedAsyncioTestCase):
record_child_session_result=AsyncMock(),
record_task_completion_async=AsyncMock(),
)
engine._active_task_runs = set()
engine._run_task_once = AsyncMock(
return_value=TaskResult(status=TaskStatus.DONE, content="done", artifacts={})
)
@@ -106,4 +105,3 @@ class SharedRoleSessionExecutionTests(unittest.IsolatedAsyncioTestCase):
engine.memory.record_assistant_turn.assert_awaited_once()
engine.memory.record_child_session_result.assert_not_awaited()
+410
View File
@@ -0,0 +1,410 @@
from __future__ import annotations
import asyncio
import json
from datetime import datetime, timedelta
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
from opc.core.models import ExecutionCheckpoint, Task, TaskStatus
from opc.layer2_organization.company_runtime_identity import (
build_company_runtime_identity_index,
)
from opc.plugins.office_ui.services.models import ServiceError
from opc.plugins.office_ui.snapshot_builder import (
_build_company_runtime_control_by_task,
_primary_session_tasks_by_session_id,
)
from opc.plugins.office_ui.ws_handler import WSHandler
def _runtime_records() -> tuple[list[Task], ExecutionCheckpoint]:
runtime_session_id = "runtime-session"
anchor = Task(
id="ui-anchor",
project_id="project-a",
session_id=runtime_session_id,
status=TaskStatus.CANCELLED,
metadata={
"exec_mode": "company",
"mode": "company",
"company_profile": "corporate",
},
created_at=datetime.now() - timedelta(minutes=3),
)
final_decider = Task(
id="final-decider",
project_id="project-a",
session_id=runtime_session_id,
parent_session_id=runtime_session_id,
status=TaskStatus.CANCELLED,
linked_work_item_id="work-item-root",
metadata={
"mode": "company",
"work_item_runtime": True,
"work_item_projection_id": "root",
"shared_role_session": True,
"shared_role_id": "ceo",
"company_runtime_root_session_id": runtime_session_id,
},
created_at=datetime.now() - timedelta(minutes=2),
)
child = Task(
id="worker",
project_id="project-a",
session_id=f"{runtime_session_id}:role:worker",
parent_session_id=runtime_session_id,
status=TaskStatus.BLOCKED,
linked_work_item_id="work-item-worker",
metadata={
"mode": "company",
"work_item_runtime": True,
"work_item_projection_id": "worker",
"shared_role_session": True,
"company_runtime_root_session_id": runtime_session_id,
},
created_at=datetime.now() - timedelta(minutes=1),
)
checkpoint = ExecutionCheckpoint(
checkpoint_id="checkpoint-1",
project_id="project-a",
session_id=runtime_session_id,
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id=final_decider.id,
payload={"parent_session_id": runtime_session_id},
)
return [anchor, final_decider, child], checkpoint
def test_identity_is_session_first_and_never_selects_shared_final_decider_as_ui_anchor() -> None:
tasks, checkpoint = _runtime_records()
index = build_company_runtime_identity_index(tasks, [checkpoint])
identity = index.resolve(
task_id="final-decider",
runtime_session_id="runtime-session",
checkpoint_id="checkpoint-1",
)
assert identity is not None
assert identity.runtime_session_id == "runtime-session"
assert identity.ui_anchor_task_id == "ui-anchor"
assert identity.config_source_task_id == "ui-anchor"
assert identity.runtime_task_ids == ("ui-anchor", "final-decider", "worker")
assert identity.pending_checkpoint_id == "checkpoint-1"
assert identity.resumable is True
assert index.resolve(task_session_id="runtime-session:role:worker") == identity
assert index.resolve(task_id="worker", runtime_session_id="other-session") is None
assert index.resolve(task_id="ui-anchor", checkpoint_id="other-checkpoint") is None
def test_config_source_uses_configured_scope_task_when_ui_anchor_has_no_config() -> None:
tasks, _checkpoint = _runtime_records()
tasks[0].metadata = {}
tasks[1].metadata.update({"exec_mode": "org", "company_profile": "custom", "org_id": "studio"})
identity = build_company_runtime_identity_index(tasks).resolve(
task_id="ui-anchor",
)
assert identity is not None
assert identity.ui_anchor_task_id == "ui-anchor"
assert identity.config_source_task_id == "final-decider"
def test_snapshot_session_representative_uses_canonical_ui_anchor() -> None:
tasks, _checkpoint = _runtime_records()
primary, ordered = _primary_session_tasks_by_session_id(
[tasks[1], tasks[0], tasks[2]],
)
assert ordered[0] == "runtime-session"
assert primary["runtime-session"].id == "ui-anchor"
def test_runtime_without_ui_anchor_never_promotes_shared_work_item() -> None:
tasks, checkpoint = _runtime_records()
shared_final = tasks[1]
shared_final.parent_session_id = None
index = build_company_runtime_identity_index([shared_final, tasks[2]], [checkpoint])
identity = index.resolve(runtime_session_id="runtime-session")
assert identity is not None
assert identity.ui_anchor_task_id == ""
assert identity.config_source_task_id == "final-decider"
primary, _ordered = _primary_session_tasks_by_session_id(
[shared_final, tasks[2]],
)
assert "runtime-session" not in primary
def test_snapshot_projects_checkpoint_control_to_cancelled_anchor_without_task_resume_identity() -> None:
tasks, checkpoint = _runtime_records()
class Store:
async def get_execution_checkpoints(self, **_kwargs):
return [checkpoint]
engine = SimpleNamespace(store=Store())
control = asyncio.run(_build_company_runtime_control_by_task(engine, tasks, "project-a"))
assert control["ui-anchor"]["runtime_control_state"] == "suspended"
assert control["ui-anchor"]["can_resume"] is True
assert control["ui-anchor"]["resume_parent_session_id"] == "runtime-session"
assert control["ui-anchor"]["pending_runtime_checkpoint_id"] == "checkpoint-1"
assert "resume_parent_task_id" not in control["ui-anchor"]
def test_service_error_transport_fields_cannot_be_overridden() -> None:
handler = WSHandler.__new__(WSHandler)
sent: list[dict] = []
async def send_ack(_ws, ok=True, **payload):
sent.append({"ok": ok, **payload})
handler._send_ack = send_ack
error = ServiceError(
"actual_code",
"actual message",
{"ok": True, "code": "wrong", "error": "wrong", "detail": "kept"},
)
asyncio.run(handler._send_service_error(object(), error, action="test_action"))
assert sent == [{
"ok": False,
"detail": "kept",
"error": "actual message",
"code": "actual_code",
"action": "test_action",
}]
def test_removed_recovery_action_receives_normal_unknown_message_ack() -> None:
handler = WSHandler.__new__(WSHandler)
handler._shutting_down = False
handler._active_message_tasks = set()
handler._send_ack = AsyncMock()
ws = object()
asyncio.run(handler._route_message(ws, json.dumps({"type": "recovery_action"})))
handler._send_ack.assert_awaited_once_with(
ws,
ok=False,
error="unknown_message_type",
action="recovery_action",
)
def test_work_item_chat_resume_uses_canonical_ui_anchor_as_engine_origin() -> None:
async def scenario() -> None:
tasks, checkpoint = _runtime_records()
handler = WSHandler.__new__(WSHandler)
handler._exec_mode = "task"
handler._company_profile = "corporate"
handler._shutting_down = False
handler._active_runtime_children = {}
handler._session_to_task = {}
handler._task_bg_context = {}
handler._company_suspend_reply_locks = {"runtime-session": asyncio.Lock()}
handler.chat_store = None
handler._set_company_runtime_control = AsyncMock()
handler._normalize_session_exec_mode = MagicMock(return_value="task")
handler._normalize_session_company_profile = MagicMock(return_value="corporate")
handler._resolve_task_session_config = MagicMock(
return_value=("company", "corporate")
)
handler._resolve_task_org_id = MagicMock(return_value="")
handler._extract_checkpoint_metadata = AsyncMock(return_value=None)
handler._sync_task_transcript_messages = AsyncMock()
handler.on_kanban_changed = AsyncMock()
handler._flush_progress = AsyncMock()
run_engine = SimpleNamespace(
project_id="project-a",
process_message=AsyncMock(return_value="resumed"),
)
target = {
"ui_anchor_task_id": "ui-anchor",
"config_task": tasks[1],
}
await handler._process_company_suspend_reply(
ui_task_id="final-decider",
runtime_session_id="runtime-session",
content="continue",
attachment_refs=None,
message_metadata={"ui_force_resume": True},
user_message_id=None,
user_message_created_at=None,
run_engine=run_engine,
run_project_id="project-a",
target=target,
checkpoint=checkpoint,
lock=handler._company_suspend_reply_locks["runtime-session"],
)
call = run_engine.process_message.await_args
assert call.kwargs["session_id"] == "runtime-session"
assert call.kwargs["origin_task_id"] == "ui-anchor"
assert handler._session_to_task["runtime-session"] == "ui-anchor"
handler.on_kanban_changed.assert_awaited_once_with(engine=run_engine)
asyncio.run(scenario())
def test_delivery_feedback_rejects_missing_canonical_identity_without_first_task_fallback() -> None:
async def scenario() -> None:
tasks, checkpoint = _runtime_records()
class Store:
is_ready = True
async def get_tasks(self, **_kwargs):
# A shared final-decider deliberately precedes the UI anchor;
# legacy first-match routing selected the wrong Task here.
return [tasks[1], tasks[0], tasks[2]]
handler = WSHandler.__new__(WSHandler)
handler._resolve_company_runtime_target = AsyncMock(return_value=None)
engine = SimpleNamespace(store=Store())
target = await handler._company_delivery_feedback_parent_target(
task_id="final-decider",
waiting_task_id="worker",
waiting_task=tasks[2],
checkpoint=checkpoint,
payload={"parent_session_id": "runtime-session"},
engine=engine,
)
assert target == {"parent_task_id": "", "parent_session_id": ""}
asyncio.run(scenario())
def test_delivery_feedback_route_consumes_missing_identity_instead_of_running_work_item() -> None:
async def scenario() -> None:
handler = WSHandler.__new__(WSHandler)
handler.chat_store = None
handler._company_delivery_feedback_reply_locks = {}
handler._load_execution_checkpoint_for_reply = AsyncMock(return_value=SimpleNamespace(
checkpoint_id="feedback-1",
checkpoint_type="company_delivery_feedback",
status="pending",
task_id="worker",
session_id="runtime-session",
payload={"waiting_task_id": "worker", "parent_session_id": "runtime-session"},
))
handler._company_delivery_feedback_parent_target = AsyncMock(return_value={
"parent_task_id": "",
"parent_session_id": "",
})
handler._track_session = MagicMock()
engine = SimpleNamespace(store=SimpleNamespace(is_ready=True))
handled = await handler._route_company_delivery_feedback_reply_if_pending(
task_id="worker",
content="looks good",
session_id="runtime-session:worker",
task=SimpleNamespace(id="worker"),
attachment_refs=None,
message_metadata={
"response_to_checkpoint_id": "feedback-1",
"response_to_checkpoint_type": "company_delivery_feedback",
},
user_message_id="message-1",
user_message_created_at=None,
run_engine=engine,
run_project_id="project-a",
reply_channel_id="session:worker",
)
assert handled is True
handler._track_session.assert_not_called()
asyncio.run(scenario())
def test_suspend_reply_identity_mismatch_and_resuming_checkpoint_fail_closed() -> None:
async def scenario() -> None:
tasks, checkpoint = _runtime_records()
handler = WSHandler.__new__(WSHandler)
handler.chat_store = None
handler._company_stop_finalize_tasks = {}
handler._company_suspend_reply_locks = {}
handler._track = MagicMock()
engine = SimpleNamespace(project_id="project-a")
target = {
"runtime_session_id": "runtime-session",
"checkpoint": checkpoint,
}
handler._resolve_company_runtime_target = AsyncMock(
side_effect=[target, None],
)
mismatched = await handler._route_company_suspend_reply_if_pending(
task_id="worker",
content="continue",
session_id="runtime-session:role:worker",
task=tasks[2],
attachment_refs=None,
message_metadata={
"response_to_checkpoint_id": "wrong-checkpoint",
"response_to_checkpoint_type": "company_runtime_interrupted",
},
user_message_id=None,
user_message_created_at=None,
run_engine=engine,
run_project_id="project-a",
)
assert mismatched is True
handler._track.assert_not_called()
checkpoint.status = "resuming"
handler._resolve_company_runtime_target = AsyncMock(return_value=target)
resuming = await handler._route_company_suspend_reply_if_pending(
task_id="worker",
content="continue again",
session_id="runtime-session:role:worker",
task=tasks[2],
attachment_refs=None,
message_metadata=None,
user_message_id=None,
user_message_created_at=None,
run_engine=engine,
run_project_id="project-a",
)
assert resuming is True
handler._track.assert_not_called()
asyncio.run(scenario())
def test_escalation_control_uses_durable_anchor_not_transient_progress_maps() -> None:
async def scenario() -> None:
tasks, checkpoint = _runtime_records()
class Store:
async def get_task(self, task_id: str):
return next((task for task in tasks if task.id == task_id), None)
async def get_tasks(self, **_kwargs):
return tasks
async def get_execution_checkpoints(self, **_kwargs):
return [checkpoint]
handler = WSHandler.__new__(WSHandler)
handler.engine = SimpleNamespace(project_id="project-a", store=Store())
handler._active_runtime_children = {"worker": "wrong-parent"}
handler._session_to_task = {"runtime-session": "wrong-parent"}
handler._ui_task_aliases = {}
resolved = await handler._resolve_escalation_session_task_id("worker")
assert resolved == "ui-anchor"
asyncio.run(scenario())
+42 -21
View File
@@ -74,6 +74,19 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
external_provider_session_id: str = "provider-session-1",
) -> tuple[CompanyWorkItemRuntimePlan, Task]:
plan = self._plan(profile)
await store.save_task(
Task(
id=f"ui-anchor-{parent_session_id}",
title="Company chat",
session_id=parent_session_id,
project_id="proj1",
status=TaskStatus.IDLE,
metadata={
"exec_mode": "company",
"company_profile": profile,
},
)
)
await store.save_delegation_work_item(
DelegationWorkItem(
work_item_id=work_item_id,
@@ -1445,7 +1458,7 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
self.assertNotIn("external_resume_session_id", resumed_task.metadata)
self.assertEqual(resumed_task.metadata["external_resume_fallback"], "context_replay")
async def test_company_runtime_checkpoint_resolves_before_long_execute(self) -> None:
async def test_company_runtime_checkpoint_stays_resuming_during_long_execute(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
engine = self._engine(store)
@@ -1472,7 +1485,7 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
)
captured["resuming_count_during_execute"] = len(resuming)
captured["resolved_count_during_execute"] = len(resolved)
captured["resume_state_during_execute"] = resolved[0].payload.get("resume_state") if resolved else ""
captured["resume_state_during_execute"] = resuming[0].payload.get("resume_state") if resuming else ""
return "runtime resumed"
engine.company_executor = DummyCompanyExecutor()
@@ -1489,9 +1502,9 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
statuses=["resolved"],
)
self.assertEqual(captured["resuming_count_during_execute"], 0)
self.assertEqual(captured["resolved_count_during_execute"], 1)
self.assertEqual(captured["resume_state_during_execute"], "handoff_complete")
self.assertEqual(captured["resuming_count_during_execute"], 1)
self.assertEqual(captured["resolved_count_during_execute"], 0)
self.assertEqual(captured["resume_state_during_execute"], "resuming")
self.assertEqual(len(resolved), 1)
self.assertEqual(resolved[0].payload.get("resume_state"), "handoff_complete")
@@ -1537,10 +1550,24 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
checkpoint_types=["company_runtime_suspended"],
statuses=["resuming"],
)
refreshed_task = await store.get_task(task.id)
refreshed_item = await store.get_delegation_work_item("work-item-1")
self.assertEqual(len(pending), 1)
self.assertEqual(resuming, [])
self.assertEqual(pending[0].payload.get("resume_state"), "failed_before_handoff")
assert refreshed_task is not None
assert refreshed_item is not None
self.assertEqual(
refreshed_task.metadata.get("dispatch_hold"),
"company_runtime_suspended",
)
self.assertEqual(
refreshed_item.metadata.get("dispatch_hold"),
"company_runtime_suspended",
)
self.assertEqual(refreshed_item.claimed_by_role_runtime_session_id, "")
self.assertEqual(refreshed_item.claimed_by_seat_id, "")
async def test_suspend_is_parent_session_idempotent(self) -> None:
store = await self._store()
@@ -1831,22 +1858,16 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
async def test_continue_clears_parent_runtime_stop_marker(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
parent = Task(
id="parent-task",
title="Parent company runtime",
session_id="sess-parent",
parent_session_id="",
status=TaskStatus.RUNNING,
project_id="proj1",
metadata={
"exec_mode": "company",
"company_profile": "corporate",
"company_runtime_stop_state": "suspended",
"company_runtime_stop_intent_id": "intent-1",
"company_runtime_stop_marked_at": "2026-04-29T11:02:40",
"company_runtime_suspended_at": "2026-04-29T11:02:40",
},
)
parent = await store.get_task("ui-anchor-sess-parent")
assert parent is not None
parent.status = TaskStatus.RUNNING
parent.metadata = {
**dict(parent.metadata or {}),
"company_runtime_stop_state": "suspended",
"company_runtime_stop_intent_id": "intent-1",
"company_runtime_stop_marked_at": "2026-04-29T11:02:40",
"company_runtime_suspended_at": "2026-04-29T11:02:40",
}
await store.save_task(parent)
engine = self._engine(store)
await engine.suspend_company_runtime(
-22
View File
@@ -1,22 +0,0 @@
from __future__ import annotations
import os
import unittest
from unittest.mock import patch
from opc import engine as engine_module
from opc.engine import OPCEngine
class EnginePidProbeTests(unittest.TestCase):
def test_current_process_is_running(self) -> None:
self.assertTrue(OPCEngine._pid_is_running(os.getpid()))
def test_posix_probe_treats_unexpected_oserror_as_not_running(self) -> None:
with patch.object(engine_module.os, "name", "posix"), patch.object(
engine_module.os,
"kill",
side_effect=OSError("platform probe failed"),
):
self.assertFalse(OPCEngine._pid_is_running(12345))
File diff suppressed because it is too large Load Diff
+295
View File
@@ -0,0 +1,295 @@
from __future__ import annotations
import asyncio
from datetime import timedelta
from opc.core.models import ExecutionCheckpoint, Task, TaskStatus
from opc.database.store import OPCStore
def test_checkpoint_compare_and_set_has_one_winner_across_store_connections(
tmp_path,
) -> None:
async def scenario() -> None:
db_path = tmp_path / "tasks.db"
first_store = OPCStore(db_path)
second_store = OPCStore(db_path)
await first_store.initialize()
await second_store.initialize()
try:
checkpoint = ExecutionCheckpoint(
checkpoint_id="checkpoint-1",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id="runtime-task",
payload={"reason": "service_restart"},
)
await first_store.save_execution_checkpoint(checkpoint)
start = asyncio.Event()
payloads = [
{"reason": "service_restart", "claimed_by": "office"},
{"reason": "service_restart", "claimed_by": "cli"},
]
async def claim(store: OPCStore, payload: dict[str, str]) -> bool:
await start.wait()
return await store.compare_and_set_execution_checkpoint(
checkpoint.checkpoint_id,
expected_statuses={"pending"},
status="resuming",
payload=payload,
)
claims = [
asyncio.create_task(claim(first_store, payloads[0])),
asyncio.create_task(claim(second_store, payloads[1])),
]
start.set()
results = await asyncio.gather(*claims)
assert results.count(True) == 1
assert results.count(False) == 1
winner = results.index(True)
rows = await first_store.get_execution_checkpoints(
project_id="project-a",
session_id="runtime-session",
)
assert len(rows) == 1
assert rows[0].status == "resuming"
assert rows[0].payload == payloads[winner]
finally:
await second_store.close()
await first_store.close()
asyncio.run(scenario())
def test_checkpoint_get_or_create_has_one_active_row_across_store_connections(
tmp_path,
) -> None:
async def scenario() -> None:
db_path = tmp_path / "tasks.db"
first_store = OPCStore(db_path)
second_store = OPCStore(db_path)
await first_store.initialize()
await second_store.initialize()
try:
checkpoint_types = {
"company_runtime_suspended",
"company_runtime_interrupted",
}
candidates = [
ExecutionCheckpoint(
checkpoint_id="checkpoint-office",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_interrupted",
task_id="runtime-task",
payload={"creator": "office"},
),
ExecutionCheckpoint(
checkpoint_id="checkpoint-cli",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_suspended",
task_id="runtime-task",
payload={"creator": "cli"},
),
]
start = asyncio.Event()
async def create(
store: OPCStore,
candidate: ExecutionCheckpoint,
) -> tuple[ExecutionCheckpoint, bool]:
await start.wait()
return await store.get_or_create_active_execution_checkpoint(
candidate,
checkpoint_types=checkpoint_types,
)
attempts = [
asyncio.create_task(create(first_store, candidates[0])),
asyncio.create_task(create(second_store, candidates[1])),
]
start.set()
results = await asyncio.gather(*attempts)
assert [created for _, created in results].count(True) == 1
assert [created for _, created in results].count(False) == 1
assert len({row.checkpoint_id for row, _ in results}) == 1
active = await first_store.get_execution_checkpoints(
project_id="project-a",
session_id="runtime-session",
checkpoint_types=list(checkpoint_types),
statuses=["pending", "resuming"],
)
assert len(active) == 1
assert active[0].checkpoint_id == results[0][0].checkpoint_id
finally:
await second_store.close()
await first_store.close()
asyncio.run(scenario())
def test_checkpoint_get_or_create_normalizes_historical_active_duplicates(
tmp_path,
) -> None:
async def scenario() -> None:
store = OPCStore(tmp_path / "tasks.db")
await store.initialize()
try:
older = ExecutionCheckpoint(
checkpoint_id="checkpoint-older",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_interrupted",
status="resuming",
payload={"created": "older"},
)
newer = ExecutionCheckpoint(
checkpoint_id="checkpoint-newer",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_suspended",
payload={"created": "newer"},
)
newer.updated_at = older.updated_at + timedelta(microseconds=1)
await store.save_execution_checkpoint(older)
await store.save_execution_checkpoint(newer)
winner, created = await store.get_or_create_active_execution_checkpoint(
ExecutionCheckpoint(
checkpoint_id="checkpoint-unused",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_interrupted",
),
checkpoint_types={
"company_runtime_suspended",
"company_runtime_interrupted",
},
)
assert created is False
assert winner.checkpoint_id == "checkpoint-newer"
active = await store.get_execution_checkpoints(
project_id="project-a",
session_id="runtime-session",
statuses=["pending", "resuming"],
)
assert [row.checkpoint_id for row in active] == ["checkpoint-newer"]
all_rows = await store.get_execution_checkpoints(
project_id="project-a",
session_id="runtime-session",
)
by_id = {row.checkpoint_id: row for row in all_rows}
assert by_id["checkpoint-older"].status == "superseded"
assert (
by_id["checkpoint-older"].payload["superseded_by_checkpoint_id"]
== "checkpoint-newer"
)
finally:
await store.close()
asyncio.run(scenario())
def test_checkpoint_completion_and_cancelled_anchor_reopen_are_atomic(tmp_path) -> None:
async def scenario() -> None:
store = OPCStore(tmp_path / "tasks.db")
await store.initialize()
try:
anchor = Task(
id="ui-anchor",
project_id="project-a",
session_id="runtime-session",
title="Company chat",
status=TaskStatus.CANCELLED,
execution_lock=True,
)
checkpoint = ExecutionCheckpoint(
checkpoint_id="checkpoint-1",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_interrupted",
status="resuming",
payload={"ui_anchor_task_id": anchor.id},
)
await store.save_task(anchor)
await store.save_execution_checkpoint(checkpoint)
completed = await store.complete_execution_checkpoint_and_reopen_ui_anchor(
checkpoint.checkpoint_id,
project_id="project-a",
session_id="runtime-session",
expected_status="resuming",
status="resolved",
payload={"resume_state": "handoff_complete"},
ui_anchor_task_id=anchor.id,
)
assert completed is True
assert (await store.get_task(anchor.id)).status == TaskStatus.IDLE
rows = await store.get_execution_checkpoints(
project_id="project-a",
session_id="runtime-session",
)
assert rows[0].status == "resolved"
assert rows[0].payload == {"resume_state": "handoff_complete"}
finally:
await store.close()
asyncio.run(scenario())
def test_checkpoint_completion_does_not_reopen_anchor_after_stop_wins(tmp_path) -> None:
async def scenario() -> None:
store = OPCStore(tmp_path / "tasks.db")
await store.initialize()
try:
anchor = Task(
id="ui-anchor",
project_id="project-a",
session_id="runtime-session",
title="Company chat",
status=TaskStatus.CANCELLED,
)
checkpoint = ExecutionCheckpoint(
checkpoint_id="checkpoint-1",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_interrupted",
status="pending",
payload={"resume_state": "interrupted"},
)
await store.save_task(anchor)
await store.save_execution_checkpoint(checkpoint)
completed = await store.complete_execution_checkpoint_and_reopen_ui_anchor(
checkpoint.checkpoint_id,
project_id="project-a",
session_id="runtime-session",
expected_status="resuming",
status="resolved",
payload={"resume_state": "handoff_complete"},
ui_anchor_task_id=anchor.id,
)
assert completed is False
assert (await store.get_task(anchor.id)).status == TaskStatus.CANCELLED
rows = await store.get_execution_checkpoints(
project_id="project-a",
session_id="runtime-session",
)
assert rows[0].status == "pending"
assert rows[0].payload == {"resume_state": "interrupted"}
finally:
await store.close()
asyncio.run(scenario())
+380
View File
@@ -0,0 +1,380 @@
from __future__ import annotations
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock
from opc.core.active_task_runs import ActiveTaskRunRegistry
from opc.core.models import ExecutionCheckpoint, Task
from opc.plugins.office_ui.ws_handler import WSHandler
def test_ws_shutdown_checkpoints_before_cancelling_and_awaiting_sessions() -> None:
async def scenario() -> None:
events: list[str] = []
started = asyncio.Event()
async def execution() -> None:
started.set()
try:
await asyncio.Event().wait()
finally:
events.append("execution_finally")
async def prepare() -> list[dict]:
events.append("checkpoint")
assert not execution_task.done()
return []
execution_task = asyncio.create_task(execution())
await started.wait()
handler = WSHandler.__new__(WSHandler)
handler.engine = SimpleNamespace()
handler._root_engine = SimpleNamespace(
prepare_active_company_runtimes_for_shutdown=prepare,
)
handler._shutting_down = False
handler._progress_flush_task = None
handler._clients = set()
handler._active_message_tasks = set()
handler._background_tasks = {execution_task}
handler._task_bg_context = {execution_task: {"task_id": "runtime-task"}}
handler._task_bg_map = {"runtime-task": {execution_task}}
await handler.shutdown(timeout=1.0)
assert events == ["checkpoint", "execution_finally"]
assert execution_task.done()
asyncio.run(scenario())
def test_ws_shutdown_checkpoint_failure_does_not_cancel_execution_or_close_the_gap() -> None:
async def scenario() -> None:
released = asyncio.Event()
async def execution() -> None:
try:
await asyncio.Event().wait()
finally:
released.set()
execution_task = asyncio.create_task(execution())
await asyncio.sleep(0)
handler = WSHandler.__new__(WSHandler)
handler.engine = SimpleNamespace()
handler._root_engine = SimpleNamespace(
prepare_active_company_runtimes_for_shutdown=AsyncMock(
side_effect=RuntimeError("checkpoint unavailable")
),
)
handler._shutting_down = False
handler._progress_flush_task = None
handler._clients = set()
handler._active_message_tasks = set()
handler._background_tasks = {execution_task}
handler._task_bg_context = {execution_task: {"task_id": "runtime-task"}}
handler._task_bg_map = {"runtime-task": {execution_task}}
try:
await handler.shutdown(timeout=1.0)
except RuntimeError as exc:
assert str(exc) == "checkpoint unavailable"
else:
raise AssertionError("shutdown must fail closed when checkpointing fails")
assert not released.is_set()
assert not execution_task.done()
execution_task.cancel()
await asyncio.gather(execution_task, return_exceptions=True)
asyncio.run(scenario())
def test_ws_shutdown_rejects_background_work_scheduled_by_late_ingress() -> None:
async def scenario() -> None:
entered = False
async def late_work() -> None:
nonlocal entered
entered = True
handler = WSHandler.__new__(WSHandler)
handler._shutting_down = True
handler._background_tasks = set()
task = handler._track(late_work())
await asyncio.gather(task, return_exceptions=True)
await asyncio.sleep(0)
assert task.cancelled()
assert entered is False
assert task not in handler._background_tasks
asyncio.run(scenario())
def test_ws_shutdown_drains_queued_duplicate_handoff_before_checkpointing() -> None:
async def scenario() -> None:
registry = ActiveTaskRunRegistry()
runtime_lock = asyncio.Lock()
execution_registered = asyncio.Event()
prepare_called = asyncio.Event()
execution_released = asyncio.Event()
async def prepare() -> list[dict]:
assert registry.is_active("project-a", "runtime-task")
assert registry.pending_handoff_count == 0
prepare_called.set()
return []
root_engine = SimpleNamespace(
_active_task_run_registry=registry,
prepare_active_company_runtimes_for_shutdown=prepare,
)
handler = WSHandler.__new__(WSHandler)
handler.engine = root_engine
handler._root_engine = root_engine
handler._shutting_down = False
handler._progress_flush_task = None
handler._clients = set()
handler._active_message_tasks = set()
handler._background_tasks = set()
handler._task_bg_context = {}
handler._task_bg_map = {}
handler._handoff_route_tasks = {}
async def execution() -> None:
async with runtime_lock:
attempt_token = registry.register("project-a", "runtime-task")
execution_registered.set()
try:
await asyncio.Event().wait()
finally:
registry.unregister("project-a", "runtime-task", attempt_token)
execution_released.set()
async def queued_duplicate() -> None:
async with runtime_lock:
attempt_token = registry.register("project-a", "runtime-task")
try:
await asyncio.Event().wait()
finally:
registry.unregister("project-a", "runtime-task", attempt_token)
first_handoff = registry.reserve_handoff()
with registry.bind_handoff(first_handoff):
first = handler._track_session(
"runtime-task",
execution(),
project_id="project-a",
engine=root_engine,
)
registry.release_handoff(first_handoff)
await execution_registered.wait()
second_handoff = registry.reserve_handoff()
with registry.bind_handoff(second_handoff):
second = handler._track_session(
"runtime-task",
queued_duplicate(),
project_id="project-a",
engine=root_engine,
)
registry.release_handoff(second_handoff)
await asyncio.sleep(0)
assert registry.pending_handoff_count == 1
await asyncio.wait_for(handler.shutdown(timeout=1.0), timeout=1.0)
assert prepare_called.is_set()
assert execution_released.is_set()
assert first.cancelled()
assert second.cancelled()
assert registry.pending_handoff_count == 0
assert not registry.is_active("project-a", "runtime-task")
asyncio.run(scenario())
def test_ws_shutdown_fails_closed_while_execution_cleanup_is_still_running() -> None:
async def scenario() -> None:
cancellation_started = asyncio.Event()
allow_cleanup = asyncio.Event()
async def execution() -> None:
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
cancellation_started.set()
await allow_cleanup.wait()
execution_task = asyncio.create_task(execution())
await asyncio.sleep(0)
handler = WSHandler.__new__(WSHandler)
handler.engine = SimpleNamespace()
handler._root_engine = SimpleNamespace(
prepare_active_company_runtimes_for_shutdown=AsyncMock(return_value=[]),
)
handler._shutting_down = False
handler._progress_flush_task = None
handler._clients = set()
handler._active_message_tasks = set()
handler._background_tasks = {execution_task}
handler._task_bg_context = {
execution_task: {"task_id": "runtime-task", "execution_handoff": True}
}
handler._task_bg_map = {"runtime-task": {execution_task}}
try:
await handler.shutdown(timeout=0.01)
except RuntimeError as exc:
assert "execution task(s)" in str(exc)
else:
raise AssertionError("shutdown must not close resources before execution cleanup")
assert cancellation_started.is_set()
assert not execution_task.done()
allow_cleanup.set()
await execution_task
asyncio.run(scenario())
def test_ws_shutdown_cancels_execution_before_waiting_for_client_close() -> None:
async def scenario() -> None:
execution_released = asyncio.Event()
close_entered = asyncio.Event()
allow_close = asyncio.Event()
async def execution() -> None:
try:
await asyncio.Event().wait()
finally:
execution_released.set()
class BlockingWebSocket:
async def close(self, **_kwargs: object) -> None:
close_entered.set()
await allow_close.wait()
execution_task = asyncio.create_task(execution())
await asyncio.sleep(0)
client = BlockingWebSocket()
handler = WSHandler.__new__(WSHandler)
handler.engine = SimpleNamespace()
handler._root_engine = SimpleNamespace(
prepare_active_company_runtimes_for_shutdown=AsyncMock(return_value=[]),
)
handler._shutting_down = False
handler._progress_flush_task = None
handler._clients = {client}
handler._active_message_tasks = set()
handler._background_tasks = {execution_task}
handler._task_bg_context = {execution_task: {"task_id": "runtime-task"}}
handler._task_bg_map = {"runtime-task": {execution_task}}
shutdown_task = asyncio.create_task(handler.shutdown(timeout=1.0))
await close_entered.wait()
assert execution_released.is_set()
assert execution_task.done()
allow_close.set()
await shutdown_task
asyncio.run(scenario())
def test_duplicate_resume_does_not_leave_shutdown_handoff_barrier_queued() -> None:
async def scenario() -> None:
registry = ActiveTaskRunRegistry()
execution_started = asyncio.Event()
execution_released = asyncio.Event()
async def prepare() -> list[dict]:
assert registry.is_active("project-a", "runtime-task")
return []
root_engine = SimpleNamespace(
project_id="project-a",
_active_task_run_registry=registry,
prepare_active_company_runtimes_for_shutdown=prepare,
)
handler = WSHandler.__new__(WSHandler)
handler.engine = root_engine
handler._root_engine = root_engine
handler.chat_store = None
handler._shutting_down = False
handler._progress_flush_task = None
handler._clients = set()
handler._active_message_tasks = set()
handler._background_tasks = set()
handler._task_bg_context = {}
handler._task_bg_map = {}
handler._company_stop_finalize_tasks = {}
handler._company_suspend_reply_locks = {}
checkpoint = ExecutionCheckpoint(
checkpoint_id="checkpoint-1",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_interrupted",
status="pending",
)
task = Task(
id="ui-task",
title="Company chat",
project_id="project-a",
session_id="runtime-session",
metadata={"exec_mode": "company"},
)
target = {
"runtime_session_id": "runtime-session",
"checkpoint": checkpoint,
}
handler._resolve_company_runtime_target = AsyncMock(return_value=target)
async def fake_resume(**_kwargs: object) -> None:
attempt = registry.register("project-a", "runtime-task")
execution_started.set()
try:
await asyncio.Event().wait()
finally:
registry.unregister("project-a", "runtime-task", attempt)
execution_released.set()
handler._process_company_suspend_reply = fake_resume
async def route_once() -> bool:
handoff = registry.reserve_handoff()
try:
with registry.bind_handoff(handoff):
return await handler._route_company_suspend_reply_if_pending(
task_id=task.id,
content="continue",
session_id=task.session_id,
task=task,
attachment_refs=None,
message_metadata=None,
user_message_id=None,
user_message_created_at=None,
run_engine=root_engine,
run_project_id="project-a",
)
finally:
registry.release_handoff(handoff)
assert await route_once() is True
await execution_started.wait()
assert await route_once() is True
assert registry.pending_handoff_count == 0
assert len(handler._background_tasks) == 1
await asyncio.wait_for(handler.shutdown(timeout=1.0), timeout=1.0)
assert execution_released.is_set()
assert registry.pending_handoff_count == 0
asyncio.run(scenario())
+8
View File
@@ -4,6 +4,7 @@ import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock
from opc.core.active_task_runs import ActiveTaskRunRegistry
from opc.core.config import OPCConfig, RoleConfig
from opc.core.org_config import (
build_org_config_payload_from_config,
@@ -137,6 +138,10 @@ def test_custom_runtime_initializes_with_parent_store(monkeypatch, tmp_path) ->
self.store = kwargs.get("store") or object()
self.owns_store = kwargs.get("owns_store")
self.run_startup_reconcile = kwargs.get("run_startup_reconcile")
self.active_task_run_registry = kwargs.get("active_task_run_registry")
self.owns_active_task_run_registry = kwargs.get(
"owns_active_task_run_registry"
)
self.bound_stores = []
self.message_bus = FakeMessageBus(self)
self.company_executor = SimpleNamespace(_signal_dispatcher_wake=lambda: None)
@@ -164,6 +169,7 @@ def test_custom_runtime_initializes_with_parent_store(monkeypatch, tmp_path) ->
parent.config = OPCConfig()
parent.project_id = None
parent.store = parent_store
parent._active_task_run_registry = ActiveTaskRunRegistry()
parent.on_progress = None
parent.on_runtime_event = None
parent.on_escalation = None
@@ -198,6 +204,8 @@ def test_custom_runtime_initializes_with_parent_store(monkeypatch, tmp_path) ->
assert runtime.store is parent_store
assert runtime.owns_store is False
assert runtime.run_startup_reconcile is False
assert runtime.active_task_run_registry is parent._active_task_run_registry
assert runtime.owns_active_task_run_registry is False
assert runtime.bound_stores == []
assert captured["kanban_callback_runtime"] is runtime
assert callable(runtime.company_executor.on_kanban_changed)
@@ -188,10 +188,8 @@ class DirectStatusWriteLintTest(unittest.TestCase):
PATTERN = re.compile(r"\.status\s*=\s*TaskStatus\.(CANCELLED|FAILED)")
MIGRATED_COMPANY_AWARE_FILES = (
"opc/plugins/office_ui/dispatcher.py",
"opc/plugins/office_ui/recovery_manager.py",
"opc/plugins/office_ui/ws_handler.py",
"opc/plugins/cli_board/services/actions.py",
"opc/plugins/cli_board/services/recovery.py",
)
MIGRATED_FIVE_STATUS_PATTERN = re.compile(
r"\.status\s*=\s*TaskStatus\.(PENDING|RUNNING|DONE|FAILED|CANCELLED)"
+517 -28
View File
@@ -27,6 +27,7 @@ from opc.database.store import _SQLiteConnectionAdapter
from opc.layer2_organization import comms as file_comms
from opc.plugins.office_ui.event_adapter import EventAdapter
from opc.plugins.office_ui.chat_store import ChatStore
from opc.plugins.office_ui.services.models import ServiceError
# ═══════════════════════════════════════════════════════════════════════
@@ -1823,6 +1824,126 @@ class TestWSHandlerSessionSend(unittest.IsolatedAsyncioTestCase):
self.assertEqual(len(ended), 1)
self.handler._dispatch_session_message.assert_not_called()
async def _seed_cancelled_markerless_company_anchor(self) -> ExecutionCheckpoint:
anchor = await self.store.get_task(self.task_id)
assert anchor is not None
anchor.status = TaskStatus.CANCELLED
anchor.metadata = {}
await self.store.save_task(anchor)
await self.store.save_task(Task(
id="shared-final-decider",
title="Final decision",
project_id="test-project",
session_id=self.session_id,
parent_session_id=self.session_id,
linked_work_item_id="work-item-final",
status=TaskStatus.BLOCKED,
metadata={
"exec_mode": "company",
"company_profile": "corporate",
"work_item_runtime": True,
"work_item_projection_id": "final",
"shared_role_session": True,
},
))
checkpoint = ExecutionCheckpoint(
checkpoint_id="cp-markerless-anchor",
project_id="test-project",
session_id=self.session_id,
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id="shared-final-decider",
payload={"parent_session_id": self.session_id},
)
await self.store.save_execution_checkpoint(checkpoint)
return checkpoint
async def test_cancelled_markerless_company_anchor_text_routes_by_checkpoint_identity(self) -> None:
checkpoint = await self._seed_cancelled_markerless_company_anchor()
self.handler._process_company_suspend_reply = AsyncMock()
fake_bg = object()
def _close_and_track(_task_id: str, coro: Any, **_kwargs: Any) -> object:
coro.close()
self.handler._task_bg_context[fake_bg] = {}
return fake_bg
self.handler._track_session = MagicMock(side_effect=_close_and_track)
ws = MagicMock()
await self.handler._handle_session_send(ws, {
"project_id": "test-project",
"task_id": self.task_id,
"content": "继续恢复这个 runtime。",
})
self.handler._process_company_suspend_reply.assert_called_once()
routed = self.handler._process_company_suspend_reply.call_args.kwargs
self.assertEqual(routed["ui_task_id"], self.task_id)
self.assertEqual(routed["runtime_session_id"], self.session_id)
self.assertEqual(routed["checkpoint"].checkpoint_id, checkpoint.checkpoint_id)
self.handler._track_session.assert_called_once()
anchor = await self.store.get_task(self.task_id)
assert anchor is not None
self.assertEqual(anchor.status, TaskStatus.CANCELLED)
async def test_cancelled_company_anchor_with_resuming_checkpoint_is_not_ended_early(self) -> None:
checkpoint = await self._seed_cancelled_markerless_company_anchor()
checkpoint.status = "resuming"
await self.store.save_execution_checkpoint(checkpoint)
self.handler._process_company_suspend_reply = AsyncMock()
self.handler._process_session_message = AsyncMock()
ws = MagicMock()
await self.handler._handle_session_send(ws, {
"project_id": "test-project",
"task_id": self.task_id,
"content": "不要重复恢复。",
})
ended = [
call for call in self.handler._send_ack.await_args_list
if call.kwargs.get("error") == "session_ended"
]
self.assertEqual(ended, [])
self.handler._process_company_suspend_reply.assert_not_called()
self.handler._process_session_message.assert_not_called()
async def test_cancelled_markerless_company_anchor_button_routes_by_checkpoint_identity(self) -> None:
checkpoint = await self._seed_cancelled_markerless_company_anchor()
self.handler._process_company_suspend_reply = AsyncMock()
fake_bg = object()
def _close_and_track(_task_id: str, coro: Any, **_kwargs: Any) -> object:
coro.close()
self.handler._task_bg_context[fake_bg] = {}
return fake_bg
self.handler._track_session = MagicMock(side_effect=_close_and_track)
ws = MagicMock()
await self.handler._handle_session_resume(ws, {
"project_id": "test-project",
"task_id": self.task_id,
"runtime_session_id": self.session_id,
"checkpoint_id": checkpoint.checkpoint_id,
})
self.handler._process_company_suspend_reply.assert_called_once()
routed = self.handler._process_company_suspend_reply.call_args.kwargs
self.assertEqual(routed["ui_task_id"], self.task_id)
self.assertEqual(routed["runtime_session_id"], self.session_id)
self.assertEqual(routed["checkpoint"].checkpoint_id, checkpoint.checkpoint_id)
self.handler._send_ack.assert_awaited_with(
ws,
ok=True,
runtime_session_id=self.session_id,
checkpoint_id=checkpoint.checkpoint_id,
)
anchor = await self.store.get_task(self.task_id)
assert anchor is not None
self.assertEqual(anchor.status, TaskStatus.CANCELLED)
async def test_done_company_session_send_is_reopened_for_followup(self) -> None:
"""Completed company chats can continue in the same CEO/company context."""
ws = MagicMock()
@@ -1980,23 +2101,23 @@ class TestWSHandlerSessionSend(unittest.IsolatedAsyncioTestCase):
}
await self.store.save_task(task)
self.engine.get_active_company_runtime_suspend_checkpoint = AsyncMock(
return_value=SimpleNamespace(
checkpoint_id="cp-suspended",
checkpoint_type="company_runtime_suspended",
status="pending",
payload={},
)
)
await self.store.save_execution_checkpoint(ExecutionCheckpoint(
checkpoint_id="cp-suspended",
project_id="test-project",
session_id=self.session_id,
checkpoint_type="company_runtime_suspended",
status="pending",
payload={"parent_session_id": self.session_id},
))
self.handler._process_company_suspend_reply = AsyncMock()
fake_bg = object()
def _close_and_track(coro: Any) -> object:
def _close_and_track(_task_id: str, coro: Any, **_kwargs: Any) -> object:
coro.close()
self.handler._task_bg_context[fake_bg] = {}
return fake_bg
self.handler._track = MagicMock(side_effect=_close_and_track)
self.handler._track_session = MagicMock()
self.handler._track_session = MagicMock(side_effect=_close_and_track)
await self.handler._handle_session_send(ws, {
"project_id": "test-project",
@@ -2004,13 +2125,39 @@ class TestWSHandlerSessionSend(unittest.IsolatedAsyncioTestCase):
"content": "改成 Sapphire Tide Runner,并让 CEO 自己修改/删除/新增 work item。",
})
self.handler._track_session.assert_not_called()
self.handler._track_session.assert_called_once()
self.handler._process_company_suspend_reply.assert_called_once()
call = self.handler._process_company_suspend_reply.call_args.kwargs
self.assertEqual(call["parent_task_id"], self.task_id)
self.assertEqual(call["parent_session_id"], self.session_id)
self.assertEqual(call["ui_task_id"], self.task_id)
self.assertEqual(call["runtime_session_id"], self.session_id)
self.assertEqual(call["checkpoint"].checkpoint_id, "cp-suspended")
self.assertEqual(call["content"], "改成 Sapphire Tide Runner,并让 CEO 自己修改/删除/新增 work item。")
async def test_rejected_company_resume_refreshes_optimistic_runtime_control(self) -> None:
task = await self.store.get_task(self.task_id)
assert task is not None
task.metadata = {"exec_mode": "company", "company_profile": "corporate"}
await self.store.save_task(task)
self.handler._refresh_runtime_control_for_client = AsyncMock()
ws = MagicMock()
await self.handler._handle_session_resume(ws, {
"project_id": "test-project",
"task_id": self.task_id,
"runtime_session_id": self.session_id,
})
self.handler._send_ack.assert_awaited_once_with(
ws,
ok=False,
error="missing_checkpoint_id",
)
self.handler._refresh_runtime_control_for_client.assert_awaited_once_with(
ws,
engine=self.engine,
project_id="test-project",
)
async def test_session_send_persists_attachment_refs_and_dispatches_them(self) -> None:
"""Uploaded session attachments should be stored and forwarded into engine execution."""
ws = MagicMock()
@@ -3264,6 +3411,37 @@ class TestWSHandlerSessionStop(unittest.IsolatedAsyncioTestCase):
self.assertTrue(any(msg["payload"]["task_id"] == "stop-parent" and msg["payload"]["status"] == "idle" for msg in status_updates))
self.assertTrue(any(msg["payload"]["task_id"] == "stop-child" and msg["payload"]["status"] == "cancelled" for msg in status_updates))
async def test_company_identity_failure_is_rejected_without_task_tree_cancel(self) -> None:
ws = MagicMock()
task = Task(
id="company-stop-mismatch",
title="Company runtime",
session_id="company-stop-session",
project_id="test-project",
status=TaskStatus.RUNNING,
metadata={"exec_mode": "company", "company_profile": "corporate"},
)
await self.store.save_task(task)
self.handler._resolve_company_runtime_target = AsyncMock(return_value=None)
self.handler._cancel_task_tree = AsyncMock()
await self.handler._handle_session_stop(
ws,
{"project_id": "test-project", "task_id": task.id},
)
self.handler._cancel_task_tree.assert_not_awaited()
self.handler._send_ack.assert_awaited_once_with(
ws,
ok=False,
error="company_runtime_identity_mismatch",
project_id="test-project",
task_id=task.id,
)
persisted = await self.store.get_task(task.id)
assert persisted is not None
self.assertEqual(persisted.status, TaskStatus.RUNNING)
# ═══════════════════════════════════════════════════════════════════════
# Test 6: WSHandler — on_opc_event child_session_created
@@ -3711,7 +3889,7 @@ class TestWSHandlerSessionDetail(unittest.IsolatedAsyncioTestCase):
"execution_checkpoint_lifecycle",
)
async def test_session_detail_includes_runtime_control_state(self) -> None:
async def test_session_detail_uses_controller_registry_for_runtime_control_state(self) -> None:
ws = MagicMock()
ws.send_json = AsyncMock()
task = Task(
@@ -3735,8 +3913,17 @@ class TestWSHandlerSessionDetail(unittest.IsolatedAsyncioTestCase):
payload = ws.send_json.await_args.args[0]["payload"]
self.assertTrue(payload["ok"])
self.assertEqual(payload["session_state"].get("runtime_control_state"), "running")
self.assertTrue(payload["session_state"].get("can_stop"))
self.assertEqual(payload["session_state"].get("runtime_control_state"), "idle")
self.assertFalse(payload["session_state"].get("can_stop"))
self.engine._task_runtime_is_live = AsyncMock(return_value=True)
await self.handler._handle_session_detail(
ws,
{"project_id": "test-project", "task_id": "custom-running-task"},
)
live_payload = ws.send_json.await_args.args[0]["payload"]
self.assertEqual(live_payload["session_state"].get("runtime_control_state"), "running")
self.assertTrue(live_payload["session_state"].get("can_stop"))
async def test_session_detail_prefers_task_description_for_role_prompt_context(self) -> None:
ws = MagicMock()
@@ -4582,10 +4769,13 @@ class TestWSHandlerShutdown(unittest.IsolatedAsyncioTestCase):
ws.send_json.assert_awaited_once()
async def test_shutdown_closes_clients_and_waits_for_active_messages(self) -> None:
async def test_shutdown_closes_clients_and_cancels_non_handoff_messages(self) -> None:
ws = MagicMock()
ws.closed = False
ws.closing = False
self.handler._root_engine.prepare_active_company_runtimes_for_shutdown = AsyncMock(
return_value=[]
)
async def _close(*_args: Any, **_kwargs: Any) -> None:
ws.closed = True
@@ -4593,23 +4783,16 @@ class TestWSHandlerShutdown(unittest.IsolatedAsyncioTestCase):
ws.close = AsyncMock(side_effect=_close)
self.handler._clients.add(ws)
blocker = asyncio.Event()
async def _active_message() -> None:
await blocker.wait()
await asyncio.Event().wait()
active_task = asyncio.create_task(_active_message())
self.handler._active_message_tasks.add(active_task)
shutdown_task = asyncio.create_task(self.handler.shutdown(timeout=0.5))
await asyncio.sleep(0.05)
self.assertFalse(shutdown_task.done())
blocker.set()
await shutdown_task
await active_task
await self.handler.shutdown(timeout=0.5)
ws.close.assert_awaited_once()
self.assertTrue(active_task.cancelled())
self.assertTrue(self.handler._shutting_down)
@@ -4967,6 +5150,312 @@ class TestOfficeServiceExecutionIdentity(unittest.IsolatedAsyncioTestCase):
async def asyncTearDown(self) -> None:
await self.chat_store._db.close()
async def test_continue_uses_scope_config_for_markerless_cancelled_ui_anchor(self) -> None:
anchor = Task(
id="service-ui-anchor",
title="Company chat",
session_id="service-runtime-session",
project_id="test-project",
status=TaskStatus.CANCELLED,
metadata={},
)
final_decider = Task(
id="service-final-decider",
title="Final decision",
session_id="service-runtime-session",
parent_session_id="service-runtime-session",
project_id="test-project",
status=TaskStatus.BLOCKED,
linked_work_item_id="service-work-item",
metadata={
"exec_mode": "company",
"company_profile": "corporate",
"work_item_runtime": True,
"work_item_projection_id": "final",
"shared_role_session": True,
},
)
checkpoint = ExecutionCheckpoint(
checkpoint_id="service-runtime-checkpoint",
project_id="test-project",
session_id="service-runtime-session",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id=final_decider.id,
payload={"parent_session_id": "service-runtime-session"},
)
await self.store.save_task(anchor)
await self.store.save_task(final_decider)
await self.store.save_execution_checkpoint(checkpoint)
await self.session_service.continue_run(
project_id="test-project",
task_id=anchor.id,
runtime_session_id="service-runtime-session",
checkpoint_id=checkpoint.checkpoint_id,
content="continue",
)
call = self.engine.process_message.await_args
self.assertEqual(call.kwargs["mode"], "company")
self.assertEqual(call.kwargs["session_id"], "service-runtime-session")
self.assertEqual(call.kwargs["origin_task_id"], anchor.id)
self.assertEqual(
call.kwargs["message_metadata"]["response_to_checkpoint_id"],
checkpoint.checkpoint_id,
)
persisted_anchor = await self.store.get_task(anchor.id)
assert persisted_anchor is not None
self.assertEqual(persisted_anchor.status, TaskStatus.CANCELLED)
async def test_company_continue_preserves_requested_work_item_as_ui_channel(self) -> None:
anchor = Task(
id="service-channel-anchor",
title="Company chat",
session_id="service-channel-runtime",
project_id="test-project",
status=TaskStatus.CANCELLED,
metadata={"exec_mode": "company", "company_profile": "corporate"},
)
work_item = Task(
id="service-channel-work-item",
title="Shared final decision",
session_id="service-channel-runtime",
parent_session_id="service-channel-runtime",
project_id="test-project",
status=TaskStatus.BLOCKED,
linked_work_item_id="service-channel-wi",
metadata={
"exec_mode": "company",
"company_profile": "corporate",
"work_item_runtime": True,
"work_item_projection_id": "final",
"shared_role_session": True,
},
)
checkpoint = ExecutionCheckpoint(
checkpoint_id="service-channel-checkpoint",
project_id="test-project",
session_id="service-channel-runtime",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id=work_item.id,
payload={"parent_session_id": "service-channel-runtime"},
)
await self.store.save_task(anchor)
await self.store.save_task(work_item)
await self.store.save_execution_checkpoint(checkpoint)
result = await self.session_service.continue_run(
project_id="test-project",
task_id=work_item.id,
runtime_session_id="service-channel-runtime",
checkpoint_id=checkpoint.checkpoint_id,
content="continue",
)
call = self.engine.process_message.await_args
self.assertEqual(result.payload["task_id"], work_item.id)
self.assertEqual(call.kwargs["session_id"], "service-channel-runtime")
self.assertEqual(call.kwargs["origin_task_id"], anchor.id)
async def test_company_identity_failure_never_falls_back_to_task_mode_control(self) -> None:
from opc.plugins.office_ui.services.models import ServiceError
task = Task(
id="service-company-control",
title="Company control",
session_id="service-company-session",
project_id="test-project",
status=TaskStatus.RUNNING,
metadata={"exec_mode": "company", "company_profile": "corporate"},
)
await self.store.save_task(task)
mismatch = ServiceError(
"company_runtime_identity_mismatch",
"identity mismatch",
)
self.session_service._resolve_company_runtime_target = AsyncMock(
side_effect=mismatch,
)
with self.assertRaises(ServiceError) as stop_error:
await self.session_service.stop(
project_id="test-project",
task_id=task.id,
)
self.assertEqual(stop_error.exception.code, "company_runtime_identity_mismatch")
with self.assertRaises(ServiceError) as continue_error:
await self.session_service.continue_run(
project_id="test-project",
task_id=task.id,
)
self.assertEqual(continue_error.exception.code, "company_runtime_identity_mismatch")
persisted = await self.store.get_task(task.id)
assert persisted is not None
self.assertEqual(persisted.status, TaskStatus.RUNNING)
async def test_session_send_from_work_item_uses_runtime_checkpoint_identity(self) -> None:
anchor = Task(
id="service-send-anchor",
title="Company chat",
session_id="service-send-runtime",
project_id="test-project",
status=TaskStatus.CANCELLED,
metadata={},
)
final_decider = Task(
id="service-send-final",
title="Final decider",
session_id="service-send-runtime",
parent_session_id="service-send-runtime",
project_id="test-project",
linked_work_item_id="service-send-final-wi",
metadata={
"exec_mode": "company",
"company_profile": "corporate",
"work_item_runtime": True,
"work_item_projection_id": "final",
"shared_role_session": True,
},
)
worker = Task(
id="service-send-worker",
title="Worker",
session_id="service-send-runtime:role:worker",
parent_session_id="service-send-runtime",
project_id="test-project",
linked_work_item_id="service-send-worker-wi",
metadata={
"exec_mode": "company",
"company_profile": "corporate",
"work_item_runtime": True,
"work_item_projection_id": "worker",
},
)
original_worker_metadata = dict(worker.metadata)
checkpoint = ExecutionCheckpoint(
checkpoint_id="service-send-checkpoint",
project_id="test-project",
session_id="service-send-runtime",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id=final_decider.id,
payload={"parent_session_id": "service-send-runtime"},
)
for task in (anchor, final_decider, worker):
await self.store.save_task(task)
await self.store.save_execution_checkpoint(checkpoint)
result = await self.session_service.send(
project_id="test-project",
task_id=worker.id,
content="revise and continue",
)
call = self.engine.process_message.await_args
self.assertEqual(result.payload["task_id"], worker.id)
self.assertEqual(result.payload["session_id"], "service-send-runtime")
self.assertEqual(call.kwargs["session_id"], "service-send-runtime")
self.assertEqual(call.kwargs["origin_task_id"], anchor.id)
self.assertEqual(call.kwargs["mode"], "company")
self.assertEqual(call.kwargs["message_metadata"], {
"response_to_checkpoint_id": checkpoint.checkpoint_id,
"response_to_checkpoint_type": checkpoint.checkpoint_type,
})
persisted_worker = await self.store.get_task(worker.id)
assert persisted_worker is not None
self.assertEqual(persisted_worker.metadata, original_worker_metadata)
async def test_session_send_rejects_resuming_checkpoint_without_engine_fallback(self) -> None:
anchor = Task(
id="service-resuming-anchor",
title="Company chat",
session_id="service-resuming-runtime",
project_id="test-project",
status=TaskStatus.CANCELLED,
metadata={"exec_mode": "company", "company_profile": "corporate"},
)
checkpoint = ExecutionCheckpoint(
checkpoint_id="service-resuming-checkpoint",
project_id="test-project",
session_id="service-resuming-runtime",
checkpoint_type="company_runtime_suspended",
status="resuming",
task_id=anchor.id,
payload={"parent_session_id": "service-resuming-runtime"},
)
await self.store.save_task(anchor)
await self.store.save_execution_checkpoint(checkpoint)
with self.assertRaises(ServiceError) as raised:
await self.session_service.send(
project_id="test-project",
task_id=anchor.id,
content="continue twice",
)
self.assertEqual(
raised.exception.code,
"company_runtime_checkpoint_not_pending",
)
self.engine.process_message.assert_not_called()
async def test_session_send_rejects_cancelled_task_mode_session(self) -> None:
task = Task(
id="service-cancelled-task-mode",
title="Cancelled task chat",
session_id="service-cancelled-task-session",
project_id="test-project",
status=TaskStatus.CANCELLED,
metadata={
"mode": "task",
"execution_mode": "task_mode",
"origin_task_id": "service-cancelled-task-mode",
},
)
await self.store.save_task(task)
with self.assertRaises(ServiceError) as raised:
await self.session_service.send(
project_id="test-project",
task_id=task.id,
content="must stay cancelled",
)
self.assertEqual(raised.exception.code, "session_ended")
self.engine.process_message.assert_not_called()
async def test_session_send_company_identity_mismatch_fails_closed(self) -> None:
task = Task(
id="service-send-mismatch",
title="Company chat",
session_id="service-send-mismatch-runtime",
project_id="test-project",
metadata={"exec_mode": "company", "company_profile": "corporate"},
)
await self.store.save_task(task)
self.session_service._resolve_company_runtime_target = AsyncMock(
side_effect=ServiceError(
"company_runtime_identity_mismatch",
"identity mismatch",
),
)
with self.assertRaises(ServiceError) as raised:
await self.session_service.send(
project_id="test-project",
task_id=task.id,
content="do not fall back",
)
self.assertEqual(
raised.exception.code,
"company_runtime_identity_mismatch",
)
self.engine.process_message.assert_not_called()
async def test_session_send_prefers_persisted_org_identity_over_call_defaults(self) -> None:
task = Task(
id="task-org-send",
-1
View File
@@ -352,7 +352,6 @@ class DeliveryCardPhaseSyncTests(unittest.IsolatedAsyncioTestCase):
executor._kanban_broadcast_task = None
executor._kanban_debounce_sec = 0.2
executor.runtime = MagicMock()
executor._active_task_runs = set()
executor._runtime_invariant_issue_keys = set()
return executor
-1
View File
@@ -384,7 +384,6 @@ def test_office_ui_work_item_paths_do_not_use_legacy_projection_identity_names()
[
REPO_ROOT / "opc/plugins/office_ui/ws_handler.py",
REPO_ROOT / "opc/plugins/office_ui/snapshot_builder.py",
REPO_ROOT / "opc/plugins/office_ui/recovery_manager.py",
],
)
assert not backend_matches, _format_matches(backend_matches)