fix: unify company runtime recovery lifecycle
This commit is contained in:
@@ -0,0 +1,549 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from functools import wraps
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from opc.core.active_task_runs import (
|
||||
ActiveTaskRunAdmissionClosed,
|
||||
ActiveTaskRunRegistry,
|
||||
)
|
||||
from opc.core.models import CompanyMemberSession, Task, TaskResult, TaskStatus
|
||||
from opc.engine import OPCEngine
|
||||
from opc.layer2_organization.company_mode import CompanyWorkItemExecutor
|
||||
from opc.layer2_organization.company_runtime_identity import is_company_runtime_task
|
||||
from opc.layer2_organization.org_work_item_planner import CompanyWorkItemRuntimePlan
|
||||
|
||||
|
||||
def _async_test(func):
|
||||
@wraps(func)
|
||||
def runner(*args, **kwargs):
|
||||
return asyncio.run(func(*args, **kwargs))
|
||||
|
||||
return runner
|
||||
|
||||
|
||||
def test_overlapping_attempts_remain_active_until_last_attempt_exits() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
first = registry.register("project-a", "task-1")
|
||||
second = registry.register("project-a", "task-1")
|
||||
|
||||
assert first != second
|
||||
assert registry.attempt_count("project-a", "task-1") == 2
|
||||
assert registry.is_active("project-a", "task-1")
|
||||
assert registry.unregister("project-a", "task-1", first)
|
||||
assert registry.is_active("project-a", "task-1")
|
||||
assert registry.unregister("project-a", "task-1", second)
|
||||
assert not registry.is_active("project-a", "task-1")
|
||||
|
||||
|
||||
def test_registry_isolates_projects_with_equal_task_ids() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
token = registry.register("project-a", "task-1")
|
||||
|
||||
assert registry.active_task_ids("project-a") == {"task-1"}
|
||||
assert registry.active_task_ids("project-b") == set()
|
||||
assert not registry.is_active("project-b", "task-1")
|
||||
assert registry.unregister("project-a", "task-1", token)
|
||||
|
||||
|
||||
def test_plain_child_task_is_not_classified_as_company_runtime_scope() -> None:
|
||||
task = Task(
|
||||
id="plain-task",
|
||||
title="Plain task",
|
||||
project_id="project-a",
|
||||
parent_session_id="parent-session",
|
||||
metadata={"mode": "task", "parent_session_id": "parent-session"},
|
||||
)
|
||||
|
||||
assert not is_company_runtime_task(task)
|
||||
|
||||
|
||||
def test_closing_admission_preserves_existing_attempts_and_rejects_new_ones() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
token = registry.register("project-a", "task-1")
|
||||
|
||||
registry.close_admission()
|
||||
|
||||
assert registry.admission_closed
|
||||
assert registry.active_task_ids("project-a") == {"task-1"}
|
||||
assert registry.is_active("project-a", "task-1")
|
||||
with pytest.raises(ActiveTaskRunAdmissionClosed):
|
||||
registry.register("project-a", "task-2")
|
||||
assert registry.unregister("project-a", "task-1", token)
|
||||
|
||||
|
||||
def test_closed_admission_allows_only_nested_live_driver_attempts() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
driver_token = registry.register("project-a", "driver-task")
|
||||
|
||||
with registry.bind_driver_attempt(driver_token):
|
||||
registry.close_admission()
|
||||
nested_token = registry.register("project-a", "claimed-child")
|
||||
assert registry.is_active("project-a", "claimed-child")
|
||||
registry.unregister("project-a", "claimed-child", nested_token)
|
||||
|
||||
with pytest.raises(ActiveTaskRunAdmissionClosed):
|
||||
registry.register("project-a", "new-ingress")
|
||||
registry.unregister("project-a", "driver-task", driver_token)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_shutdown_barrier_allows_only_reserved_handoff_to_register() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
handoff_token = registry.reserve_handoff()
|
||||
|
||||
with registry.bind_handoff(handoff_token):
|
||||
barrier = asyncio.create_task(
|
||||
registry.close_admission_and_wait_for_handoffs()
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert not barrier.done()
|
||||
attempt_token = registry.register("project-a", "task-1")
|
||||
|
||||
registry.release_handoff(handoff_token)
|
||||
await asyncio.wait_for(barrier, timeout=0.1)
|
||||
|
||||
# The handoff wait ends at real coroutine registration, not at the end of
|
||||
# that execution attempt.
|
||||
assert registry.is_active("project-a", "task-1")
|
||||
assert registry.pending_handoff_count == 0
|
||||
with pytest.raises(ActiveTaskRunAdmissionClosed):
|
||||
registry.register("project-a", "late-task")
|
||||
assert registry.unregister("project-a", "task-1", attempt_token)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_shutdown_barrier_drains_request_that_exits_before_registration() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
handoff_token = registry.reserve_handoff()
|
||||
barrier = asyncio.create_task(registry.close_admission_and_wait_for_handoffs())
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert not barrier.done()
|
||||
assert registry.release_handoff(handoff_token)
|
||||
await asyncio.wait_for(barrier, timeout=0.1)
|
||||
|
||||
assert registry.pending_handoff_count == 0
|
||||
assert registry.active_task_ids("project-a") == set()
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_revoked_handoff_cannot_block_shutdown_or_register_late() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
handoff_token = registry.reserve_handoff()
|
||||
|
||||
with registry.bind_handoff(handoff_token):
|
||||
assert registry.retain_current_handoff() == handoff_token
|
||||
registry.close_admission()
|
||||
assert registry.revoke_handoff(handoff_token)
|
||||
await asyncio.wait_for(
|
||||
registry.close_admission_and_wait_for_handoffs(),
|
||||
timeout=0.1,
|
||||
)
|
||||
with pytest.raises(ActiveTaskRunAdmissionClosed):
|
||||
registry.register("project-a", "late-task")
|
||||
|
||||
assert registry.pending_handoff_count == 0
|
||||
assert not registry.release_handoff(handoff_token)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_engine_turns_closed_admission_into_infrastructure_cancellation() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
registry.close_admission()
|
||||
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
|
||||
engine._run_task_once = AsyncMock()
|
||||
task = Task(
|
||||
id="late-task",
|
||||
title="Late task",
|
||||
project_id="project-a",
|
||||
status=TaskStatus.PENDING,
|
||||
)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await engine._execute_task(task)
|
||||
|
||||
engine._run_task_once.assert_not_awaited()
|
||||
assert registry.active_task_ids("project-a") == set()
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_task_liveness_uses_registry_only() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
|
||||
engine.store = SimpleNamespace(get_latest_external_session_for_task=AsyncMock())
|
||||
task = Task(
|
||||
id="task-1",
|
||||
title="Live task",
|
||||
project_id="project-a",
|
||||
status=TaskStatus.RUNNING,
|
||||
)
|
||||
|
||||
assert not await engine._task_runtime_is_live(task)
|
||||
engine.store.get_latest_external_session_for_task.assert_not_awaited()
|
||||
|
||||
token = registry.register("project-a", task.id)
|
||||
assert await engine._task_runtime_is_live(task)
|
||||
registry.unregister("project-a", task.id, token)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_project_delegate_receives_controller_registry() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
root = OPCEngine(project_id="project-a", active_task_run_registry=registry)
|
||||
root._initialized = True
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class FakeDelegate:
|
||||
def __init__(self, **kwargs: object) -> None:
|
||||
captured.update(kwargs)
|
||||
self.store = None
|
||||
|
||||
async def initialize(self) -> None:
|
||||
return None
|
||||
|
||||
with patch("opc.engine.OPCEngine", FakeDelegate):
|
||||
delegate = await root._get_project_delegate("project-b")
|
||||
|
||||
assert delegate is root._project_engine_delegates["project-b"]
|
||||
assert captured["active_task_run_registry"] is registry
|
||||
assert captured["owns_active_task_run_registry"] is False
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_shutdown_cancellation_does_not_write_business_cancelled() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
|
||||
engine._shutting_down = True
|
||||
engine.store = SimpleNamespace(
|
||||
is_ready=True,
|
||||
get_task=AsyncMock(),
|
||||
save_task=AsyncMock(),
|
||||
)
|
||||
engine._run_task_once = AsyncMock(side_effect=asyncio.CancelledError)
|
||||
task = Task(
|
||||
id="task-1",
|
||||
title="Interrupted task",
|
||||
project_id="project-a",
|
||||
status=TaskStatus.RUNNING,
|
||||
)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await engine._execute_task(task)
|
||||
|
||||
engine.store.get_task.assert_not_awaited()
|
||||
engine.store.save_task.assert_not_awaited()
|
||||
assert not registry.is_active("project-a", task.id)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_company_cancellation_never_synthesizes_hold_without_checkpoint() -> None:
|
||||
engine = OPCEngine(project_id="project-a")
|
||||
engine.store = SimpleNamespace(
|
||||
is_ready=True,
|
||||
get_task=AsyncMock(),
|
||||
save_task=AsyncMock(),
|
||||
)
|
||||
engine._run_task_once = AsyncMock(side_effect=asyncio.CancelledError)
|
||||
task = Task(
|
||||
id="company-task",
|
||||
title="Company task",
|
||||
project_id="project-a",
|
||||
status=TaskStatus.RUNNING,
|
||||
metadata={"work_item_runtime": True},
|
||||
)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await engine._execute_task(task)
|
||||
|
||||
engine.store.get_task.assert_not_awaited()
|
||||
engine.store.save_task.assert_not_awaited()
|
||||
assert "company_runtime_suspended_at" not in task.metadata
|
||||
assert "last_stop_reason" not in task.metadata
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_suspended_checkpoint_discards_racing_task_completion() -> None:
|
||||
engine = OPCEngine(project_id="project-a")
|
||||
engine._shutting_down = False
|
||||
task = Task(
|
||||
id="company-task",
|
||||
title="Company task",
|
||||
project_id="project-a",
|
||||
status=TaskStatus.RUNNING,
|
||||
metadata={"work_item_runtime": True},
|
||||
)
|
||||
suspended = Task(
|
||||
id=task.id,
|
||||
title=task.title,
|
||||
project_id=task.project_id,
|
||||
status=TaskStatus.BLOCKED,
|
||||
metadata={
|
||||
"work_item_runtime": True,
|
||||
"dispatch_hold": "company_runtime_suspended",
|
||||
"company_runtime_stop_state": "suspended",
|
||||
},
|
||||
)
|
||||
engine.store = SimpleNamespace(
|
||||
get_task=AsyncMock(return_value=suspended),
|
||||
save_task=AsyncMock(),
|
||||
)
|
||||
engine._run_task_once = AsyncMock(
|
||||
return_value=TaskResult(status=TaskStatus.DONE, content="done", artifacts={})
|
||||
)
|
||||
engine._apply_runtime_state_to_task = MagicMock()
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await engine._execute_task(task)
|
||||
|
||||
engine.store.save_task.assert_not_awaited()
|
||||
engine._apply_runtime_state_to_task.assert_not_called()
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_attempt_stays_active_until_result_persistence_finishes() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
|
||||
save_started = asyncio.Event()
|
||||
allow_save = asyncio.Event()
|
||||
|
||||
async def blocked_save(_task: Task) -> None:
|
||||
save_started.set()
|
||||
await allow_save.wait()
|
||||
|
||||
engine.store = SimpleNamespace(
|
||||
get_task=AsyncMock(return_value=None),
|
||||
save_task=blocked_save,
|
||||
)
|
||||
engine._run_task_once = AsyncMock(
|
||||
return_value=TaskResult(status=TaskStatus.IDLE, content="done", artifacts={})
|
||||
)
|
||||
engine._apply_runtime_state_to_task = MagicMock()
|
||||
task = Task(
|
||||
id="persisting-task",
|
||||
title="Persisting task",
|
||||
project_id="project-a",
|
||||
status=TaskStatus.RUNNING,
|
||||
)
|
||||
|
||||
execution = asyncio.create_task(engine._execute_task(task))
|
||||
await save_started.wait()
|
||||
assert registry.is_active("project-a", task.id)
|
||||
allow_save.set()
|
||||
await execution
|
||||
assert not registry.is_active("project-a", task.id)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_claimed_work_item_ownership_covers_post_execution_finalize_gap() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
|
||||
engine.store = SimpleNamespace(
|
||||
get_task=AsyncMock(return_value=None),
|
||||
save_task=AsyncMock(),
|
||||
)
|
||||
engine._run_task_once = AsyncMock(
|
||||
return_value=TaskResult(status=TaskStatus.IDLE, content="done", artifacts={})
|
||||
)
|
||||
engine._apply_runtime_state_to_task = MagicMock()
|
||||
task = Task(
|
||||
id="finalizing-work-item",
|
||||
title="Finalizing work item",
|
||||
project_id="project-a",
|
||||
parent_session_id="runtime-session",
|
||||
status=TaskStatus.RUNNING,
|
||||
metadata={"work_item_runtime": True},
|
||||
)
|
||||
inner_finished = asyncio.Event()
|
||||
allow_finalize = asyncio.Event()
|
||||
executor = object.__new__(CompanyWorkItemExecutor)
|
||||
executor.active_task_run_registry = registry
|
||||
|
||||
async def run_claimed(*_args: object, **_kwargs: object) -> TaskResult:
|
||||
result = await engine._execute_task(task)
|
||||
inner_finished.set()
|
||||
await allow_finalize.wait()
|
||||
return result
|
||||
|
||||
executor._run_claimed_work_item = run_claimed
|
||||
owned = executor._create_claimed_work_item_task(
|
||||
CompanyMemberSession(
|
||||
role_id="executor",
|
||||
seat_id="seat::executor",
|
||||
member_session_id="role-session",
|
||||
),
|
||||
task,
|
||||
{},
|
||||
)
|
||||
await inner_finished.wait()
|
||||
|
||||
assert registry.attempt_count("project-a", task.id) == 1
|
||||
allow_finalize.set()
|
||||
await owned
|
||||
assert not registry.is_active("project-a", task.id)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_work_item_claim_and_spawn_share_stop_scope_lock() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
executor = object.__new__(CompanyWorkItemExecutor)
|
||||
executor.active_task_run_registry = registry
|
||||
claim_entered = asyncio.Event()
|
||||
allow_claim = asyncio.Event()
|
||||
allow_child_exit = asyncio.Event()
|
||||
stop_acquired = asyncio.Event()
|
||||
task = Task(
|
||||
id="claimed-task",
|
||||
title="Claimed task",
|
||||
project_id="project-a",
|
||||
session_id="role-session",
|
||||
parent_session_id="runtime-session",
|
||||
metadata={"work_item_runtime": True},
|
||||
)
|
||||
member_session = CompanyMemberSession(
|
||||
role_id="executor",
|
||||
seat_id="seat::executor",
|
||||
member_session_id="role-session",
|
||||
)
|
||||
|
||||
async def claim_runnable_tasks(
|
||||
_tasks: list[Task],
|
||||
*,
|
||||
work_items: list[object],
|
||||
) -> list[tuple[CompanyMemberSession, Task]]:
|
||||
del work_items
|
||||
claim_entered.set()
|
||||
await allow_claim.wait()
|
||||
return [(member_session, task)]
|
||||
|
||||
async def run_claimed(*_args: object, **_kwargs: object) -> None:
|
||||
await allow_child_exit.wait()
|
||||
|
||||
executor.runtime = SimpleNamespace(
|
||||
claim_runnable_tasks=claim_runnable_tasks,
|
||||
)
|
||||
executor._run_claimed_work_item = run_claimed
|
||||
active: dict[asyncio.Task, tuple[CompanyMemberSession, Task]] = {}
|
||||
scheduled = asyncio.create_task(
|
||||
executor._claim_and_create_work_item_tasks([task], [], active)
|
||||
)
|
||||
await claim_entered.wait()
|
||||
|
||||
async def stop_scope() -> None:
|
||||
async with registry.scope_lock("project-a", "runtime-session"):
|
||||
assert registry.is_active("project-a", task.id)
|
||||
stop_acquired.set()
|
||||
|
||||
stopping = asyncio.create_task(stop_scope())
|
||||
await asyncio.sleep(0)
|
||||
assert not stop_acquired.is_set()
|
||||
|
||||
allow_claim.set()
|
||||
await scheduled
|
||||
await stopping
|
||||
assert len(active) == 1
|
||||
|
||||
allow_child_exit.set()
|
||||
await asyncio.gather(*active)
|
||||
assert not registry.is_active("project-a", task.id)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_company_executor_driver_ownership_covers_idle_scheduler_window() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
entered = asyncio.Event()
|
||||
allow_exit = asyncio.Event()
|
||||
executor = object.__new__(CompanyWorkItemExecutor)
|
||||
executor.active_task_run_registry = registry
|
||||
|
||||
async def idle_scheduler(
|
||||
_plan: CompanyWorkItemRuntimePlan,
|
||||
_tasks: list[Task],
|
||||
) -> str:
|
||||
entered.set()
|
||||
await allow_exit.wait()
|
||||
return "done"
|
||||
|
||||
executor._execute_multi_team_org = idle_scheduler
|
||||
task = Task(
|
||||
id="driver-task",
|
||||
title="Driver task",
|
||||
project_id="project-a",
|
||||
parent_session_id="runtime-session",
|
||||
metadata={"work_item_runtime": True},
|
||||
)
|
||||
execution = asyncio.create_task(
|
||||
executor.execute(CompanyWorkItemRuntimePlan(), [task])
|
||||
)
|
||||
await entered.wait()
|
||||
|
||||
assert registry.is_active("project-a", task.id)
|
||||
allow_exit.set()
|
||||
assert await execution == "done"
|
||||
assert not registry.is_active("project-a", task.id)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_borrowed_engine_shutdown_keeps_controller_registry_open() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
root_token = registry.register("project-a", "root-attempt")
|
||||
borrowed = OPCEngine(
|
||||
project_id="project-a",
|
||||
active_task_run_registry=registry,
|
||||
owns_active_task_run_registry=False,
|
||||
)
|
||||
|
||||
await borrowed.shutdown()
|
||||
|
||||
assert not registry.admission_closed
|
||||
assert registry.is_active("project-a", "root-attempt")
|
||||
next_token = registry.register("project-a", "next-attempt")
|
||||
registry.unregister("project-a", "next-attempt", next_token)
|
||||
registry.unregister("project-a", "root-attempt", root_token)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_shutdown_preparation_includes_project_delegates() -> None:
|
||||
engine = OPCEngine(project_id="project-a")
|
||||
delegate_prepare = AsyncMock(
|
||||
return_value=[{"session_id": "delegate-session", "checkpoint_id": "checkpoint-1"}]
|
||||
)
|
||||
engine._project_engine_delegates["project-b"] = SimpleNamespace(
|
||||
prepare_active_company_runtimes_for_shutdown=delegate_prepare,
|
||||
)
|
||||
|
||||
prepared = await engine.prepare_active_company_runtimes_for_shutdown()
|
||||
|
||||
assert prepared == [{"session_id": "delegate-session", "checkpoint_id": "checkpoint-1"}]
|
||||
delegate_prepare.assert_awaited_once()
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_engine_shutdown_prepares_before_closing_subsystems() -> None:
|
||||
engine = OPCEngine(project_id="project-a")
|
||||
engine.prepare_active_company_runtimes_for_shutdown = AsyncMock(return_value=[])
|
||||
|
||||
await engine.shutdown()
|
||||
|
||||
engine.prepare_active_company_runtimes_for_shutdown.assert_awaited_once()
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_engine_shutdown_does_not_close_store_when_durable_prepare_fails() -> None:
|
||||
engine = OPCEngine(project_id="project-a")
|
||||
engine.prepare_active_company_runtimes_for_shutdown = AsyncMock(
|
||||
side_effect=RuntimeError("checkpoint failed")
|
||||
)
|
||||
engine.store = SimpleNamespace(close=AsyncMock())
|
||||
engine.message_bus.stop = MagicMock()
|
||||
|
||||
with pytest.raises(RuntimeError, match="checkpoint failed"):
|
||||
await engine.shutdown()
|
||||
|
||||
engine.message_bus.stop.assert_not_called()
|
||||
engine.store.close.assert_not_awaited()
|
||||
+264
-50
@@ -1116,6 +1116,34 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
self.assertEqual(calls, ["first", "second"])
|
||||
self.assertIn("Queued #1", console.export_text())
|
||||
|
||||
def test_chat_turn_controller_prepares_checkpoint_before_cancelling_active_turn(self) -> None:
|
||||
state, _engine = self._make_state()
|
||||
|
||||
async def _run() -> list[str]:
|
||||
order: list[str] = []
|
||||
started = asyncio.Event()
|
||||
|
||||
async def active_turn() -> None:
|
||||
started.set()
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
order.append("cancelled")
|
||||
|
||||
async def prepare() -> list[dict[str, Any]]:
|
||||
order.append("prepared")
|
||||
return []
|
||||
|
||||
state.engine.prepare_active_company_runtimes_for_shutdown = prepare
|
||||
controller = ChatTurnController(state)
|
||||
controller.active_task = asyncio.create_task(active_turn())
|
||||
await started.wait()
|
||||
|
||||
await controller.shutdown()
|
||||
return order
|
||||
|
||||
self.assertEqual(asyncio.run(_run()), ["prepared", "cancelled"])
|
||||
|
||||
def test_busy_slash_policy_allows_readonly_and_blocks_mutating_commands(self) -> None:
|
||||
self.assertEqual(_busy_slash_policy("kanban", []), BusyCommandPolicy.IMMEDIATE_READONLY)
|
||||
self.assertEqual(_busy_slash_policy("logs", ["task-1"]), BusyCommandPolicy.IMMEDIATE_READONLY)
|
||||
@@ -1168,6 +1196,17 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
"org_id": "quantum_harbor",
|
||||
"preferred_agent": "codex",
|
||||
})
|
||||
store.checkpoints = [
|
||||
SimpleNamespace(
|
||||
checkpoint_id="cp-org-interrupted",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id="task-1",
|
||||
session_id="sess-1",
|
||||
updated_at=datetime(2026, 5, 17, 12, 0),
|
||||
payload={},
|
||||
)
|
||||
]
|
||||
state, engine = self._make_state(store=store)
|
||||
state.mode = "company"
|
||||
state.company_profile = "corporate"
|
||||
@@ -1183,7 +1222,59 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
self.assertEqual(engine.calls[-1]["org_id"], "quantum_harbor")
|
||||
self.assertIsNone(engine.calls[-1]["company_profile"])
|
||||
self.assertEqual(engine.calls[-1]["preferred_agent"], "codex")
|
||||
self.assertEqual(engine.calls[-1]["message_metadata"], {"ui_force_resume": True})
|
||||
self.assertEqual(engine.calls[-1]["message_metadata"], {
|
||||
"ui_force_resume": True,
|
||||
"response_to_checkpoint_id": "cp-org-interrupted",
|
||||
"response_to_checkpoint_type": "company_runtime_interrupted",
|
||||
})
|
||||
|
||||
def test_company_continue_requires_a_durable_runtime_checkpoint(self) -> None:
|
||||
console = Console(record=True, force_terminal=False, width=120)
|
||||
store = self._Store()
|
||||
store.tasks[0].metadata.update({
|
||||
"exec_mode": "company",
|
||||
"company_profile": "corporate",
|
||||
})
|
||||
state, engine = self._make_state(store=store)
|
||||
|
||||
async def _run() -> None:
|
||||
await _handle_chat_slash_command(state, "/continue")
|
||||
|
||||
with patch("opc.cli.app.console", console):
|
||||
asyncio.run(_run())
|
||||
|
||||
self.assertEqual(engine.calls, [])
|
||||
self.assertIn("No suspended or interrupted company runtime", console.export_text())
|
||||
|
||||
def test_continue_slash_routes_to_durable_runtime_checkpoint(self) -> None:
|
||||
store = self._Store()
|
||||
store.checkpoints = [
|
||||
SimpleNamespace(
|
||||
checkpoint_id="cp-interrupted",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id="task-1",
|
||||
session_id="sess-1",
|
||||
updated_at=datetime(2026, 5, 17, 12, 0),
|
||||
payload={},
|
||||
)
|
||||
]
|
||||
state, engine = self._make_state(store=store)
|
||||
state.mode = "company"
|
||||
state.runtime_control_state = "suspended"
|
||||
|
||||
async def _run() -> None:
|
||||
await _handle_chat_slash_command(state, "/continue")
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
self.assertEqual(engine.calls[-1]["session_id"], "sess-1")
|
||||
self.assertEqual(engine.calls[-1]["message_metadata"], {
|
||||
"ui_force_resume": True,
|
||||
"response_to_checkpoint_id": "cp-interrupted",
|
||||
"response_to_checkpoint_type": "company_runtime_interrupted",
|
||||
})
|
||||
self.assertEqual(state.runtime_control_checkpoint_id, "cp-interrupted")
|
||||
|
||||
def test_plain_message_after_stop_routes_to_suspend_checkpoint(self) -> None:
|
||||
store = self._Store()
|
||||
@@ -1209,7 +1300,127 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
|
||||
self.assertEqual(engine.calls[-1]["message_metadata"]["response_to_checkpoint_id"], "cp-suspend")
|
||||
self.assertEqual(engine.calls[-1]["message_metadata"]["response_to_checkpoint_type"], "company_runtime_suspended")
|
||||
self.assertEqual(state.runtime_control_state, "running")
|
||||
self.assertEqual(state.runtime_control_state, "suspended")
|
||||
|
||||
def test_plain_message_from_company_child_uses_root_runtime_checkpoint(self) -> None:
|
||||
store = self._Store()
|
||||
now = datetime(2026, 5, 17, 12, 0)
|
||||
store.tasks[0].metadata.update({
|
||||
"exec_mode": "company",
|
||||
"mode": "company",
|
||||
"company_profile": "corporate",
|
||||
})
|
||||
store.tasks[0].parent_session_id = ""
|
||||
store.tasks.append(SimpleNamespace(
|
||||
id="worker-child",
|
||||
title="Worker",
|
||||
description="Worker turn",
|
||||
status=TaskStatus.BLOCKED,
|
||||
priority=3,
|
||||
assigned_to="worker",
|
||||
session_id="sess-1:role:worker",
|
||||
parent_session_id="sess-1",
|
||||
project_id="demo",
|
||||
created_at=now,
|
||||
tags=[],
|
||||
result={},
|
||||
linked_work_item_id="worker-item",
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"work_item_runtime": True,
|
||||
"work_item_projection_id": "worker",
|
||||
"company_runtime_root_session_id": "sess-1",
|
||||
},
|
||||
context_snapshot={},
|
||||
))
|
||||
store.checkpoints = [SimpleNamespace(
|
||||
checkpoint_id="cp-child-interrupted",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id="task-1",
|
||||
session_id="sess-1",
|
||||
project_id="demo",
|
||||
updated_at=now,
|
||||
payload={"parent_session_id": "sess-1"},
|
||||
)]
|
||||
state, engine = self._make_state(store=store)
|
||||
state.session_id = "sess-1:role:worker"
|
||||
# The selected child channel may have left the CLI's ambient mode in
|
||||
# task mode. Runtime config, not ambient state, owns resumed execution.
|
||||
state.mode = "task"
|
||||
|
||||
asyncio.run(_process_interactive_chat_message(state, "revise and continue"))
|
||||
|
||||
call = engine.calls[-1]
|
||||
self.assertEqual(call["session_id"], "sess-1")
|
||||
self.assertEqual(call["origin_task_id"], "task-1")
|
||||
self.assertEqual(call["mode"], "company")
|
||||
self.assertEqual(call["company_profile"], "corporate")
|
||||
self.assertEqual(call["message_metadata"], {
|
||||
"response_to_checkpoint_id": "cp-child-interrupted",
|
||||
"response_to_checkpoint_type": "company_runtime_interrupted",
|
||||
})
|
||||
|
||||
def test_plain_message_during_resuming_checkpoint_fails_closed(self) -> None:
|
||||
console = Console(record=True, force_terminal=False, width=140)
|
||||
store = self._Store()
|
||||
store.tasks[0].metadata.update({
|
||||
"exec_mode": "company",
|
||||
"mode": "company",
|
||||
"company_profile": "corporate",
|
||||
})
|
||||
store.checkpoints = [SimpleNamespace(
|
||||
checkpoint_id="cp-resuming",
|
||||
checkpoint_type="company_runtime_suspended",
|
||||
status="resuming",
|
||||
task_id="task-1",
|
||||
session_id="sess-1",
|
||||
project_id="demo",
|
||||
updated_at=datetime(2026, 5, 17, 12, 0),
|
||||
payload={"parent_session_id": "sess-1"},
|
||||
)]
|
||||
state, engine = self._make_state(store=store)
|
||||
state.mode = "company"
|
||||
|
||||
with patch("opc.cli.app.console", console):
|
||||
asyncio.run(_process_interactive_chat_message(state, "continue again"))
|
||||
|
||||
self.assertEqual(engine.calls, [])
|
||||
self.assertIn("checkpoint is resuming", console.export_text())
|
||||
|
||||
def test_explicit_cli_runtime_checkpoint_mismatch_fails_closed(self) -> None:
|
||||
console = Console(record=True, force_terminal=False, width=140)
|
||||
store = self._Store()
|
||||
store.tasks[0].metadata.update({
|
||||
"exec_mode": "company",
|
||||
"mode": "company",
|
||||
"company_profile": "corporate",
|
||||
})
|
||||
store.checkpoints = [SimpleNamespace(
|
||||
checkpoint_id="cp-current",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id="task-1",
|
||||
session_id="sess-1",
|
||||
project_id="demo",
|
||||
updated_at=datetime(2026, 5, 17, 12, 0),
|
||||
payload={"parent_session_id": "sess-1"},
|
||||
)]
|
||||
state, engine = self._make_state(store=store)
|
||||
state.mode = "company"
|
||||
|
||||
with patch("opc.cli.app.console", console):
|
||||
asyncio.run(_process_interactive_chat_message(
|
||||
state,
|
||||
"continue stale",
|
||||
message_metadata={
|
||||
"response_to_checkpoint_id": "cp-stale",
|
||||
"response_to_checkpoint_type": "company_runtime_interrupted",
|
||||
},
|
||||
))
|
||||
|
||||
self.assertEqual(engine.calls, [])
|
||||
self.assertIn("checkpoint identity mismatch", console.export_text())
|
||||
|
||||
def test_session_resume_still_switches_session(self) -> None:
|
||||
state, engine = self._make_state(store=self._Store())
|
||||
@@ -1287,8 +1498,6 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
self.assertEqual(output["payload"]["checkpoint_id"], "cp-stop")
|
||||
self.assertEqual(output["cancelled"], ["task-company"])
|
||||
self.assertEqual(output["suspend_calls"][0]["session_id"], "sess-company")
|
||||
self.assertEqual(output["task"].metadata["dispatch_hold"], "company_runtime_suspended")
|
||||
self.assertEqual(output["task"].metadata["company_runtime_stop_state"], "suspended")
|
||||
|
||||
def test_session_service_continue_uses_force_resume_metadata(self) -> None:
|
||||
class Store:
|
||||
@@ -1305,10 +1514,17 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"company_profile": "corporate",
|
||||
"dispatch_hold": "company_runtime_suspended",
|
||||
"company_runtime_stop_state": "suspended",
|
||||
},
|
||||
)
|
||||
self.checkpoint = SimpleNamespace(
|
||||
checkpoint_id="cp-company",
|
||||
checkpoint_type="company_runtime_suspended",
|
||||
status="pending",
|
||||
project_id="demo",
|
||||
session_id="sess-company",
|
||||
task_id="task-company",
|
||||
payload={},
|
||||
)
|
||||
|
||||
async def get_task(self, task_id: str):
|
||||
return self.task if task_id == self.task.id else None
|
||||
@@ -1319,6 +1535,9 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
async def get_session(self, session_id: str):
|
||||
return SimpleNamespace(session_id=session_id, project_id="demo")
|
||||
|
||||
async def get_execution_checkpoints(self, **_kwargs):
|
||||
return [self.checkpoint]
|
||||
|
||||
async def save_task(self, task):
|
||||
self.task = task
|
||||
|
||||
@@ -1331,7 +1550,12 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
|
||||
engine = SimpleNamespace(project_id="demo", store=Store(), process_message=process_message)
|
||||
context = OfficeServiceContext(engine=engine, agent_store=None, chat_store=None, event_adapter=None)
|
||||
result = await SessionService(context).continue_run(project_id="demo", target="sess-company")
|
||||
result = await SessionService(context).continue_run(
|
||||
project_id="demo",
|
||||
target="sess-company",
|
||||
runtime_session_id="sess-company",
|
||||
checkpoint_id="cp-company",
|
||||
)
|
||||
self.assertEqual(result.payload["response"], "resumed")
|
||||
return calls
|
||||
|
||||
@@ -1340,7 +1564,11 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
self.assertEqual(calls[-1]["content"], "Resume the existing runtime.")
|
||||
self.assertEqual(calls[-1]["session_id"], "sess-company")
|
||||
self.assertEqual(calls[-1]["mode"], "company")
|
||||
self.assertEqual(calls[-1]["message_metadata"], {"ui_force_resume": True})
|
||||
self.assertEqual(calls[-1]["message_metadata"], {
|
||||
"ui_force_resume": True,
|
||||
"response_to_checkpoint_id": "cp-company",
|
||||
"response_to_checkpoint_type": "company_runtime_suspended",
|
||||
})
|
||||
|
||||
def test_session_service_continue_preserves_custom_org_id(self) -> None:
|
||||
class Store:
|
||||
@@ -1359,10 +1587,17 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
"company_profile": "custom",
|
||||
"org_id": "quantum_harbor",
|
||||
"preferred_agent": "codex",
|
||||
"dispatch_hold": "company_runtime_suspended",
|
||||
"company_runtime_stop_state": "suspended",
|
||||
},
|
||||
)
|
||||
self.checkpoint = SimpleNamespace(
|
||||
checkpoint_id="cp-org",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
project_id="demo",
|
||||
session_id="sess-org",
|
||||
task_id="task-org",
|
||||
payload={},
|
||||
)
|
||||
|
||||
async def get_task(self, task_id: str):
|
||||
return self.task if task_id == self.task.id else None
|
||||
@@ -1373,6 +1608,9 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
async def get_session(self, session_id: str):
|
||||
return SimpleNamespace(session_id=session_id, project_id="demo")
|
||||
|
||||
async def get_execution_checkpoints(self, **_kwargs):
|
||||
return [self.checkpoint]
|
||||
|
||||
async def save_task(self, task):
|
||||
self.task = task
|
||||
|
||||
@@ -1385,7 +1623,12 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
|
||||
engine = SimpleNamespace(project_id="demo", store=Store(), process_message=process_message)
|
||||
context = OfficeServiceContext(engine=engine, agent_store=None, chat_store=None, event_adapter=None)
|
||||
result = await SessionService(context).continue_run(project_id="demo", target="sess-org")
|
||||
result = await SessionService(context).continue_run(
|
||||
project_id="demo",
|
||||
target="sess-org",
|
||||
runtime_session_id="sess-org",
|
||||
checkpoint_id="cp-org",
|
||||
)
|
||||
self.assertEqual(result.payload["response"], "resumed")
|
||||
return calls
|
||||
|
||||
@@ -1394,7 +1637,11 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
self.assertEqual(calls[-1]["mode"], "org")
|
||||
self.assertEqual(calls[-1]["org_id"], "quantum_harbor")
|
||||
self.assertIsNone(calls[-1]["company_profile"])
|
||||
self.assertEqual(calls[-1]["message_metadata"], {"ui_force_resume": True})
|
||||
self.assertEqual(calls[-1]["message_metadata"], {
|
||||
"ui_force_resume": True,
|
||||
"response_to_checkpoint_id": "cp-org",
|
||||
"response_to_checkpoint_type": "company_runtime_interrupted",
|
||||
})
|
||||
|
||||
def test_queue_slash_lists_and_drops_queued_prompts(self) -> None:
|
||||
console = Console(record=True, force_terminal=False, width=160)
|
||||
@@ -2648,49 +2895,16 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
self.assertIn("Pending Checkpoints", rendered)
|
||||
self.assertIn("cp-1", rendered)
|
||||
|
||||
def test_recover_slash_lists_and_resumes_interrupted_runtime(self) -> None:
|
||||
def test_legacy_recover_slash_is_not_registered(self) -> None:
|
||||
console = Console(record=True, force_terminal=False, width=200)
|
||||
state, _engine = self._make_state(store=self._Store())
|
||||
resume_calls: list[str] = []
|
||||
|
||||
class _FakeRecoveryManager:
|
||||
async def get_status(self):
|
||||
return SimpleNamespace(
|
||||
interrupted=[
|
||||
SimpleNamespace(
|
||||
parent_task_id="parent-task",
|
||||
parent_session_id="parent-session",
|
||||
title="Interrupted Runtime",
|
||||
profile="corporate",
|
||||
interrupted_at="2026-05-03T12:00:00",
|
||||
work_items=[
|
||||
SimpleNamespace(projection_id="wi-1", interrupted=True),
|
||||
SimpleNamespace(projection_id="wi-2", interrupted=False),
|
||||
],
|
||||
)
|
||||
],
|
||||
active_recoveries=[],
|
||||
)
|
||||
|
||||
async def resume(self, parent_task_id: str):
|
||||
resume_calls.append(parent_task_id)
|
||||
if parent_task_id == "parent-task":
|
||||
return {"ok": True, "resumed_work_item_projection_ids": ["wi-1"]}
|
||||
return {"ok": False, "error": "not_found"}
|
||||
|
||||
async def _run() -> None:
|
||||
await _handle_chat_slash_command(state, "/recover")
|
||||
await _handle_chat_slash_command(state, "/recover resume parent-task")
|
||||
await _handle_chat_slash_command(state, "/recover resume cp-1")
|
||||
|
||||
with patch("opc.cli.app.console", console), patch("opc.cli.app._get_chat_recovery_manager", return_value=_FakeRecoveryManager()):
|
||||
asyncio.run(_run())
|
||||
with patch("opc.cli.app.console", console):
|
||||
asyncio.run(_handle_chat_slash_command(state, "/recover"))
|
||||
|
||||
rendered = console.export_text()
|
||||
self.assertIn("Interrupted Runtime", rendered)
|
||||
self.assertIn("Recovery started for parent-task", rendered)
|
||||
self.assertIn("Checkpoint cp-1 is not resumed directly", rendered)
|
||||
self.assertEqual(resume_calls, ["parent-task", "cp-1"])
|
||||
self.assertIn("Unknown command: /recover", rendered)
|
||||
self.assertNotIn("Interrupted Company Runtimes", rendered)
|
||||
|
||||
def test_logs_slash_renders_task_and_session_runtime_details(self) -> None:
|
||||
console = Console(record=True, force_terminal=False, width=220)
|
||||
|
||||
@@ -527,6 +527,25 @@ class CompanyCollaborationTests(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
await runtime.bootstrap([task])
|
||||
runtime.enqueue_runnable_work_items([work_item], task_by_work_item_id={"work-item-1": task})
|
||||
|
||||
async def claim_work_item(*_args: object, **kwargs: object) -> DelegationWorkItem:
|
||||
role_session_id = str(kwargs["role_runtime_session_id"])
|
||||
work_item.phase = Phase.RUNNING
|
||||
work_item.role_runtime_session_id = role_session_id
|
||||
work_item.claimed_by_role_runtime_session_id = role_session_id
|
||||
work_item.claimed_by_seat_id = str(kwargs.get("seat_id", ""))
|
||||
work_item.metadata = {
|
||||
**dict(work_item.metadata or {}),
|
||||
"claimed_by_role_session_id": role_session_id,
|
||||
"claimed_task_id": str(kwargs["task_id"]),
|
||||
}
|
||||
return work_item
|
||||
|
||||
runtime.store = SimpleNamespace(
|
||||
is_ready=True,
|
||||
claim_delegation_work_item_if_dispatchable=claim_work_item,
|
||||
save_delegation_role_session=AsyncMock(),
|
||||
)
|
||||
claims = await runtime.claim_runnable_tasks([task], work_items=[work_item])
|
||||
|
||||
self.assertEqual(len(claims), 1)
|
||||
@@ -534,6 +553,61 @@ class CompanyCollaborationTests(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(claimed_task.id, task.id)
|
||||
self.assertEqual(claimed_session.member_session_id, "role-session::proj1::root-a::executor::backend-architect")
|
||||
|
||||
async def test_company_runtime_does_not_spawn_after_atomic_claim_loses_to_hold(self) -> None:
|
||||
runtime = CompanyRuntime(
|
||||
org_engine=DummyOrgEngine(),
|
||||
communication=DummyRuntimeCommunication(),
|
||||
)
|
||||
task = Task(
|
||||
id="held-work-item-task",
|
||||
title="Held Work Item",
|
||||
session_id="root-a",
|
||||
parent_session_id="root-a",
|
||||
assigned_to="executor",
|
||||
status=TaskStatus.PENDING,
|
||||
project_id="proj1",
|
||||
metadata={
|
||||
"work_item_projection_id": "held_execution",
|
||||
"work_item_role_id": "executor",
|
||||
"employee_assignment": {
|
||||
"employee_id": "backend-architect",
|
||||
"role_id": "executor",
|
||||
},
|
||||
},
|
||||
)
|
||||
set_linked_work_item_id(task, "held-work-item")
|
||||
work_item = DelegationWorkItem(
|
||||
work_item_id="held-work-item",
|
||||
run_id="run-1",
|
||||
cell_id="cell-1",
|
||||
role_id="executor",
|
||||
projection_id="held_execution",
|
||||
phase=Phase.READY,
|
||||
)
|
||||
held_after_race = DelegationWorkItem(
|
||||
**{
|
||||
**work_item.__dict__,
|
||||
"metadata": {"dispatch_hold": "company_runtime_suspended"},
|
||||
}
|
||||
)
|
||||
claim = AsyncMock(return_value=None)
|
||||
runtime.store = SimpleNamespace(
|
||||
is_ready=True,
|
||||
claim_delegation_work_item_if_dispatchable=claim,
|
||||
get_delegation_work_item=AsyncMock(return_value=held_after_race),
|
||||
)
|
||||
await runtime.bootstrap([task])
|
||||
runtime.enqueue_runnable_work_items(
|
||||
[work_item],
|
||||
task_by_work_item_id={work_item.work_item_id: task},
|
||||
)
|
||||
|
||||
claims = await runtime.claim_runnable_tasks([task], work_items=[work_item])
|
||||
|
||||
self.assertEqual(claims, [])
|
||||
self.assertNotIn(work_item.work_item_id, runtime._claimed_work_item_ids)
|
||||
claim.assert_awaited_once()
|
||||
|
||||
async def test_company_runtime_does_not_double_claim_same_work_item(self) -> None:
|
||||
runtime = CompanyRuntime(
|
||||
org_engine=DummyOrgEngine(),
|
||||
@@ -2722,9 +2796,11 @@ class CompanyCollaborationTests(unittest.IsolatedAsyncioTestCase):
|
||||
engine.project_id = "proj1"
|
||||
engine.store = store
|
||||
engine.company_executor = DummyExecutor()
|
||||
attempt_token = engine._active_task_run_registry.register("proj1", running_task.id)
|
||||
|
||||
response = await engine._maybe_resume_existing_company_runtime("缁х画", "sess-parent-live")
|
||||
refreshed = await store.get_task(running_task.id)
|
||||
engine._active_task_run_registry.unregister("proj1", running_task.id, attempt_token)
|
||||
|
||||
self.assertIn("already in progress", response)
|
||||
self.assertEqual(refreshed.status, TaskStatus.RUNNING)
|
||||
|
||||
@@ -80,7 +80,6 @@ class SharedRoleSessionExecutionTests(unittest.IsolatedAsyncioTestCase):
|
||||
record_child_session_result=AsyncMock(),
|
||||
record_task_completion_async=AsyncMock(),
|
||||
)
|
||||
engine._active_task_runs = set()
|
||||
engine._run_task_once = AsyncMock(
|
||||
return_value=TaskResult(status=TaskStatus.DONE, content="done", artifacts={})
|
||||
)
|
||||
@@ -106,4 +105,3 @@ class SharedRoleSessionExecutionTests(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
engine.memory.record_assistant_turn.assert_awaited_once()
|
||||
engine.memory.record_child_session_result.assert_not_awaited()
|
||||
|
||||
|
||||
@@ -0,0 +1,410 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from opc.core.models import ExecutionCheckpoint, Task, TaskStatus
|
||||
from opc.layer2_organization.company_runtime_identity import (
|
||||
build_company_runtime_identity_index,
|
||||
)
|
||||
from opc.plugins.office_ui.services.models import ServiceError
|
||||
from opc.plugins.office_ui.snapshot_builder import (
|
||||
_build_company_runtime_control_by_task,
|
||||
_primary_session_tasks_by_session_id,
|
||||
)
|
||||
from opc.plugins.office_ui.ws_handler import WSHandler
|
||||
|
||||
|
||||
def _runtime_records() -> tuple[list[Task], ExecutionCheckpoint]:
|
||||
runtime_session_id = "runtime-session"
|
||||
anchor = Task(
|
||||
id="ui-anchor",
|
||||
project_id="project-a",
|
||||
session_id=runtime_session_id,
|
||||
status=TaskStatus.CANCELLED,
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"mode": "company",
|
||||
"company_profile": "corporate",
|
||||
},
|
||||
created_at=datetime.now() - timedelta(minutes=3),
|
||||
)
|
||||
final_decider = Task(
|
||||
id="final-decider",
|
||||
project_id="project-a",
|
||||
session_id=runtime_session_id,
|
||||
parent_session_id=runtime_session_id,
|
||||
status=TaskStatus.CANCELLED,
|
||||
linked_work_item_id="work-item-root",
|
||||
metadata={
|
||||
"mode": "company",
|
||||
"work_item_runtime": True,
|
||||
"work_item_projection_id": "root",
|
||||
"shared_role_session": True,
|
||||
"shared_role_id": "ceo",
|
||||
"company_runtime_root_session_id": runtime_session_id,
|
||||
},
|
||||
created_at=datetime.now() - timedelta(minutes=2),
|
||||
)
|
||||
child = Task(
|
||||
id="worker",
|
||||
project_id="project-a",
|
||||
session_id=f"{runtime_session_id}:role:worker",
|
||||
parent_session_id=runtime_session_id,
|
||||
status=TaskStatus.BLOCKED,
|
||||
linked_work_item_id="work-item-worker",
|
||||
metadata={
|
||||
"mode": "company",
|
||||
"work_item_runtime": True,
|
||||
"work_item_projection_id": "worker",
|
||||
"shared_role_session": True,
|
||||
"company_runtime_root_session_id": runtime_session_id,
|
||||
},
|
||||
created_at=datetime.now() - timedelta(minutes=1),
|
||||
)
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-1",
|
||||
project_id="project-a",
|
||||
session_id=runtime_session_id,
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id=final_decider.id,
|
||||
payload={"parent_session_id": runtime_session_id},
|
||||
)
|
||||
return [anchor, final_decider, child], checkpoint
|
||||
|
||||
|
||||
def test_identity_is_session_first_and_never_selects_shared_final_decider_as_ui_anchor() -> None:
|
||||
tasks, checkpoint = _runtime_records()
|
||||
index = build_company_runtime_identity_index(tasks, [checkpoint])
|
||||
|
||||
identity = index.resolve(
|
||||
task_id="final-decider",
|
||||
runtime_session_id="runtime-session",
|
||||
checkpoint_id="checkpoint-1",
|
||||
)
|
||||
|
||||
assert identity is not None
|
||||
assert identity.runtime_session_id == "runtime-session"
|
||||
assert identity.ui_anchor_task_id == "ui-anchor"
|
||||
assert identity.config_source_task_id == "ui-anchor"
|
||||
assert identity.runtime_task_ids == ("ui-anchor", "final-decider", "worker")
|
||||
assert identity.pending_checkpoint_id == "checkpoint-1"
|
||||
assert identity.resumable is True
|
||||
assert index.resolve(task_session_id="runtime-session:role:worker") == identity
|
||||
assert index.resolve(task_id="worker", runtime_session_id="other-session") is None
|
||||
assert index.resolve(task_id="ui-anchor", checkpoint_id="other-checkpoint") is None
|
||||
|
||||
|
||||
def test_config_source_uses_configured_scope_task_when_ui_anchor_has_no_config() -> None:
|
||||
tasks, _checkpoint = _runtime_records()
|
||||
tasks[0].metadata = {}
|
||||
tasks[1].metadata.update({"exec_mode": "org", "company_profile": "custom", "org_id": "studio"})
|
||||
|
||||
identity = build_company_runtime_identity_index(tasks).resolve(
|
||||
task_id="ui-anchor",
|
||||
)
|
||||
|
||||
assert identity is not None
|
||||
assert identity.ui_anchor_task_id == "ui-anchor"
|
||||
assert identity.config_source_task_id == "final-decider"
|
||||
|
||||
|
||||
def test_snapshot_session_representative_uses_canonical_ui_anchor() -> None:
|
||||
tasks, _checkpoint = _runtime_records()
|
||||
primary, ordered = _primary_session_tasks_by_session_id(
|
||||
[tasks[1], tasks[0], tasks[2]],
|
||||
)
|
||||
|
||||
assert ordered[0] == "runtime-session"
|
||||
assert primary["runtime-session"].id == "ui-anchor"
|
||||
|
||||
|
||||
def test_runtime_without_ui_anchor_never_promotes_shared_work_item() -> None:
|
||||
tasks, checkpoint = _runtime_records()
|
||||
shared_final = tasks[1]
|
||||
shared_final.parent_session_id = None
|
||||
index = build_company_runtime_identity_index([shared_final, tasks[2]], [checkpoint])
|
||||
|
||||
identity = index.resolve(runtime_session_id="runtime-session")
|
||||
|
||||
assert identity is not None
|
||||
assert identity.ui_anchor_task_id == ""
|
||||
assert identity.config_source_task_id == "final-decider"
|
||||
primary, _ordered = _primary_session_tasks_by_session_id(
|
||||
[shared_final, tasks[2]],
|
||||
)
|
||||
assert "runtime-session" not in primary
|
||||
|
||||
|
||||
def test_snapshot_projects_checkpoint_control_to_cancelled_anchor_without_task_resume_identity() -> None:
|
||||
tasks, checkpoint = _runtime_records()
|
||||
|
||||
class Store:
|
||||
async def get_execution_checkpoints(self, **_kwargs):
|
||||
return [checkpoint]
|
||||
|
||||
engine = SimpleNamespace(store=Store())
|
||||
control = asyncio.run(_build_company_runtime_control_by_task(engine, tasks, "project-a"))
|
||||
|
||||
assert control["ui-anchor"]["runtime_control_state"] == "suspended"
|
||||
assert control["ui-anchor"]["can_resume"] is True
|
||||
assert control["ui-anchor"]["resume_parent_session_id"] == "runtime-session"
|
||||
assert control["ui-anchor"]["pending_runtime_checkpoint_id"] == "checkpoint-1"
|
||||
assert "resume_parent_task_id" not in control["ui-anchor"]
|
||||
|
||||
|
||||
def test_service_error_transport_fields_cannot_be_overridden() -> None:
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
sent: list[dict] = []
|
||||
|
||||
async def send_ack(_ws, ok=True, **payload):
|
||||
sent.append({"ok": ok, **payload})
|
||||
|
||||
handler._send_ack = send_ack
|
||||
error = ServiceError(
|
||||
"actual_code",
|
||||
"actual message",
|
||||
{"ok": True, "code": "wrong", "error": "wrong", "detail": "kept"},
|
||||
)
|
||||
|
||||
asyncio.run(handler._send_service_error(object(), error, action="test_action"))
|
||||
|
||||
assert sent == [{
|
||||
"ok": False,
|
||||
"detail": "kept",
|
||||
"error": "actual message",
|
||||
"code": "actual_code",
|
||||
"action": "test_action",
|
||||
}]
|
||||
|
||||
|
||||
def test_removed_recovery_action_receives_normal_unknown_message_ack() -> None:
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler._shutting_down = False
|
||||
handler._active_message_tasks = set()
|
||||
handler._send_ack = AsyncMock()
|
||||
ws = object()
|
||||
|
||||
asyncio.run(handler._route_message(ws, json.dumps({"type": "recovery_action"})))
|
||||
|
||||
handler._send_ack.assert_awaited_once_with(
|
||||
ws,
|
||||
ok=False,
|
||||
error="unknown_message_type",
|
||||
action="recovery_action",
|
||||
)
|
||||
|
||||
|
||||
def test_work_item_chat_resume_uses_canonical_ui_anchor_as_engine_origin() -> None:
|
||||
async def scenario() -> None:
|
||||
tasks, checkpoint = _runtime_records()
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler._exec_mode = "task"
|
||||
handler._company_profile = "corporate"
|
||||
handler._shutting_down = False
|
||||
handler._active_runtime_children = {}
|
||||
handler._session_to_task = {}
|
||||
handler._task_bg_context = {}
|
||||
handler._company_suspend_reply_locks = {"runtime-session": asyncio.Lock()}
|
||||
handler.chat_store = None
|
||||
handler._set_company_runtime_control = AsyncMock()
|
||||
handler._normalize_session_exec_mode = MagicMock(return_value="task")
|
||||
handler._normalize_session_company_profile = MagicMock(return_value="corporate")
|
||||
handler._resolve_task_session_config = MagicMock(
|
||||
return_value=("company", "corporate")
|
||||
)
|
||||
handler._resolve_task_org_id = MagicMock(return_value="")
|
||||
handler._extract_checkpoint_metadata = AsyncMock(return_value=None)
|
||||
handler._sync_task_transcript_messages = AsyncMock()
|
||||
handler.on_kanban_changed = AsyncMock()
|
||||
handler._flush_progress = AsyncMock()
|
||||
run_engine = SimpleNamespace(
|
||||
project_id="project-a",
|
||||
process_message=AsyncMock(return_value="resumed"),
|
||||
)
|
||||
target = {
|
||||
"ui_anchor_task_id": "ui-anchor",
|
||||
"config_task": tasks[1],
|
||||
}
|
||||
|
||||
await handler._process_company_suspend_reply(
|
||||
ui_task_id="final-decider",
|
||||
runtime_session_id="runtime-session",
|
||||
content="continue",
|
||||
attachment_refs=None,
|
||||
message_metadata={"ui_force_resume": True},
|
||||
user_message_id=None,
|
||||
user_message_created_at=None,
|
||||
run_engine=run_engine,
|
||||
run_project_id="project-a",
|
||||
target=target,
|
||||
checkpoint=checkpoint,
|
||||
lock=handler._company_suspend_reply_locks["runtime-session"],
|
||||
)
|
||||
|
||||
call = run_engine.process_message.await_args
|
||||
assert call.kwargs["session_id"] == "runtime-session"
|
||||
assert call.kwargs["origin_task_id"] == "ui-anchor"
|
||||
assert handler._session_to_task["runtime-session"] == "ui-anchor"
|
||||
handler.on_kanban_changed.assert_awaited_once_with(engine=run_engine)
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_delivery_feedback_rejects_missing_canonical_identity_without_first_task_fallback() -> None:
|
||||
async def scenario() -> None:
|
||||
tasks, checkpoint = _runtime_records()
|
||||
|
||||
class Store:
|
||||
is_ready = True
|
||||
|
||||
async def get_tasks(self, **_kwargs):
|
||||
# A shared final-decider deliberately precedes the UI anchor;
|
||||
# legacy first-match routing selected the wrong Task here.
|
||||
return [tasks[1], tasks[0], tasks[2]]
|
||||
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler._resolve_company_runtime_target = AsyncMock(return_value=None)
|
||||
engine = SimpleNamespace(store=Store())
|
||||
|
||||
target = await handler._company_delivery_feedback_parent_target(
|
||||
task_id="final-decider",
|
||||
waiting_task_id="worker",
|
||||
waiting_task=tasks[2],
|
||||
checkpoint=checkpoint,
|
||||
payload={"parent_session_id": "runtime-session"},
|
||||
engine=engine,
|
||||
)
|
||||
|
||||
assert target == {"parent_task_id": "", "parent_session_id": ""}
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_delivery_feedback_route_consumes_missing_identity_instead_of_running_work_item() -> None:
|
||||
async def scenario() -> None:
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.chat_store = None
|
||||
handler._company_delivery_feedback_reply_locks = {}
|
||||
handler._load_execution_checkpoint_for_reply = AsyncMock(return_value=SimpleNamespace(
|
||||
checkpoint_id="feedback-1",
|
||||
checkpoint_type="company_delivery_feedback",
|
||||
status="pending",
|
||||
task_id="worker",
|
||||
session_id="runtime-session",
|
||||
payload={"waiting_task_id": "worker", "parent_session_id": "runtime-session"},
|
||||
))
|
||||
handler._company_delivery_feedback_parent_target = AsyncMock(return_value={
|
||||
"parent_task_id": "",
|
||||
"parent_session_id": "",
|
||||
})
|
||||
handler._track_session = MagicMock()
|
||||
engine = SimpleNamespace(store=SimpleNamespace(is_ready=True))
|
||||
|
||||
handled = await handler._route_company_delivery_feedback_reply_if_pending(
|
||||
task_id="worker",
|
||||
content="looks good",
|
||||
session_id="runtime-session:worker",
|
||||
task=SimpleNamespace(id="worker"),
|
||||
attachment_refs=None,
|
||||
message_metadata={
|
||||
"response_to_checkpoint_id": "feedback-1",
|
||||
"response_to_checkpoint_type": "company_delivery_feedback",
|
||||
},
|
||||
user_message_id="message-1",
|
||||
user_message_created_at=None,
|
||||
run_engine=engine,
|
||||
run_project_id="project-a",
|
||||
reply_channel_id="session:worker",
|
||||
)
|
||||
|
||||
assert handled is True
|
||||
handler._track_session.assert_not_called()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_suspend_reply_identity_mismatch_and_resuming_checkpoint_fail_closed() -> None:
|
||||
async def scenario() -> None:
|
||||
tasks, checkpoint = _runtime_records()
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.chat_store = None
|
||||
handler._company_stop_finalize_tasks = {}
|
||||
handler._company_suspend_reply_locks = {}
|
||||
handler._track = MagicMock()
|
||||
engine = SimpleNamespace(project_id="project-a")
|
||||
target = {
|
||||
"runtime_session_id": "runtime-session",
|
||||
"checkpoint": checkpoint,
|
||||
}
|
||||
|
||||
handler._resolve_company_runtime_target = AsyncMock(
|
||||
side_effect=[target, None],
|
||||
)
|
||||
mismatched = await handler._route_company_suspend_reply_if_pending(
|
||||
task_id="worker",
|
||||
content="continue",
|
||||
session_id="runtime-session:role:worker",
|
||||
task=tasks[2],
|
||||
attachment_refs=None,
|
||||
message_metadata={
|
||||
"response_to_checkpoint_id": "wrong-checkpoint",
|
||||
"response_to_checkpoint_type": "company_runtime_interrupted",
|
||||
},
|
||||
user_message_id=None,
|
||||
user_message_created_at=None,
|
||||
run_engine=engine,
|
||||
run_project_id="project-a",
|
||||
)
|
||||
assert mismatched is True
|
||||
handler._track.assert_not_called()
|
||||
|
||||
checkpoint.status = "resuming"
|
||||
handler._resolve_company_runtime_target = AsyncMock(return_value=target)
|
||||
resuming = await handler._route_company_suspend_reply_if_pending(
|
||||
task_id="worker",
|
||||
content="continue again",
|
||||
session_id="runtime-session:role:worker",
|
||||
task=tasks[2],
|
||||
attachment_refs=None,
|
||||
message_metadata=None,
|
||||
user_message_id=None,
|
||||
user_message_created_at=None,
|
||||
run_engine=engine,
|
||||
run_project_id="project-a",
|
||||
)
|
||||
assert resuming is True
|
||||
handler._track.assert_not_called()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_escalation_control_uses_durable_anchor_not_transient_progress_maps() -> None:
|
||||
async def scenario() -> None:
|
||||
tasks, checkpoint = _runtime_records()
|
||||
|
||||
class Store:
|
||||
async def get_task(self, task_id: str):
|
||||
return next((task for task in tasks if task.id == task_id), None)
|
||||
|
||||
async def get_tasks(self, **_kwargs):
|
||||
return tasks
|
||||
|
||||
async def get_execution_checkpoints(self, **_kwargs):
|
||||
return [checkpoint]
|
||||
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.engine = SimpleNamespace(project_id="project-a", store=Store())
|
||||
handler._active_runtime_children = {"worker": "wrong-parent"}
|
||||
handler._session_to_task = {"runtime-session": "wrong-parent"}
|
||||
handler._ui_task_aliases = {}
|
||||
|
||||
resolved = await handler._resolve_escalation_session_task_id("worker")
|
||||
|
||||
assert resolved == "ui-anchor"
|
||||
|
||||
asyncio.run(scenario())
|
||||
@@ -74,6 +74,19 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
|
||||
external_provider_session_id: str = "provider-session-1",
|
||||
) -> tuple[CompanyWorkItemRuntimePlan, Task]:
|
||||
plan = self._plan(profile)
|
||||
await store.save_task(
|
||||
Task(
|
||||
id=f"ui-anchor-{parent_session_id}",
|
||||
title="Company chat",
|
||||
session_id=parent_session_id,
|
||||
project_id="proj1",
|
||||
status=TaskStatus.IDLE,
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"company_profile": profile,
|
||||
},
|
||||
)
|
||||
)
|
||||
await store.save_delegation_work_item(
|
||||
DelegationWorkItem(
|
||||
work_item_id=work_item_id,
|
||||
@@ -1445,7 +1458,7 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertNotIn("external_resume_session_id", resumed_task.metadata)
|
||||
self.assertEqual(resumed_task.metadata["external_resume_fallback"], "context_replay")
|
||||
|
||||
async def test_company_runtime_checkpoint_resolves_before_long_execute(self) -> None:
|
||||
async def test_company_runtime_checkpoint_stays_resuming_during_long_execute(self) -> None:
|
||||
store = await self._store()
|
||||
_, task = await self._seed_runtime(store)
|
||||
engine = self._engine(store)
|
||||
@@ -1472,7 +1485,7 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
|
||||
)
|
||||
captured["resuming_count_during_execute"] = len(resuming)
|
||||
captured["resolved_count_during_execute"] = len(resolved)
|
||||
captured["resume_state_during_execute"] = resolved[0].payload.get("resume_state") if resolved else ""
|
||||
captured["resume_state_during_execute"] = resuming[0].payload.get("resume_state") if resuming else ""
|
||||
return "runtime resumed"
|
||||
|
||||
engine.company_executor = DummyCompanyExecutor()
|
||||
@@ -1489,9 +1502,9 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
|
||||
statuses=["resolved"],
|
||||
)
|
||||
|
||||
self.assertEqual(captured["resuming_count_during_execute"], 0)
|
||||
self.assertEqual(captured["resolved_count_during_execute"], 1)
|
||||
self.assertEqual(captured["resume_state_during_execute"], "handoff_complete")
|
||||
self.assertEqual(captured["resuming_count_during_execute"], 1)
|
||||
self.assertEqual(captured["resolved_count_during_execute"], 0)
|
||||
self.assertEqual(captured["resume_state_during_execute"], "resuming")
|
||||
self.assertEqual(len(resolved), 1)
|
||||
self.assertEqual(resolved[0].payload.get("resume_state"), "handoff_complete")
|
||||
|
||||
@@ -1537,10 +1550,24 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
|
||||
checkpoint_types=["company_runtime_suspended"],
|
||||
statuses=["resuming"],
|
||||
)
|
||||
refreshed_task = await store.get_task(task.id)
|
||||
refreshed_item = await store.get_delegation_work_item("work-item-1")
|
||||
|
||||
self.assertEqual(len(pending), 1)
|
||||
self.assertEqual(resuming, [])
|
||||
self.assertEqual(pending[0].payload.get("resume_state"), "failed_before_handoff")
|
||||
assert refreshed_task is not None
|
||||
assert refreshed_item is not None
|
||||
self.assertEqual(
|
||||
refreshed_task.metadata.get("dispatch_hold"),
|
||||
"company_runtime_suspended",
|
||||
)
|
||||
self.assertEqual(
|
||||
refreshed_item.metadata.get("dispatch_hold"),
|
||||
"company_runtime_suspended",
|
||||
)
|
||||
self.assertEqual(refreshed_item.claimed_by_role_runtime_session_id, "")
|
||||
self.assertEqual(refreshed_item.claimed_by_seat_id, "")
|
||||
|
||||
async def test_suspend_is_parent_session_idempotent(self) -> None:
|
||||
store = await self._store()
|
||||
@@ -1831,22 +1858,16 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_continue_clears_parent_runtime_stop_marker(self) -> None:
|
||||
store = await self._store()
|
||||
_, task = await self._seed_runtime(store)
|
||||
parent = Task(
|
||||
id="parent-task",
|
||||
title="Parent company runtime",
|
||||
session_id="sess-parent",
|
||||
parent_session_id="",
|
||||
status=TaskStatus.RUNNING,
|
||||
project_id="proj1",
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"company_profile": "corporate",
|
||||
"company_runtime_stop_state": "suspended",
|
||||
"company_runtime_stop_intent_id": "intent-1",
|
||||
"company_runtime_stop_marked_at": "2026-04-29T11:02:40",
|
||||
"company_runtime_suspended_at": "2026-04-29T11:02:40",
|
||||
},
|
||||
)
|
||||
parent = await store.get_task("ui-anchor-sess-parent")
|
||||
assert parent is not None
|
||||
parent.status = TaskStatus.RUNNING
|
||||
parent.metadata = {
|
||||
**dict(parent.metadata or {}),
|
||||
"company_runtime_stop_state": "suspended",
|
||||
"company_runtime_stop_intent_id": "intent-1",
|
||||
"company_runtime_stop_marked_at": "2026-04-29T11:02:40",
|
||||
"company_runtime_suspended_at": "2026-04-29T11:02:40",
|
||||
}
|
||||
await store.save_task(parent)
|
||||
engine = self._engine(store)
|
||||
await engine.suspend_company_runtime(
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from opc import engine as engine_module
|
||||
from opc.engine import OPCEngine
|
||||
|
||||
|
||||
class EnginePidProbeTests(unittest.TestCase):
|
||||
def test_current_process_is_running(self) -> None:
|
||||
self.assertTrue(OPCEngine._pid_is_running(os.getpid()))
|
||||
|
||||
def test_posix_probe_treats_unexpected_oserror_as_not_running(self) -> None:
|
||||
with patch.object(engine_module.os, "name", "posix"), patch.object(
|
||||
engine_module.os,
|
||||
"kill",
|
||||
side_effect=OSError("platform probe failed"),
|
||||
):
|
||||
self.assertFalse(OPCEngine._pid_is_running(12345))
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,295 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import timedelta
|
||||
|
||||
from opc.core.models import ExecutionCheckpoint, Task, TaskStatus
|
||||
from opc.database.store import OPCStore
|
||||
|
||||
|
||||
def test_checkpoint_compare_and_set_has_one_winner_across_store_connections(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
async def scenario() -> None:
|
||||
db_path = tmp_path / "tasks.db"
|
||||
first_store = OPCStore(db_path)
|
||||
second_store = OPCStore(db_path)
|
||||
await first_store.initialize()
|
||||
await second_store.initialize()
|
||||
try:
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-1",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id="runtime-task",
|
||||
payload={"reason": "service_restart"},
|
||||
)
|
||||
await first_store.save_execution_checkpoint(checkpoint)
|
||||
|
||||
start = asyncio.Event()
|
||||
payloads = [
|
||||
{"reason": "service_restart", "claimed_by": "office"},
|
||||
{"reason": "service_restart", "claimed_by": "cli"},
|
||||
]
|
||||
|
||||
async def claim(store: OPCStore, payload: dict[str, str]) -> bool:
|
||||
await start.wait()
|
||||
return await store.compare_and_set_execution_checkpoint(
|
||||
checkpoint.checkpoint_id,
|
||||
expected_statuses={"pending"},
|
||||
status="resuming",
|
||||
payload=payload,
|
||||
)
|
||||
|
||||
claims = [
|
||||
asyncio.create_task(claim(first_store, payloads[0])),
|
||||
asyncio.create_task(claim(second_store, payloads[1])),
|
||||
]
|
||||
start.set()
|
||||
results = await asyncio.gather(*claims)
|
||||
|
||||
assert results.count(True) == 1
|
||||
assert results.count(False) == 1
|
||||
winner = results.index(True)
|
||||
rows = await first_store.get_execution_checkpoints(
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
)
|
||||
assert len(rows) == 1
|
||||
assert rows[0].status == "resuming"
|
||||
assert rows[0].payload == payloads[winner]
|
||||
finally:
|
||||
await second_store.close()
|
||||
await first_store.close()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_checkpoint_get_or_create_has_one_active_row_across_store_connections(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
async def scenario() -> None:
|
||||
db_path = tmp_path / "tasks.db"
|
||||
first_store = OPCStore(db_path)
|
||||
second_store = OPCStore(db_path)
|
||||
await first_store.initialize()
|
||||
await second_store.initialize()
|
||||
try:
|
||||
checkpoint_types = {
|
||||
"company_runtime_suspended",
|
||||
"company_runtime_interrupted",
|
||||
}
|
||||
candidates = [
|
||||
ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-office",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
task_id="runtime-task",
|
||||
payload={"creator": "office"},
|
||||
),
|
||||
ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-cli",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_suspended",
|
||||
task_id="runtime-task",
|
||||
payload={"creator": "cli"},
|
||||
),
|
||||
]
|
||||
start = asyncio.Event()
|
||||
|
||||
async def create(
|
||||
store: OPCStore,
|
||||
candidate: ExecutionCheckpoint,
|
||||
) -> tuple[ExecutionCheckpoint, bool]:
|
||||
await start.wait()
|
||||
return await store.get_or_create_active_execution_checkpoint(
|
||||
candidate,
|
||||
checkpoint_types=checkpoint_types,
|
||||
)
|
||||
|
||||
attempts = [
|
||||
asyncio.create_task(create(first_store, candidates[0])),
|
||||
asyncio.create_task(create(second_store, candidates[1])),
|
||||
]
|
||||
start.set()
|
||||
results = await asyncio.gather(*attempts)
|
||||
|
||||
assert [created for _, created in results].count(True) == 1
|
||||
assert [created for _, created in results].count(False) == 1
|
||||
assert len({row.checkpoint_id for row, _ in results}) == 1
|
||||
|
||||
active = await first_store.get_execution_checkpoints(
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_types=list(checkpoint_types),
|
||||
statuses=["pending", "resuming"],
|
||||
)
|
||||
assert len(active) == 1
|
||||
assert active[0].checkpoint_id == results[0][0].checkpoint_id
|
||||
finally:
|
||||
await second_store.close()
|
||||
await first_store.close()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_checkpoint_get_or_create_normalizes_historical_active_duplicates(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
async def scenario() -> None:
|
||||
store = OPCStore(tmp_path / "tasks.db")
|
||||
await store.initialize()
|
||||
try:
|
||||
older = ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-older",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="resuming",
|
||||
payload={"created": "older"},
|
||||
)
|
||||
newer = ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-newer",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_suspended",
|
||||
payload={"created": "newer"},
|
||||
)
|
||||
newer.updated_at = older.updated_at + timedelta(microseconds=1)
|
||||
await store.save_execution_checkpoint(older)
|
||||
await store.save_execution_checkpoint(newer)
|
||||
|
||||
winner, created = await store.get_or_create_active_execution_checkpoint(
|
||||
ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-unused",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
),
|
||||
checkpoint_types={
|
||||
"company_runtime_suspended",
|
||||
"company_runtime_interrupted",
|
||||
},
|
||||
)
|
||||
|
||||
assert created is False
|
||||
assert winner.checkpoint_id == "checkpoint-newer"
|
||||
active = await store.get_execution_checkpoints(
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
statuses=["pending", "resuming"],
|
||||
)
|
||||
assert [row.checkpoint_id for row in active] == ["checkpoint-newer"]
|
||||
all_rows = await store.get_execution_checkpoints(
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
)
|
||||
by_id = {row.checkpoint_id: row for row in all_rows}
|
||||
assert by_id["checkpoint-older"].status == "superseded"
|
||||
assert (
|
||||
by_id["checkpoint-older"].payload["superseded_by_checkpoint_id"]
|
||||
== "checkpoint-newer"
|
||||
)
|
||||
finally:
|
||||
await store.close()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_checkpoint_completion_and_cancelled_anchor_reopen_are_atomic(tmp_path) -> None:
|
||||
async def scenario() -> None:
|
||||
store = OPCStore(tmp_path / "tasks.db")
|
||||
await store.initialize()
|
||||
try:
|
||||
anchor = Task(
|
||||
id="ui-anchor",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
title="Company chat",
|
||||
status=TaskStatus.CANCELLED,
|
||||
execution_lock=True,
|
||||
)
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-1",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="resuming",
|
||||
payload={"ui_anchor_task_id": anchor.id},
|
||||
)
|
||||
await store.save_task(anchor)
|
||||
await store.save_execution_checkpoint(checkpoint)
|
||||
|
||||
completed = await store.complete_execution_checkpoint_and_reopen_ui_anchor(
|
||||
checkpoint.checkpoint_id,
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
expected_status="resuming",
|
||||
status="resolved",
|
||||
payload={"resume_state": "handoff_complete"},
|
||||
ui_anchor_task_id=anchor.id,
|
||||
)
|
||||
|
||||
assert completed is True
|
||||
assert (await store.get_task(anchor.id)).status == TaskStatus.IDLE
|
||||
rows = await store.get_execution_checkpoints(
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
)
|
||||
assert rows[0].status == "resolved"
|
||||
assert rows[0].payload == {"resume_state": "handoff_complete"}
|
||||
finally:
|
||||
await store.close()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_checkpoint_completion_does_not_reopen_anchor_after_stop_wins(tmp_path) -> None:
|
||||
async def scenario() -> None:
|
||||
store = OPCStore(tmp_path / "tasks.db")
|
||||
await store.initialize()
|
||||
try:
|
||||
anchor = Task(
|
||||
id="ui-anchor",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
title="Company chat",
|
||||
status=TaskStatus.CANCELLED,
|
||||
)
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-1",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
payload={"resume_state": "interrupted"},
|
||||
)
|
||||
await store.save_task(anchor)
|
||||
await store.save_execution_checkpoint(checkpoint)
|
||||
|
||||
completed = await store.complete_execution_checkpoint_and_reopen_ui_anchor(
|
||||
checkpoint.checkpoint_id,
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
expected_status="resuming",
|
||||
status="resolved",
|
||||
payload={"resume_state": "handoff_complete"},
|
||||
ui_anchor_task_id=anchor.id,
|
||||
)
|
||||
|
||||
assert completed is False
|
||||
assert (await store.get_task(anchor.id)).status == TaskStatus.CANCELLED
|
||||
rows = await store.get_execution_checkpoints(
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
)
|
||||
assert rows[0].status == "pending"
|
||||
assert rows[0].payload == {"resume_state": "interrupted"}
|
||||
finally:
|
||||
await store.close()
|
||||
|
||||
asyncio.run(scenario())
|
||||
@@ -0,0 +1,380 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from opc.core.active_task_runs import ActiveTaskRunRegistry
|
||||
from opc.core.models import ExecutionCheckpoint, Task
|
||||
from opc.plugins.office_ui.ws_handler import WSHandler
|
||||
|
||||
|
||||
def test_ws_shutdown_checkpoints_before_cancelling_and_awaiting_sessions() -> None:
|
||||
async def scenario() -> None:
|
||||
events: list[str] = []
|
||||
started = asyncio.Event()
|
||||
|
||||
async def execution() -> None:
|
||||
started.set()
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
events.append("execution_finally")
|
||||
|
||||
async def prepare() -> list[dict]:
|
||||
events.append("checkpoint")
|
||||
assert not execution_task.done()
|
||||
return []
|
||||
|
||||
execution_task = asyncio.create_task(execution())
|
||||
await started.wait()
|
||||
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.engine = SimpleNamespace()
|
||||
handler._root_engine = SimpleNamespace(
|
||||
prepare_active_company_runtimes_for_shutdown=prepare,
|
||||
)
|
||||
handler._shutting_down = False
|
||||
handler._progress_flush_task = None
|
||||
handler._clients = set()
|
||||
handler._active_message_tasks = set()
|
||||
handler._background_tasks = {execution_task}
|
||||
handler._task_bg_context = {execution_task: {"task_id": "runtime-task"}}
|
||||
handler._task_bg_map = {"runtime-task": {execution_task}}
|
||||
|
||||
await handler.shutdown(timeout=1.0)
|
||||
|
||||
assert events == ["checkpoint", "execution_finally"]
|
||||
assert execution_task.done()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_ws_shutdown_checkpoint_failure_does_not_cancel_execution_or_close_the_gap() -> None:
|
||||
async def scenario() -> None:
|
||||
released = asyncio.Event()
|
||||
|
||||
async def execution() -> None:
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
released.set()
|
||||
|
||||
execution_task = asyncio.create_task(execution())
|
||||
await asyncio.sleep(0)
|
||||
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.engine = SimpleNamespace()
|
||||
handler._root_engine = SimpleNamespace(
|
||||
prepare_active_company_runtimes_for_shutdown=AsyncMock(
|
||||
side_effect=RuntimeError("checkpoint unavailable")
|
||||
),
|
||||
)
|
||||
handler._shutting_down = False
|
||||
handler._progress_flush_task = None
|
||||
handler._clients = set()
|
||||
handler._active_message_tasks = set()
|
||||
handler._background_tasks = {execution_task}
|
||||
handler._task_bg_context = {execution_task: {"task_id": "runtime-task"}}
|
||||
handler._task_bg_map = {"runtime-task": {execution_task}}
|
||||
|
||||
try:
|
||||
await handler.shutdown(timeout=1.0)
|
||||
except RuntimeError as exc:
|
||||
assert str(exc) == "checkpoint unavailable"
|
||||
else:
|
||||
raise AssertionError("shutdown must fail closed when checkpointing fails")
|
||||
|
||||
assert not released.is_set()
|
||||
assert not execution_task.done()
|
||||
execution_task.cancel()
|
||||
await asyncio.gather(execution_task, return_exceptions=True)
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_ws_shutdown_rejects_background_work_scheduled_by_late_ingress() -> None:
|
||||
async def scenario() -> None:
|
||||
entered = False
|
||||
|
||||
async def late_work() -> None:
|
||||
nonlocal entered
|
||||
entered = True
|
||||
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler._shutting_down = True
|
||||
handler._background_tasks = set()
|
||||
task = handler._track(late_work())
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert task.cancelled()
|
||||
assert entered is False
|
||||
assert task not in handler._background_tasks
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_ws_shutdown_drains_queued_duplicate_handoff_before_checkpointing() -> None:
|
||||
async def scenario() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
runtime_lock = asyncio.Lock()
|
||||
execution_registered = asyncio.Event()
|
||||
prepare_called = asyncio.Event()
|
||||
execution_released = asyncio.Event()
|
||||
|
||||
async def prepare() -> list[dict]:
|
||||
assert registry.is_active("project-a", "runtime-task")
|
||||
assert registry.pending_handoff_count == 0
|
||||
prepare_called.set()
|
||||
return []
|
||||
|
||||
root_engine = SimpleNamespace(
|
||||
_active_task_run_registry=registry,
|
||||
prepare_active_company_runtimes_for_shutdown=prepare,
|
||||
)
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.engine = root_engine
|
||||
handler._root_engine = root_engine
|
||||
handler._shutting_down = False
|
||||
handler._progress_flush_task = None
|
||||
handler._clients = set()
|
||||
handler._active_message_tasks = set()
|
||||
handler._background_tasks = set()
|
||||
handler._task_bg_context = {}
|
||||
handler._task_bg_map = {}
|
||||
handler._handoff_route_tasks = {}
|
||||
|
||||
async def execution() -> None:
|
||||
async with runtime_lock:
|
||||
attempt_token = registry.register("project-a", "runtime-task")
|
||||
execution_registered.set()
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
registry.unregister("project-a", "runtime-task", attempt_token)
|
||||
execution_released.set()
|
||||
|
||||
async def queued_duplicate() -> None:
|
||||
async with runtime_lock:
|
||||
attempt_token = registry.register("project-a", "runtime-task")
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
registry.unregister("project-a", "runtime-task", attempt_token)
|
||||
|
||||
first_handoff = registry.reserve_handoff()
|
||||
with registry.bind_handoff(first_handoff):
|
||||
first = handler._track_session(
|
||||
"runtime-task",
|
||||
execution(),
|
||||
project_id="project-a",
|
||||
engine=root_engine,
|
||||
)
|
||||
registry.release_handoff(first_handoff)
|
||||
await execution_registered.wait()
|
||||
|
||||
second_handoff = registry.reserve_handoff()
|
||||
with registry.bind_handoff(second_handoff):
|
||||
second = handler._track_session(
|
||||
"runtime-task",
|
||||
queued_duplicate(),
|
||||
project_id="project-a",
|
||||
engine=root_engine,
|
||||
)
|
||||
registry.release_handoff(second_handoff)
|
||||
await asyncio.sleep(0)
|
||||
assert registry.pending_handoff_count == 1
|
||||
|
||||
await asyncio.wait_for(handler.shutdown(timeout=1.0), timeout=1.0)
|
||||
|
||||
assert prepare_called.is_set()
|
||||
assert execution_released.is_set()
|
||||
assert first.cancelled()
|
||||
assert second.cancelled()
|
||||
assert registry.pending_handoff_count == 0
|
||||
assert not registry.is_active("project-a", "runtime-task")
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_ws_shutdown_fails_closed_while_execution_cleanup_is_still_running() -> None:
|
||||
async def scenario() -> None:
|
||||
cancellation_started = asyncio.Event()
|
||||
allow_cleanup = asyncio.Event()
|
||||
|
||||
async def execution() -> None:
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
except asyncio.CancelledError:
|
||||
cancellation_started.set()
|
||||
await allow_cleanup.wait()
|
||||
|
||||
execution_task = asyncio.create_task(execution())
|
||||
await asyncio.sleep(0)
|
||||
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.engine = SimpleNamespace()
|
||||
handler._root_engine = SimpleNamespace(
|
||||
prepare_active_company_runtimes_for_shutdown=AsyncMock(return_value=[]),
|
||||
)
|
||||
handler._shutting_down = False
|
||||
handler._progress_flush_task = None
|
||||
handler._clients = set()
|
||||
handler._active_message_tasks = set()
|
||||
handler._background_tasks = {execution_task}
|
||||
handler._task_bg_context = {
|
||||
execution_task: {"task_id": "runtime-task", "execution_handoff": True}
|
||||
}
|
||||
handler._task_bg_map = {"runtime-task": {execution_task}}
|
||||
|
||||
try:
|
||||
await handler.shutdown(timeout=0.01)
|
||||
except RuntimeError as exc:
|
||||
assert "execution task(s)" in str(exc)
|
||||
else:
|
||||
raise AssertionError("shutdown must not close resources before execution cleanup")
|
||||
|
||||
assert cancellation_started.is_set()
|
||||
assert not execution_task.done()
|
||||
allow_cleanup.set()
|
||||
await execution_task
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_ws_shutdown_cancels_execution_before_waiting_for_client_close() -> None:
|
||||
async def scenario() -> None:
|
||||
execution_released = asyncio.Event()
|
||||
close_entered = asyncio.Event()
|
||||
allow_close = asyncio.Event()
|
||||
|
||||
async def execution() -> None:
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
execution_released.set()
|
||||
|
||||
class BlockingWebSocket:
|
||||
async def close(self, **_kwargs: object) -> None:
|
||||
close_entered.set()
|
||||
await allow_close.wait()
|
||||
|
||||
execution_task = asyncio.create_task(execution())
|
||||
await asyncio.sleep(0)
|
||||
client = BlockingWebSocket()
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.engine = SimpleNamespace()
|
||||
handler._root_engine = SimpleNamespace(
|
||||
prepare_active_company_runtimes_for_shutdown=AsyncMock(return_value=[]),
|
||||
)
|
||||
handler._shutting_down = False
|
||||
handler._progress_flush_task = None
|
||||
handler._clients = {client}
|
||||
handler._active_message_tasks = set()
|
||||
handler._background_tasks = {execution_task}
|
||||
handler._task_bg_context = {execution_task: {"task_id": "runtime-task"}}
|
||||
handler._task_bg_map = {"runtime-task": {execution_task}}
|
||||
|
||||
shutdown_task = asyncio.create_task(handler.shutdown(timeout=1.0))
|
||||
await close_entered.wait()
|
||||
|
||||
assert execution_released.is_set()
|
||||
assert execution_task.done()
|
||||
allow_close.set()
|
||||
await shutdown_task
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_duplicate_resume_does_not_leave_shutdown_handoff_barrier_queued() -> None:
|
||||
async def scenario() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
execution_started = asyncio.Event()
|
||||
execution_released = asyncio.Event()
|
||||
|
||||
async def prepare() -> list[dict]:
|
||||
assert registry.is_active("project-a", "runtime-task")
|
||||
return []
|
||||
|
||||
root_engine = SimpleNamespace(
|
||||
project_id="project-a",
|
||||
_active_task_run_registry=registry,
|
||||
prepare_active_company_runtimes_for_shutdown=prepare,
|
||||
)
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.engine = root_engine
|
||||
handler._root_engine = root_engine
|
||||
handler.chat_store = None
|
||||
handler._shutting_down = False
|
||||
handler._progress_flush_task = None
|
||||
handler._clients = set()
|
||||
handler._active_message_tasks = set()
|
||||
handler._background_tasks = set()
|
||||
handler._task_bg_context = {}
|
||||
handler._task_bg_map = {}
|
||||
handler._company_stop_finalize_tasks = {}
|
||||
handler._company_suspend_reply_locks = {}
|
||||
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-1",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
)
|
||||
task = Task(
|
||||
id="ui-task",
|
||||
title="Company chat",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
metadata={"exec_mode": "company"},
|
||||
)
|
||||
target = {
|
||||
"runtime_session_id": "runtime-session",
|
||||
"checkpoint": checkpoint,
|
||||
}
|
||||
handler._resolve_company_runtime_target = AsyncMock(return_value=target)
|
||||
|
||||
async def fake_resume(**_kwargs: object) -> None:
|
||||
attempt = registry.register("project-a", "runtime-task")
|
||||
execution_started.set()
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
registry.unregister("project-a", "runtime-task", attempt)
|
||||
execution_released.set()
|
||||
|
||||
handler._process_company_suspend_reply = fake_resume
|
||||
|
||||
async def route_once() -> bool:
|
||||
handoff = registry.reserve_handoff()
|
||||
try:
|
||||
with registry.bind_handoff(handoff):
|
||||
return await handler._route_company_suspend_reply_if_pending(
|
||||
task_id=task.id,
|
||||
content="continue",
|
||||
session_id=task.session_id,
|
||||
task=task,
|
||||
attachment_refs=None,
|
||||
message_metadata=None,
|
||||
user_message_id=None,
|
||||
user_message_created_at=None,
|
||||
run_engine=root_engine,
|
||||
run_project_id="project-a",
|
||||
)
|
||||
finally:
|
||||
registry.release_handoff(handoff)
|
||||
|
||||
assert await route_once() is True
|
||||
await execution_started.wait()
|
||||
assert await route_once() is True
|
||||
assert registry.pending_handoff_count == 0
|
||||
assert len(handler._background_tasks) == 1
|
||||
|
||||
await asyncio.wait_for(handler.shutdown(timeout=1.0), timeout=1.0)
|
||||
|
||||
assert execution_released.is_set()
|
||||
assert registry.pending_handoff_count == 0
|
||||
|
||||
asyncio.run(scenario())
|
||||
@@ -4,6 +4,7 @@ import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from opc.core.active_task_runs import ActiveTaskRunRegistry
|
||||
from opc.core.config import OPCConfig, RoleConfig
|
||||
from opc.core.org_config import (
|
||||
build_org_config_payload_from_config,
|
||||
@@ -137,6 +138,10 @@ def test_custom_runtime_initializes_with_parent_store(monkeypatch, tmp_path) ->
|
||||
self.store = kwargs.get("store") or object()
|
||||
self.owns_store = kwargs.get("owns_store")
|
||||
self.run_startup_reconcile = kwargs.get("run_startup_reconcile")
|
||||
self.active_task_run_registry = kwargs.get("active_task_run_registry")
|
||||
self.owns_active_task_run_registry = kwargs.get(
|
||||
"owns_active_task_run_registry"
|
||||
)
|
||||
self.bound_stores = []
|
||||
self.message_bus = FakeMessageBus(self)
|
||||
self.company_executor = SimpleNamespace(_signal_dispatcher_wake=lambda: None)
|
||||
@@ -164,6 +169,7 @@ def test_custom_runtime_initializes_with_parent_store(monkeypatch, tmp_path) ->
|
||||
parent.config = OPCConfig()
|
||||
parent.project_id = None
|
||||
parent.store = parent_store
|
||||
parent._active_task_run_registry = ActiveTaskRunRegistry()
|
||||
parent.on_progress = None
|
||||
parent.on_runtime_event = None
|
||||
parent.on_escalation = None
|
||||
@@ -198,6 +204,8 @@ def test_custom_runtime_initializes_with_parent_store(monkeypatch, tmp_path) ->
|
||||
assert runtime.store is parent_store
|
||||
assert runtime.owns_store is False
|
||||
assert runtime.run_startup_reconcile is False
|
||||
assert runtime.active_task_run_registry is parent._active_task_run_registry
|
||||
assert runtime.owns_active_task_run_registry is False
|
||||
assert runtime.bound_stores == []
|
||||
assert captured["kanban_callback_runtime"] is runtime
|
||||
assert callable(runtime.company_executor.on_kanban_changed)
|
||||
|
||||
@@ -188,10 +188,8 @@ class DirectStatusWriteLintTest(unittest.TestCase):
|
||||
PATTERN = re.compile(r"\.status\s*=\s*TaskStatus\.(CANCELLED|FAILED)")
|
||||
MIGRATED_COMPANY_AWARE_FILES = (
|
||||
"opc/plugins/office_ui/dispatcher.py",
|
||||
"opc/plugins/office_ui/recovery_manager.py",
|
||||
"opc/plugins/office_ui/ws_handler.py",
|
||||
"opc/plugins/cli_board/services/actions.py",
|
||||
"opc/plugins/cli_board/services/recovery.py",
|
||||
)
|
||||
MIGRATED_FIVE_STATUS_PATTERN = re.compile(
|
||||
r"\.status\s*=\s*TaskStatus\.(PENDING|RUNNING|DONE|FAILED|CANCELLED)"
|
||||
|
||||
@@ -27,6 +27,7 @@ from opc.database.store import _SQLiteConnectionAdapter
|
||||
from opc.layer2_organization import comms as file_comms
|
||||
from opc.plugins.office_ui.event_adapter import EventAdapter
|
||||
from opc.plugins.office_ui.chat_store import ChatStore
|
||||
from opc.plugins.office_ui.services.models import ServiceError
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
@@ -1823,6 +1824,126 @@ class TestWSHandlerSessionSend(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(len(ended), 1)
|
||||
self.handler._dispatch_session_message.assert_not_called()
|
||||
|
||||
async def _seed_cancelled_markerless_company_anchor(self) -> ExecutionCheckpoint:
|
||||
anchor = await self.store.get_task(self.task_id)
|
||||
assert anchor is not None
|
||||
anchor.status = TaskStatus.CANCELLED
|
||||
anchor.metadata = {}
|
||||
await self.store.save_task(anchor)
|
||||
await self.store.save_task(Task(
|
||||
id="shared-final-decider",
|
||||
title="Final decision",
|
||||
project_id="test-project",
|
||||
session_id=self.session_id,
|
||||
parent_session_id=self.session_id,
|
||||
linked_work_item_id="work-item-final",
|
||||
status=TaskStatus.BLOCKED,
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"company_profile": "corporate",
|
||||
"work_item_runtime": True,
|
||||
"work_item_projection_id": "final",
|
||||
"shared_role_session": True,
|
||||
},
|
||||
))
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="cp-markerless-anchor",
|
||||
project_id="test-project",
|
||||
session_id=self.session_id,
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id="shared-final-decider",
|
||||
payload={"parent_session_id": self.session_id},
|
||||
)
|
||||
await self.store.save_execution_checkpoint(checkpoint)
|
||||
return checkpoint
|
||||
|
||||
async def test_cancelled_markerless_company_anchor_text_routes_by_checkpoint_identity(self) -> None:
|
||||
checkpoint = await self._seed_cancelled_markerless_company_anchor()
|
||||
self.handler._process_company_suspend_reply = AsyncMock()
|
||||
fake_bg = object()
|
||||
|
||||
def _close_and_track(_task_id: str, coro: Any, **_kwargs: Any) -> object:
|
||||
coro.close()
|
||||
self.handler._task_bg_context[fake_bg] = {}
|
||||
return fake_bg
|
||||
|
||||
self.handler._track_session = MagicMock(side_effect=_close_and_track)
|
||||
ws = MagicMock()
|
||||
|
||||
await self.handler._handle_session_send(ws, {
|
||||
"project_id": "test-project",
|
||||
"task_id": self.task_id,
|
||||
"content": "继续恢复这个 runtime。",
|
||||
})
|
||||
|
||||
self.handler._process_company_suspend_reply.assert_called_once()
|
||||
routed = self.handler._process_company_suspend_reply.call_args.kwargs
|
||||
self.assertEqual(routed["ui_task_id"], self.task_id)
|
||||
self.assertEqual(routed["runtime_session_id"], self.session_id)
|
||||
self.assertEqual(routed["checkpoint"].checkpoint_id, checkpoint.checkpoint_id)
|
||||
self.handler._track_session.assert_called_once()
|
||||
anchor = await self.store.get_task(self.task_id)
|
||||
assert anchor is not None
|
||||
self.assertEqual(anchor.status, TaskStatus.CANCELLED)
|
||||
|
||||
async def test_cancelled_company_anchor_with_resuming_checkpoint_is_not_ended_early(self) -> None:
|
||||
checkpoint = await self._seed_cancelled_markerless_company_anchor()
|
||||
checkpoint.status = "resuming"
|
||||
await self.store.save_execution_checkpoint(checkpoint)
|
||||
self.handler._process_company_suspend_reply = AsyncMock()
|
||||
self.handler._process_session_message = AsyncMock()
|
||||
ws = MagicMock()
|
||||
|
||||
await self.handler._handle_session_send(ws, {
|
||||
"project_id": "test-project",
|
||||
"task_id": self.task_id,
|
||||
"content": "不要重复恢复。",
|
||||
})
|
||||
|
||||
ended = [
|
||||
call for call in self.handler._send_ack.await_args_list
|
||||
if call.kwargs.get("error") == "session_ended"
|
||||
]
|
||||
self.assertEqual(ended, [])
|
||||
self.handler._process_company_suspend_reply.assert_not_called()
|
||||
self.handler._process_session_message.assert_not_called()
|
||||
|
||||
async def test_cancelled_markerless_company_anchor_button_routes_by_checkpoint_identity(self) -> None:
|
||||
checkpoint = await self._seed_cancelled_markerless_company_anchor()
|
||||
self.handler._process_company_suspend_reply = AsyncMock()
|
||||
fake_bg = object()
|
||||
|
||||
def _close_and_track(_task_id: str, coro: Any, **_kwargs: Any) -> object:
|
||||
coro.close()
|
||||
self.handler._task_bg_context[fake_bg] = {}
|
||||
return fake_bg
|
||||
|
||||
self.handler._track_session = MagicMock(side_effect=_close_and_track)
|
||||
ws = MagicMock()
|
||||
|
||||
await self.handler._handle_session_resume(ws, {
|
||||
"project_id": "test-project",
|
||||
"task_id": self.task_id,
|
||||
"runtime_session_id": self.session_id,
|
||||
"checkpoint_id": checkpoint.checkpoint_id,
|
||||
})
|
||||
|
||||
self.handler._process_company_suspend_reply.assert_called_once()
|
||||
routed = self.handler._process_company_suspend_reply.call_args.kwargs
|
||||
self.assertEqual(routed["ui_task_id"], self.task_id)
|
||||
self.assertEqual(routed["runtime_session_id"], self.session_id)
|
||||
self.assertEqual(routed["checkpoint"].checkpoint_id, checkpoint.checkpoint_id)
|
||||
self.handler._send_ack.assert_awaited_with(
|
||||
ws,
|
||||
ok=True,
|
||||
runtime_session_id=self.session_id,
|
||||
checkpoint_id=checkpoint.checkpoint_id,
|
||||
)
|
||||
anchor = await self.store.get_task(self.task_id)
|
||||
assert anchor is not None
|
||||
self.assertEqual(anchor.status, TaskStatus.CANCELLED)
|
||||
|
||||
async def test_done_company_session_send_is_reopened_for_followup(self) -> None:
|
||||
"""Completed company chats can continue in the same CEO/company context."""
|
||||
ws = MagicMock()
|
||||
@@ -1980,23 +2101,23 @@ class TestWSHandlerSessionSend(unittest.IsolatedAsyncioTestCase):
|
||||
}
|
||||
await self.store.save_task(task)
|
||||
|
||||
self.engine.get_active_company_runtime_suspend_checkpoint = AsyncMock(
|
||||
return_value=SimpleNamespace(
|
||||
checkpoint_id="cp-suspended",
|
||||
checkpoint_type="company_runtime_suspended",
|
||||
status="pending",
|
||||
payload={},
|
||||
)
|
||||
)
|
||||
await self.store.save_execution_checkpoint(ExecutionCheckpoint(
|
||||
checkpoint_id="cp-suspended",
|
||||
project_id="test-project",
|
||||
session_id=self.session_id,
|
||||
checkpoint_type="company_runtime_suspended",
|
||||
status="pending",
|
||||
payload={"parent_session_id": self.session_id},
|
||||
))
|
||||
self.handler._process_company_suspend_reply = AsyncMock()
|
||||
fake_bg = object()
|
||||
|
||||
def _close_and_track(coro: Any) -> object:
|
||||
def _close_and_track(_task_id: str, coro: Any, **_kwargs: Any) -> object:
|
||||
coro.close()
|
||||
self.handler._task_bg_context[fake_bg] = {}
|
||||
return fake_bg
|
||||
|
||||
self.handler._track = MagicMock(side_effect=_close_and_track)
|
||||
self.handler._track_session = MagicMock()
|
||||
self.handler._track_session = MagicMock(side_effect=_close_and_track)
|
||||
|
||||
await self.handler._handle_session_send(ws, {
|
||||
"project_id": "test-project",
|
||||
@@ -2004,13 +2125,39 @@ class TestWSHandlerSessionSend(unittest.IsolatedAsyncioTestCase):
|
||||
"content": "改成 Sapphire Tide Runner,并让 CEO 自己修改/删除/新增 work item。",
|
||||
})
|
||||
|
||||
self.handler._track_session.assert_not_called()
|
||||
self.handler._track_session.assert_called_once()
|
||||
self.handler._process_company_suspend_reply.assert_called_once()
|
||||
call = self.handler._process_company_suspend_reply.call_args.kwargs
|
||||
self.assertEqual(call["parent_task_id"], self.task_id)
|
||||
self.assertEqual(call["parent_session_id"], self.session_id)
|
||||
self.assertEqual(call["ui_task_id"], self.task_id)
|
||||
self.assertEqual(call["runtime_session_id"], self.session_id)
|
||||
self.assertEqual(call["checkpoint"].checkpoint_id, "cp-suspended")
|
||||
self.assertEqual(call["content"], "改成 Sapphire Tide Runner,并让 CEO 自己修改/删除/新增 work item。")
|
||||
|
||||
async def test_rejected_company_resume_refreshes_optimistic_runtime_control(self) -> None:
|
||||
task = await self.store.get_task(self.task_id)
|
||||
assert task is not None
|
||||
task.metadata = {"exec_mode": "company", "company_profile": "corporate"}
|
||||
await self.store.save_task(task)
|
||||
self.handler._refresh_runtime_control_for_client = AsyncMock()
|
||||
ws = MagicMock()
|
||||
|
||||
await self.handler._handle_session_resume(ws, {
|
||||
"project_id": "test-project",
|
||||
"task_id": self.task_id,
|
||||
"runtime_session_id": self.session_id,
|
||||
})
|
||||
|
||||
self.handler._send_ack.assert_awaited_once_with(
|
||||
ws,
|
||||
ok=False,
|
||||
error="missing_checkpoint_id",
|
||||
)
|
||||
self.handler._refresh_runtime_control_for_client.assert_awaited_once_with(
|
||||
ws,
|
||||
engine=self.engine,
|
||||
project_id="test-project",
|
||||
)
|
||||
|
||||
async def test_session_send_persists_attachment_refs_and_dispatches_them(self) -> None:
|
||||
"""Uploaded session attachments should be stored and forwarded into engine execution."""
|
||||
ws = MagicMock()
|
||||
@@ -3264,6 +3411,37 @@ class TestWSHandlerSessionStop(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertTrue(any(msg["payload"]["task_id"] == "stop-parent" and msg["payload"]["status"] == "idle" for msg in status_updates))
|
||||
self.assertTrue(any(msg["payload"]["task_id"] == "stop-child" and msg["payload"]["status"] == "cancelled" for msg in status_updates))
|
||||
|
||||
async def test_company_identity_failure_is_rejected_without_task_tree_cancel(self) -> None:
|
||||
ws = MagicMock()
|
||||
task = Task(
|
||||
id="company-stop-mismatch",
|
||||
title="Company runtime",
|
||||
session_id="company-stop-session",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.RUNNING,
|
||||
metadata={"exec_mode": "company", "company_profile": "corporate"},
|
||||
)
|
||||
await self.store.save_task(task)
|
||||
self.handler._resolve_company_runtime_target = AsyncMock(return_value=None)
|
||||
self.handler._cancel_task_tree = AsyncMock()
|
||||
|
||||
await self.handler._handle_session_stop(
|
||||
ws,
|
||||
{"project_id": "test-project", "task_id": task.id},
|
||||
)
|
||||
|
||||
self.handler._cancel_task_tree.assert_not_awaited()
|
||||
self.handler._send_ack.assert_awaited_once_with(
|
||||
ws,
|
||||
ok=False,
|
||||
error="company_runtime_identity_mismatch",
|
||||
project_id="test-project",
|
||||
task_id=task.id,
|
||||
)
|
||||
persisted = await self.store.get_task(task.id)
|
||||
assert persisted is not None
|
||||
self.assertEqual(persisted.status, TaskStatus.RUNNING)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Test 6: WSHandler — on_opc_event child_session_created
|
||||
@@ -3711,7 +3889,7 @@ class TestWSHandlerSessionDetail(unittest.IsolatedAsyncioTestCase):
|
||||
"execution_checkpoint_lifecycle",
|
||||
)
|
||||
|
||||
async def test_session_detail_includes_runtime_control_state(self) -> None:
|
||||
async def test_session_detail_uses_controller_registry_for_runtime_control_state(self) -> None:
|
||||
ws = MagicMock()
|
||||
ws.send_json = AsyncMock()
|
||||
task = Task(
|
||||
@@ -3735,8 +3913,17 @@ class TestWSHandlerSessionDetail(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
payload = ws.send_json.await_args.args[0]["payload"]
|
||||
self.assertTrue(payload["ok"])
|
||||
self.assertEqual(payload["session_state"].get("runtime_control_state"), "running")
|
||||
self.assertTrue(payload["session_state"].get("can_stop"))
|
||||
self.assertEqual(payload["session_state"].get("runtime_control_state"), "idle")
|
||||
self.assertFalse(payload["session_state"].get("can_stop"))
|
||||
|
||||
self.engine._task_runtime_is_live = AsyncMock(return_value=True)
|
||||
await self.handler._handle_session_detail(
|
||||
ws,
|
||||
{"project_id": "test-project", "task_id": "custom-running-task"},
|
||||
)
|
||||
live_payload = ws.send_json.await_args.args[0]["payload"]
|
||||
self.assertEqual(live_payload["session_state"].get("runtime_control_state"), "running")
|
||||
self.assertTrue(live_payload["session_state"].get("can_stop"))
|
||||
|
||||
async def test_session_detail_prefers_task_description_for_role_prompt_context(self) -> None:
|
||||
ws = MagicMock()
|
||||
@@ -4582,10 +4769,13 @@ class TestWSHandlerShutdown(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
ws.send_json.assert_awaited_once()
|
||||
|
||||
async def test_shutdown_closes_clients_and_waits_for_active_messages(self) -> None:
|
||||
async def test_shutdown_closes_clients_and_cancels_non_handoff_messages(self) -> None:
|
||||
ws = MagicMock()
|
||||
ws.closed = False
|
||||
ws.closing = False
|
||||
self.handler._root_engine.prepare_active_company_runtimes_for_shutdown = AsyncMock(
|
||||
return_value=[]
|
||||
)
|
||||
|
||||
async def _close(*_args: Any, **_kwargs: Any) -> None:
|
||||
ws.closed = True
|
||||
@@ -4593,23 +4783,16 @@ class TestWSHandlerShutdown(unittest.IsolatedAsyncioTestCase):
|
||||
ws.close = AsyncMock(side_effect=_close)
|
||||
self.handler._clients.add(ws)
|
||||
|
||||
blocker = asyncio.Event()
|
||||
|
||||
async def _active_message() -> None:
|
||||
await blocker.wait()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
active_task = asyncio.create_task(_active_message())
|
||||
self.handler._active_message_tasks.add(active_task)
|
||||
|
||||
shutdown_task = asyncio.create_task(self.handler.shutdown(timeout=0.5))
|
||||
await asyncio.sleep(0.05)
|
||||
self.assertFalse(shutdown_task.done())
|
||||
|
||||
blocker.set()
|
||||
await shutdown_task
|
||||
await active_task
|
||||
await self.handler.shutdown(timeout=0.5)
|
||||
|
||||
ws.close.assert_awaited_once()
|
||||
self.assertTrue(active_task.cancelled())
|
||||
self.assertTrue(self.handler._shutting_down)
|
||||
|
||||
|
||||
@@ -4967,6 +5150,312 @@ class TestOfficeServiceExecutionIdentity(unittest.IsolatedAsyncioTestCase):
|
||||
async def asyncTearDown(self) -> None:
|
||||
await self.chat_store._db.close()
|
||||
|
||||
async def test_continue_uses_scope_config_for_markerless_cancelled_ui_anchor(self) -> None:
|
||||
anchor = Task(
|
||||
id="service-ui-anchor",
|
||||
title="Company chat",
|
||||
session_id="service-runtime-session",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.CANCELLED,
|
||||
metadata={},
|
||||
)
|
||||
final_decider = Task(
|
||||
id="service-final-decider",
|
||||
title="Final decision",
|
||||
session_id="service-runtime-session",
|
||||
parent_session_id="service-runtime-session",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.BLOCKED,
|
||||
linked_work_item_id="service-work-item",
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"company_profile": "corporate",
|
||||
"work_item_runtime": True,
|
||||
"work_item_projection_id": "final",
|
||||
"shared_role_session": True,
|
||||
},
|
||||
)
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="service-runtime-checkpoint",
|
||||
project_id="test-project",
|
||||
session_id="service-runtime-session",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id=final_decider.id,
|
||||
payload={"parent_session_id": "service-runtime-session"},
|
||||
)
|
||||
await self.store.save_task(anchor)
|
||||
await self.store.save_task(final_decider)
|
||||
await self.store.save_execution_checkpoint(checkpoint)
|
||||
|
||||
await self.session_service.continue_run(
|
||||
project_id="test-project",
|
||||
task_id=anchor.id,
|
||||
runtime_session_id="service-runtime-session",
|
||||
checkpoint_id=checkpoint.checkpoint_id,
|
||||
content="continue",
|
||||
)
|
||||
|
||||
call = self.engine.process_message.await_args
|
||||
self.assertEqual(call.kwargs["mode"], "company")
|
||||
self.assertEqual(call.kwargs["session_id"], "service-runtime-session")
|
||||
self.assertEqual(call.kwargs["origin_task_id"], anchor.id)
|
||||
self.assertEqual(
|
||||
call.kwargs["message_metadata"]["response_to_checkpoint_id"],
|
||||
checkpoint.checkpoint_id,
|
||||
)
|
||||
persisted_anchor = await self.store.get_task(anchor.id)
|
||||
assert persisted_anchor is not None
|
||||
self.assertEqual(persisted_anchor.status, TaskStatus.CANCELLED)
|
||||
|
||||
async def test_company_continue_preserves_requested_work_item_as_ui_channel(self) -> None:
|
||||
anchor = Task(
|
||||
id="service-channel-anchor",
|
||||
title="Company chat",
|
||||
session_id="service-channel-runtime",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.CANCELLED,
|
||||
metadata={"exec_mode": "company", "company_profile": "corporate"},
|
||||
)
|
||||
work_item = Task(
|
||||
id="service-channel-work-item",
|
||||
title="Shared final decision",
|
||||
session_id="service-channel-runtime",
|
||||
parent_session_id="service-channel-runtime",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.BLOCKED,
|
||||
linked_work_item_id="service-channel-wi",
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"company_profile": "corporate",
|
||||
"work_item_runtime": True,
|
||||
"work_item_projection_id": "final",
|
||||
"shared_role_session": True,
|
||||
},
|
||||
)
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="service-channel-checkpoint",
|
||||
project_id="test-project",
|
||||
session_id="service-channel-runtime",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id=work_item.id,
|
||||
payload={"parent_session_id": "service-channel-runtime"},
|
||||
)
|
||||
await self.store.save_task(anchor)
|
||||
await self.store.save_task(work_item)
|
||||
await self.store.save_execution_checkpoint(checkpoint)
|
||||
|
||||
result = await self.session_service.continue_run(
|
||||
project_id="test-project",
|
||||
task_id=work_item.id,
|
||||
runtime_session_id="service-channel-runtime",
|
||||
checkpoint_id=checkpoint.checkpoint_id,
|
||||
content="continue",
|
||||
)
|
||||
|
||||
call = self.engine.process_message.await_args
|
||||
self.assertEqual(result.payload["task_id"], work_item.id)
|
||||
self.assertEqual(call.kwargs["session_id"], "service-channel-runtime")
|
||||
self.assertEqual(call.kwargs["origin_task_id"], anchor.id)
|
||||
|
||||
async def test_company_identity_failure_never_falls_back_to_task_mode_control(self) -> None:
|
||||
from opc.plugins.office_ui.services.models import ServiceError
|
||||
|
||||
task = Task(
|
||||
id="service-company-control",
|
||||
title="Company control",
|
||||
session_id="service-company-session",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.RUNNING,
|
||||
metadata={"exec_mode": "company", "company_profile": "corporate"},
|
||||
)
|
||||
await self.store.save_task(task)
|
||||
mismatch = ServiceError(
|
||||
"company_runtime_identity_mismatch",
|
||||
"identity mismatch",
|
||||
)
|
||||
self.session_service._resolve_company_runtime_target = AsyncMock(
|
||||
side_effect=mismatch,
|
||||
)
|
||||
|
||||
with self.assertRaises(ServiceError) as stop_error:
|
||||
await self.session_service.stop(
|
||||
project_id="test-project",
|
||||
task_id=task.id,
|
||||
)
|
||||
self.assertEqual(stop_error.exception.code, "company_runtime_identity_mismatch")
|
||||
|
||||
with self.assertRaises(ServiceError) as continue_error:
|
||||
await self.session_service.continue_run(
|
||||
project_id="test-project",
|
||||
task_id=task.id,
|
||||
)
|
||||
self.assertEqual(continue_error.exception.code, "company_runtime_identity_mismatch")
|
||||
persisted = await self.store.get_task(task.id)
|
||||
assert persisted is not None
|
||||
self.assertEqual(persisted.status, TaskStatus.RUNNING)
|
||||
|
||||
async def test_session_send_from_work_item_uses_runtime_checkpoint_identity(self) -> None:
|
||||
anchor = Task(
|
||||
id="service-send-anchor",
|
||||
title="Company chat",
|
||||
session_id="service-send-runtime",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.CANCELLED,
|
||||
metadata={},
|
||||
)
|
||||
final_decider = Task(
|
||||
id="service-send-final",
|
||||
title="Final decider",
|
||||
session_id="service-send-runtime",
|
||||
parent_session_id="service-send-runtime",
|
||||
project_id="test-project",
|
||||
linked_work_item_id="service-send-final-wi",
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"company_profile": "corporate",
|
||||
"work_item_runtime": True,
|
||||
"work_item_projection_id": "final",
|
||||
"shared_role_session": True,
|
||||
},
|
||||
)
|
||||
worker = Task(
|
||||
id="service-send-worker",
|
||||
title="Worker",
|
||||
session_id="service-send-runtime:role:worker",
|
||||
parent_session_id="service-send-runtime",
|
||||
project_id="test-project",
|
||||
linked_work_item_id="service-send-worker-wi",
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"company_profile": "corporate",
|
||||
"work_item_runtime": True,
|
||||
"work_item_projection_id": "worker",
|
||||
},
|
||||
)
|
||||
original_worker_metadata = dict(worker.metadata)
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="service-send-checkpoint",
|
||||
project_id="test-project",
|
||||
session_id="service-send-runtime",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id=final_decider.id,
|
||||
payload={"parent_session_id": "service-send-runtime"},
|
||||
)
|
||||
for task in (anchor, final_decider, worker):
|
||||
await self.store.save_task(task)
|
||||
await self.store.save_execution_checkpoint(checkpoint)
|
||||
|
||||
result = await self.session_service.send(
|
||||
project_id="test-project",
|
||||
task_id=worker.id,
|
||||
content="revise and continue",
|
||||
)
|
||||
|
||||
call = self.engine.process_message.await_args
|
||||
self.assertEqual(result.payload["task_id"], worker.id)
|
||||
self.assertEqual(result.payload["session_id"], "service-send-runtime")
|
||||
self.assertEqual(call.kwargs["session_id"], "service-send-runtime")
|
||||
self.assertEqual(call.kwargs["origin_task_id"], anchor.id)
|
||||
self.assertEqual(call.kwargs["mode"], "company")
|
||||
self.assertEqual(call.kwargs["message_metadata"], {
|
||||
"response_to_checkpoint_id": checkpoint.checkpoint_id,
|
||||
"response_to_checkpoint_type": checkpoint.checkpoint_type,
|
||||
})
|
||||
persisted_worker = await self.store.get_task(worker.id)
|
||||
assert persisted_worker is not None
|
||||
self.assertEqual(persisted_worker.metadata, original_worker_metadata)
|
||||
|
||||
async def test_session_send_rejects_resuming_checkpoint_without_engine_fallback(self) -> None:
|
||||
anchor = Task(
|
||||
id="service-resuming-anchor",
|
||||
title="Company chat",
|
||||
session_id="service-resuming-runtime",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.CANCELLED,
|
||||
metadata={"exec_mode": "company", "company_profile": "corporate"},
|
||||
)
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="service-resuming-checkpoint",
|
||||
project_id="test-project",
|
||||
session_id="service-resuming-runtime",
|
||||
checkpoint_type="company_runtime_suspended",
|
||||
status="resuming",
|
||||
task_id=anchor.id,
|
||||
payload={"parent_session_id": "service-resuming-runtime"},
|
||||
)
|
||||
await self.store.save_task(anchor)
|
||||
await self.store.save_execution_checkpoint(checkpoint)
|
||||
|
||||
with self.assertRaises(ServiceError) as raised:
|
||||
await self.session_service.send(
|
||||
project_id="test-project",
|
||||
task_id=anchor.id,
|
||||
content="continue twice",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
raised.exception.code,
|
||||
"company_runtime_checkpoint_not_pending",
|
||||
)
|
||||
self.engine.process_message.assert_not_called()
|
||||
|
||||
async def test_session_send_rejects_cancelled_task_mode_session(self) -> None:
|
||||
task = Task(
|
||||
id="service-cancelled-task-mode",
|
||||
title="Cancelled task chat",
|
||||
session_id="service-cancelled-task-session",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.CANCELLED,
|
||||
metadata={
|
||||
"mode": "task",
|
||||
"execution_mode": "task_mode",
|
||||
"origin_task_id": "service-cancelled-task-mode",
|
||||
},
|
||||
)
|
||||
await self.store.save_task(task)
|
||||
|
||||
with self.assertRaises(ServiceError) as raised:
|
||||
await self.session_service.send(
|
||||
project_id="test-project",
|
||||
task_id=task.id,
|
||||
content="must stay cancelled",
|
||||
)
|
||||
|
||||
self.assertEqual(raised.exception.code, "session_ended")
|
||||
self.engine.process_message.assert_not_called()
|
||||
|
||||
async def test_session_send_company_identity_mismatch_fails_closed(self) -> None:
|
||||
task = Task(
|
||||
id="service-send-mismatch",
|
||||
title="Company chat",
|
||||
session_id="service-send-mismatch-runtime",
|
||||
project_id="test-project",
|
||||
metadata={"exec_mode": "company", "company_profile": "corporate"},
|
||||
)
|
||||
await self.store.save_task(task)
|
||||
self.session_service._resolve_company_runtime_target = AsyncMock(
|
||||
side_effect=ServiceError(
|
||||
"company_runtime_identity_mismatch",
|
||||
"identity mismatch",
|
||||
),
|
||||
)
|
||||
|
||||
with self.assertRaises(ServiceError) as raised:
|
||||
await self.session_service.send(
|
||||
project_id="test-project",
|
||||
task_id=task.id,
|
||||
content="do not fall back",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
raised.exception.code,
|
||||
"company_runtime_identity_mismatch",
|
||||
)
|
||||
self.engine.process_message.assert_not_called()
|
||||
|
||||
async def test_session_send_prefers_persisted_org_identity_over_call_defaults(self) -> None:
|
||||
task = Task(
|
||||
id="task-org-send",
|
||||
|
||||
@@ -352,7 +352,6 @@ class DeliveryCardPhaseSyncTests(unittest.IsolatedAsyncioTestCase):
|
||||
executor._kanban_broadcast_task = None
|
||||
executor._kanban_debounce_sec = 0.2
|
||||
executor.runtime = MagicMock()
|
||||
executor._active_task_runs = set()
|
||||
executor._runtime_invariant_issue_keys = set()
|
||||
return executor
|
||||
|
||||
|
||||
@@ -384,7 +384,6 @@ def test_office_ui_work_item_paths_do_not_use_legacy_projection_identity_names()
|
||||
[
|
||||
REPO_ROOT / "opc/plugins/office_ui/ws_handler.py",
|
||||
REPO_ROOT / "opc/plugins/office_ui/snapshot_builder.py",
|
||||
REPO_ROOT / "opc/plugins/office_ui/recovery_manager.py",
|
||||
],
|
||||
)
|
||||
assert not backend_matches, _format_matches(backend_matches)
|
||||
|
||||
Reference in New Issue
Block a user