from __future__ import annotations import os import asyncio from datetime import datetime, timedelta from functools import wraps from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock import pytest from opc.core.active_task_runs import ActiveTaskRunRegistry from opc.core.models import ( CompanyMemberSession, DelegationWorkItem, ExecutionCheckpoint, ExternalSession, Phase, Task, TaskStatus, ) from opc.database.store import OPCStore from opc.engine import OPCEngine from opc.layer2_organization.company_mode import CompanyWorkItemExecutor from opc.layer2_organization.org_work_item_planner import ( CompanyWorkItemRuntimePlan, WorkItemProjectionSpec, serialize_company_work_item_plan, ) from opc.layer2_organization.work_item_links import set_linked_work_item_id def _async_test(func): @wraps(func) def runner(*args, **kwargs): return asyncio.run(func(*args, **kwargs)) return runner def _runtime_plan() -> CompanyWorkItemRuntimePlan: return CompanyWorkItemRuntimePlan( profile="corporate", projections=[ WorkItemProjectionSpec( projection_id="execution", turn_type="execute", title="Execution", summary="Execute the work.", role_id="executor", ) ], metadata={"execution_model": "multi_team_org"}, ) def _runtime_task(*, task_id: str, status: TaskStatus, metadata: dict | None = None) -> Task: return Task( id=task_id, title="Execution", session_id=f"role-session-{task_id}", parent_session_id="root-session", project_id="project-a", status=status, metadata={ "work_item_projection_id": "execution", "work_item_runtime": True, "execution_model": "multi_team_org", "company_work_item_plan": serialize_company_work_item_plan(_runtime_plan()), **dict(metadata or {}), }, ) def _ui_anchor(*, status: TaskStatus = TaskStatus.IDLE) -> Task: return Task( id="ui-anchor", title="Company chat", session_id="root-session", project_id="project-a", status=status, metadata={"exec_mode": "company", "company_profile": "corporate"}, ) @pytest.mark.parametrize("pid", [None, os.getpid(), 99999999]) @_async_test async def test_startup_creates_exactly_one_checkpoint_without_persisted_liveness( tmp_path: Path, pid: int | None, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: task = _runtime_task(task_id="running-task", status=TaskStatus.RUNNING) await store.save_task(task) await store.save_external_session( ExternalSession( agent_type="codex", project_id="project-a", session_id="codex:project-a:running-task", opc_session_id=str(task.session_id or ""), task_id=task.id, workspace_path=str(tmp_path), run_mode="interactive", status="working", metadata={ **({"pid": pid} if pid is not None else {}), "status_heartbeat_seconds": 30, }, updated_at=datetime.now(), ) ) engine = OPCEngine(project_id="project-a") engine.store = store engine.company_executor = SimpleNamespace(execute=AsyncMock()) assert await engine._reconcile_interrupted_project_tasks() == 1 assert await engine._reconcile_interrupted_project_tasks() == 0 checkpoints = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_interrupted"], statuses=["pending", "resuming"], ) assert len(checkpoints) == 1 engine.company_executor.execute.assert_not_awaited() finally: await store.close() @_async_test async def test_startup_normalizes_duplicate_active_scope_checkpoints( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: task = _runtime_task( task_id="duplicate-checkpoint-task", status=TaskStatus.RUNNING, ) await store.save_task(task) older = ExecutionCheckpoint( checkpoint_id="checkpoint-older", project_id="project-a", session_id="root-session", checkpoint_type="company_runtime_interrupted", status="pending", task_id=task.id, payload={"reason": "older"}, ) winner = ExecutionCheckpoint( checkpoint_id="checkpoint-winner", project_id="project-a", session_id="root-session", checkpoint_type="company_runtime_suspended", status="resuming", task_id=task.id, payload={"reason": "winner"}, ) winner.updated_at = older.updated_at + timedelta(microseconds=1) await store.save_execution_checkpoint(older) await store.save_execution_checkpoint(winner) engine = OPCEngine(project_id="project-a") engine.store = store await engine._reconcile_interrupted_project_tasks() active = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=[ "company_runtime_interrupted", "company_runtime_suspended", ], statuses=["pending", "resuming"], ) assert [checkpoint.checkpoint_id for checkpoint in active] == [ winner.checkpoint_id ] assert active[0].status == "pending" await store.resolve_execution_checkpoint(winner.checkpoint_id) assert await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=[ "company_runtime_interrupted", "company_runtime_suspended", ], statuses=["pending", "resuming"], ) == [] all_rows = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", ) by_id = {checkpoint.checkpoint_id: checkpoint for checkpoint in all_rows} assert by_id[older.checkpoint_id].status == "superseded" assert by_id[winner.checkpoint_id].status == "resolved" finally: await store.close() @_async_test async def test_legacy_failed_interruption_is_migrated_but_business_failure_is_not( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: legacy = _runtime_task( task_id="legacy-interrupted", status=TaskStatus.FAILED, metadata={ "interrupted_recovery": { "previous_status": "running", "reason": "process disappeared", } }, ) legacy.result = { "content": "interrupted", "artifacts": { "interrupted": True, "interrupted_previous_status": "running", }, } await store.save_delegation_work_item( DelegationWorkItem( work_item_id="legacy-work-item", run_id="legacy-run", cell_id="team::executor", role_id="executor", seat_id="seat::executor", title="Legacy work", kind="execute", projection_id="execution", phase=Phase.PAUSED, ) ) set_linked_work_item_id(legacy, "legacy-work-item") await store.save_task(legacy) await store.link_work_item_runtime_task("legacy-work-item", legacy.id) # Use a separate scope so the genuine failure cannot be included in # the legacy task's scope checkpoint. genuine = _runtime_task( task_id="business-failure", status=TaskStatus.FAILED, metadata={"interrupted_recovery": {"previous_status": "running"}}, ) genuine.parent_session_id = "failed-root-session" genuine.result = { "content": "real failure", "artifacts": {"interrupted": True, "interrupted_previous_status": "running"}, } await store.save_delegation_work_item( DelegationWorkItem( work_item_id="failed-work-item", run_id="failed-run", cell_id="team::executor", role_id="executor", seat_id="seat::executor", title="Failed work", kind="execute", projection_id="execution", phase=Phase.FAILED, ) ) set_linked_work_item_id(genuine, "failed-work-item") await store.save_task(genuine) await store.link_work_item_runtime_task("failed-work-item", genuine.id) engine = OPCEngine(project_id="project-a") engine.store = store await engine._reconcile_interrupted_project_tasks() migrated = await store.get_task(legacy.id) untouched = await store.get_task(genuine.id) checkpoints = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_interrupted"], statuses=["pending", "resuming"], ) failed_checkpoints = await store.get_execution_checkpoints( project_id="project-a", session_id="failed-root-session", checkpoint_types=["company_runtime_interrupted"], statuses=["pending", "resuming"], ) assert migrated is not None assert migrated.status != TaskStatus.FAILED assert "interrupted_recovery" not in migrated.metadata assert not (migrated.result or {}).get("artifacts", {}).get("interrupted") assert checkpoints[0].payload["legacy_interrupted_recovery"][0]["task_id"] == legacy.id assert untouched is not None and untouched.status == TaskStatus.FAILED assert failed_checkpoints == [] finally: await store.close() @_async_test async def test_legacy_migration_does_not_change_task_before_checkpoint_commit( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: legacy = _runtime_task( task_id="legacy-crash-window", status=TaskStatus.FAILED, metadata={"interrupted_recovery": {"previous_status": "running"}}, ) await store.save_task(legacy) engine = OPCEngine(project_id="project-a") engine.store = store engine._save_company_runtime_suspend_checkpoint = AsyncMock( side_effect=RuntimeError("checkpoint write failed") ) with pytest.raises(RuntimeError, match="checkpoint write failed"): await engine._reconcile_interrupted_project_tasks() persisted = await store.get_task(legacy.id) assert persisted is not None and persisted.status == TaskStatus.FAILED assert persisted.metadata["interrupted_recovery"]["previous_status"] == "running" finally: await store.close() @_async_test async def test_initialize_fails_closed_when_startup_checkpoint_write_fails( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: await store.save_task( _runtime_task(task_id="startup-write-failure", status=TaskStatus.RUNNING) ) engine = OPCEngine( opc_home=tmp_path / "opc-home", project_id="project-a", store=store, owns_store=False, ) engine._save_company_runtime_suspend_checkpoint = AsyncMock( side_effect=RuntimeError("checkpoint write failed") ) with pytest.raises(RuntimeError, match="checkpoint write failed"): await engine.initialize() assert not engine._initialized finally: await store.close() @_async_test async def test_shutdown_preparation_reuses_existing_checkpoint(tmp_path: Path) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() registry = ActiveTaskRunRegistry() try: task = _runtime_task(task_id="active-task", status=TaskStatus.RUNNING) work_item = DelegationWorkItem( work_item_id="active-work-item", run_id="active-run", cell_id="team::executor", role_id="executor", seat_id="seat::executor", title="Active work", kind="execute", projection_id="execution", phase=Phase.RUNNING, claimed_by_role_runtime_session_id="runtime-session", claimed_by_seat_id="seat::executor", metadata={ "claimed_by_role_session_id": "role-session", "claimed_task_id": task.id, }, ) await store.save_delegation_work_item(work_item) set_linked_work_item_id(task, work_item.work_item_id) await store.save_task(task) await store.link_work_item_runtime_task(work_item.work_item_id, task.id) token = registry.register("project-a", task.id) engine = OPCEngine( project_id="project-a", active_task_run_registry=registry, owns_active_task_run_registry=True, ) engine.store = store first = await engine.prepare_active_company_runtimes_for_shutdown() task_after_first = await store.get_task(task.id) work_item_after_first = await store.get_delegation_work_item(work_item.work_item_id) second = await engine.prepare_active_company_runtimes_for_shutdown() task_after_second = await store.get_task(task.id) work_item_after_second = await store.get_delegation_work_item(work_item.work_item_id) checkpoints = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_interrupted"], statuses=["pending", "resuming"], ) assert len(checkpoints) == 1 assert first[0]["checkpoint_id"] == second[0]["checkpoint_id"] assert not first[0]["idempotent"] assert second[0]["idempotent"] assert second[0]["task_ids"] == [] assert task_after_first is not None and task_after_second is not None assert task_after_second.metadata == task_after_first.metadata assert work_item_after_first is not None and work_item_after_second is not None assert work_item_after_second.metadata == work_item_after_first.metadata assert engine._shutting_down registry.unregister("project-a", task.id, token) finally: await store.close() @_async_test async def test_shutdown_captures_post_execution_pre_finalize_work_item( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() registry = ActiveTaskRunRegistry() try: await store.save_task(_ui_anchor()) task = _runtime_task(task_id="finalize-gap-task", status=TaskStatus.DONE) work_item = DelegationWorkItem( work_item_id="finalize-gap-item", run_id="finalize-gap-run", cell_id="team::executor", role_id="executor", seat_id="seat::executor", title="Finalize gap", kind="execute", projection_id="execution", phase=Phase.RUNNING, ) await store.save_delegation_work_item(work_item) set_linked_work_item_id(task, work_item.work_item_id) await store.save_task(task) await store.link_work_item_runtime_task(work_item.work_item_id, task.id) token = registry.register("project-a", task.id) engine = OPCEngine( project_id="project-a", active_task_run_registry=registry, owns_active_task_run_registry=True, ) engine.store = store prepared = await engine.prepare_active_company_runtimes_for_shutdown() checkpoints = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_interrupted"], statuses=["pending"], ) persisted_task = await store.get_task(task.id) persisted_item = await store.get_delegation_work_item(work_item.work_item_id) assert len(prepared) == 1 assert len(checkpoints) == 1 assert persisted_task is not None and persisted_task.status == TaskStatus.DONE assert persisted_task.metadata["dispatch_hold"] == "company_runtime_suspended" assert persisted_item is not None and persisted_item.phase == Phase.RUNNING assert persisted_item.metadata["dispatch_hold"] == "company_runtime_suspended" registry.unregister("project-a", task.id, token) finally: await store.close() @_async_test async def test_shutdown_admission_close_cannot_orphan_claim_before_child_spawn( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() registry = ActiveTaskRunRegistry() driver_ownership = None child_execution = None try: await store.save_task(_ui_anchor()) task = _runtime_task(task_id="claim-spawn-gap-task", status=TaskStatus.PENDING) work_item = DelegationWorkItem( work_item_id="claim-spawn-gap-item", run_id="claim-spawn-gap-run", cell_id="team::executor", role_id="executor", seat_id="seat::executor", title="Claim/spawn gap", kind="execute", projection_id="execution", phase=Phase.READY, ) await store.save_delegation_work_item(work_item) set_linked_work_item_id(task, work_item.work_item_id) await store.save_task(task) await store.link_work_item_runtime_task(work_item.work_item_id, task.id) engine = OPCEngine( project_id="project-a", active_task_run_registry=registry, owns_active_task_run_registry=True, ) engine.store = store executor = object.__new__(CompanyWorkItemExecutor) executor.active_task_run_registry = registry child_started = asyncio.Event() hold_child = asyncio.Event() async def run_claimed(*_args: object, **_kwargs: object): child_started.set() await hold_child.wait() executor._run_claimed_work_item = run_claimed driver_ownership = executor.acquire_driver_ownership([task]) assert driver_ownership is not None with driver_ownership.bind(): claimed = await store.claim_delegation_work_item_if_dispatchable( work_item.work_item_id, expected_phase=Phase.READY, role_runtime_session_id="role-session", seat_id="seat::executor", task_id=task.id, ) assert claimed is not None registry.close_admission() child_execution = executor._create_claimed_work_item_task( CompanyMemberSession( member_session_id="member-session", role_session_id="role-session", role_id="executor", seat_id="seat::executor", ), task, {}, ) await child_started.wait() prepared = await engine.prepare_active_company_runtimes_for_shutdown() checkpoints = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_interrupted"], statuses=["pending"], ) item_after = await store.get_delegation_work_item(work_item.work_item_id) assert len(prepared) == 1 assert len(checkpoints) == 1 assert item_after is not None assert item_after.metadata["dispatch_hold"] == "company_runtime_suspended" assert item_after.claimed_by_role_runtime_session_id == "" finally: if child_execution is not None: child_execution.cancel() await asyncio.gather(child_execution, return_exceptions=True) if driver_ownership is not None: driver_ownership.release() await store.close() @_async_test async def test_startup_uses_nonterminal_work_item_over_terminal_task_projection( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: await store.save_task(_ui_anchor()) stale_task = _runtime_task(task_id="stale-terminal-task", status=TaskStatus.DONE) terminal_task = _runtime_task(task_id="terminal-pair-task", status=TaskStatus.DONE) stale_item = DelegationWorkItem( work_item_id="stale-nonterminal-item", run_id="terminal-projection-run", cell_id="team::executor", role_id="executor", seat_id="seat::executor", title="Still running", kind="execute", projection_id="execution", phase=Phase.RUNNING, ) terminal_item = DelegationWorkItem( work_item_id="terminal-item", run_id="terminal-projection-run", cell_id="team::reviewer", role_id="reviewer", seat_id="seat::reviewer", title="Already approved", kind="review", projection_id="review", phase=Phase.APPROVED, ) for task, item in ( (stale_task, stale_item), (terminal_task, terminal_item), ): await store.save_delegation_work_item(item) set_linked_work_item_id(task, item.work_item_id) await store.save_task(task) await store.link_work_item_runtime_task(item.work_item_id, task.id) engine = OPCEngine(project_id="project-a") engine.store = store updated = await engine._reconcile_interrupted_project_tasks() checkpoints = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_interrupted"], statuses=["pending"], ) stale_after = await store.get_task(stale_task.id) stale_item_after = await store.get_delegation_work_item(stale_item.work_item_id) terminal_after = await store.get_task(terminal_task.id) terminal_item_after = await store.get_delegation_work_item(terminal_item.work_item_id) assert updated >= 1 assert len(checkpoints) == 1 assert stale_after is not None and stale_after.status == TaskStatus.DONE assert stale_after.metadata["dispatch_hold"] == "company_runtime_suspended" assert stale_item_after is not None assert stale_item_after.metadata["dispatch_hold"] == "company_runtime_suspended" assert terminal_after is not None and terminal_after.status == TaskStatus.DONE assert "dispatch_hold" not in terminal_after.metadata assert terminal_item_after is not None and terminal_item_after.phase == Phase.APPROVED assert "dispatch_hold" not in terminal_item_after.metadata finally: await store.close() @_async_test async def test_startup_interrupts_ready_work_despite_stale_human_task_projection( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: await store.save_task(_ui_anchor()) task = _runtime_task( task_id="pre-claim-task", status=TaskStatus.AWAITING_HUMAN, ) work_item = DelegationWorkItem( work_item_id="pre-claim-item", run_id="pre-claim-run", cell_id="team::executor", role_id="executor", seat_id="seat::executor", title="Ready before first claim", kind="execute", projection_id="execution", phase=Phase.READY, ) await store.save_delegation_work_item(work_item) set_linked_work_item_id(task, work_item.work_item_id) await store.save_task(task) await store.link_work_item_runtime_task(work_item.work_item_id, task.id) engine = OPCEngine(project_id="project-a") engine.store = store engine.company_executor = SimpleNamespace(execute=AsyncMock()) updated = await engine._reconcile_interrupted_project_tasks() checkpoints = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_interrupted"], statuses=["pending"], ) task_after = await store.get_task(task.id) item_after = await store.get_delegation_work_item(work_item.work_item_id) assert updated >= 1 assert len(checkpoints) == 1 assert task_after is not None assert task_after.metadata["dispatch_hold"] == "company_runtime_suspended" assert item_after is not None and item_after.phase == Phase.READY assert item_after.metadata["dispatch_hold"] == "company_runtime_suspended" engine.company_executor.execute.assert_not_awaited() finally: await store.close() @_async_test async def test_startup_preserves_stable_human_work_item_despite_stale_task_projection( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: await store.save_task(_ui_anchor()) task = _runtime_task( task_id="stable-human-task", status=TaskStatus.PENDING, ) work_item = DelegationWorkItem( work_item_id="stable-human-item", run_id="stable-human-run", cell_id="team::reviewer", role_id="reviewer", seat_id="seat::reviewer", title="Awaiting human", kind="review", projection_id="execution", phase=Phase.AWAITING_HUMAN, ) await store.save_delegation_work_item(work_item) set_linked_work_item_id(task, work_item.work_item_id) await store.save_task(task) await store.link_work_item_runtime_task(work_item.work_item_id, task.id) engine = OPCEngine(project_id="project-a") engine.store = store updated = await engine._reconcile_interrupted_project_tasks() first_task_after = await store.get_task(task.id) assert first_task_after is not None first_marker = dict( first_task_after.metadata.get( "startup_reconcile_preserved_waiting_state", {}, ) or {} ) repeated = await engine._reconcile_interrupted_project_tasks() interrupted = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_interrupted"], statuses=["pending", "resuming"], ) task_after = await store.get_task(task.id) item_after = await store.get_delegation_work_item(work_item.work_item_id) assert updated >= 1 assert repeated == 0 assert interrupted == [] assert task_after is not None assert task_after.status == TaskStatus.PENDING assert task_after.metadata["startup_reconcile_preserved_waiting_state"] == first_marker assert "dispatch_hold" not in task_after.metadata assert item_after is not None and item_after.phase == Phase.AWAITING_HUMAN assert "dispatch_hold" not in item_after.metadata finally: await store.close() @_async_test async def test_resume_candidates_use_nonterminal_work_item_over_terminal_task( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: task = _runtime_task( task_id="terminal-resume-projection", status=TaskStatus.DONE, metadata={ "dispatch_hold": "company_runtime_suspended", "company_runtime_stop_state": "suspended", "suspended_task_status": TaskStatus.DONE.value, }, ) work_item = DelegationWorkItem( work_item_id="terminal-resume-item", run_id="terminal-resume-run", cell_id="team::executor", role_id="executor", seat_id="seat::executor", title="Resume authoritative item", kind="execute", projection_id="execution", phase=Phase.RUNNING, metadata={ "dispatch_hold": "company_runtime_suspended", "suspended_phase": Phase.RUNNING.value, }, ) await store.save_delegation_work_item(work_item) set_linked_work_item_id(task, work_item.work_item_id) await store.save_task(task) await store.link_work_item_runtime_task(work_item.work_item_id, task.id) engine = OPCEngine(project_id="project-a") engine.store = store candidates = await engine._company_suspend_resume_candidate_task_ids([task]) resumed = await engine._prepare_company_runtime_tasks_for_resume( [task], { "checkpoint_id": "resume-checkpoint", "checkpoint_type": "company_runtime_interrupted", "active_work_items": [ { "work_item_id": work_item.work_item_id, "phase": Phase.RUNNING.value, } ], }, resume_task_ids={task.id}, ) persisted_item = await store.get_delegation_work_item(work_item.work_item_id) assert candidates == {task.id} assert resumed[0].status == TaskStatus.RUNNING assert "dispatch_hold" not in resumed[0].metadata assert persisted_item is not None and persisted_item.phase == Phase.RUNNING assert not str(persisted_item.metadata.get("dispatch_hold", "") or "").strip() finally: await store.close() @_async_test async def test_user_stop_and_shutdown_share_exactly_one_scope_checkpoint( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() registry = ActiveTaskRunRegistry() try: task = _runtime_task(task_id="concurrent-task", status=TaskStatus.RUNNING) work_item = DelegationWorkItem( work_item_id="concurrent-work-item", run_id="concurrent-run", cell_id="team::executor", role_id="executor", seat_id="seat::executor", title="Concurrent work", kind="execute", projection_id="execution", phase=Phase.RUNNING, ) await store.save_task(_ui_anchor()) await store.save_delegation_work_item(work_item) set_linked_work_item_id(task, work_item.work_item_id) await store.save_task(task) await store.link_work_item_runtime_task(work_item.work_item_id, task.id) token = registry.register("project-a", task.id) engine = OPCEngine( project_id="project-a", active_task_run_registry=registry, owns_active_task_run_registry=True, ) engine.store = store stopped, prepared = await asyncio.gather( engine.suspend_company_runtime( origin_task_id=task.id, session_id="root-session", reason="user_stop", ), engine.prepare_active_company_runtimes_for_shutdown(), ) checkpoints = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=[ "company_runtime_suspended", "company_runtime_interrupted", ], statuses=["pending", "resuming"], ) assert stopped is not None assert len(prepared) == 1 assert len(checkpoints) == 1 assert stopped["checkpoint_id"] == prepared[0]["checkpoint_id"] assert {stopped["idempotent"], prepared[0]["idempotent"]} == {False, True} registry.unregister("project-a", task.id, token) finally: await store.close() @_async_test async def test_atomic_work_item_claim_rejects_durable_hold_and_post_commit_stop( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: held = DelegationWorkItem( work_item_id="held-claim-item", run_id="claim-run", cell_id="team::executor", role_id="executor", seat_id="seat::executor", title="Held claim", kind="execute", projection_id="held", phase=Phase.READY, metadata={"dispatch_hold": "company_runtime_suspended"}, ) await store.save_delegation_work_item(held) rejected = await store.claim_delegation_work_item_if_dispatchable( held.work_item_id, expected_phase=Phase.READY, role_runtime_session_id="role-session", seat_id="seat::executor", task_id="runtime-task", ) held_after = await store.get_delegation_work_item(held.work_item_id) assert rejected is None assert held_after is not None assert held_after.claimed_by_role_runtime_session_id == "" revised = DelegationWorkItem( work_item_id="revised-claim-item", run_id="claim-run", cell_id="team::executor", role_id="executor", seat_id="seat::executor", title="Revised claim", kind="execute", projection_id="revised", phase=Phase.READY, metadata={"manager_mutation_revision": 2}, ) await store.save_delegation_work_item(revised) stale_revision = await store.claim_delegation_work_item_if_dispatchable( revised.work_item_id, expected_phase=Phase.READY, role_runtime_session_id="role-session", seat_id="seat::executor", task_id="runtime-task", work_item_revision=1, ) revised_after = await store.get_delegation_work_item(revised.work_item_id) assert stale_revision is None assert revised_after is not None assert revised_after.phase == Phase.READY assert revised_after.claimed_by_role_runtime_session_id == "" raced = DelegationWorkItem( work_item_id="post-commit-stop-item", run_id="claim-run", cell_id="team::executor", role_id="executor", seat_id="seat::executor", title="Post-commit stop", kind="execute", projection_id="raced", phase=Phase.READY, ) await store.save_delegation_work_item(raced) original_get = store.get_delegation_work_item stop_injected = False async def get_after_stop(work_item_id: str): nonlocal stop_injected if work_item_id == raced.work_item_id and not stop_injected: stop_injected = True await store.update_delegation_work_item( work_item_id, metadata_updates={ "dispatch_hold": "company_runtime_suspended", "claimed_by_role_session_id": "", "claimed_task_id": "", }, claimed_by_role_runtime_session_id="", claimed_by_seat_id="", ) return await original_get(work_item_id) store.get_delegation_work_item = get_after_stop # type: ignore[method-assign] post_commit = await store.claim_delegation_work_item_if_dispatchable( raced.work_item_id, expected_phase=Phase.READY, role_runtime_session_id="role-session", seat_id="seat::executor", task_id="runtime-task", ) raced_after = await original_get(raced.work_item_id) assert post_commit is None assert raced_after is not None assert raced_after.metadata["dispatch_hold"] == "company_runtime_suspended" assert raced_after.claimed_by_role_runtime_session_id == "" finally: await store.close() @_async_test async def test_shared_final_decider_without_parent_uses_runtime_identity_scope( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: anchor = _ui_anchor(status=TaskStatus.RUNNING) shared_final = Task( id="shared-final", title="Final decision", session_id="root-session", parent_session_id=None, project_id="project-a", status=TaskStatus.RUNNING, metadata={ "work_item_runtime": True, "work_item_projection_id": "execution", "shared_role_session": True, "company_runtime_root_session_id": "root-session", "company_work_item_plan": serialize_company_work_item_plan( _runtime_plan() ), }, ) await store.save_task(anchor) await store.save_task(shared_final) engine = OPCEngine(project_id="project-a") engine.store = store snapshot = await engine._load_company_runtime_snapshot("root-session") reconciled = await engine._reconcile_interrupted_project_tasks() checkpoints = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_interrupted"], statuses=["pending"], ) assert snapshot is not None assert [task.id for task in snapshot[1]] == [shared_final.id] assert reconciled == 2 assert len(checkpoints) == 1 refreshed_anchor = await store.get_task(anchor.id) assert refreshed_anchor is not None assert refreshed_anchor.status == TaskStatus.IDLE finally: await store.close() @_async_test async def test_successful_checkpoint_handoff_reopens_cancelled_ui_anchor( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: anchor = Task( id="ui-anchor", title="Company chat", session_id="root-session", project_id="project-a", status=TaskStatus.CANCELLED, metadata={}, ) runtime_task = _runtime_task(task_id="runtime-task", status=TaskStatus.BLOCKED) await store.save_task(anchor) await store.save_task(runtime_task) checkpoint = ExecutionCheckpoint( project_id="project-a", session_id="root-session", checkpoint_type="company_runtime_interrupted", task_id=runtime_task.id, payload={"task_ids": [runtime_task.id], "parent_session_id": "root-session"}, ) await store.save_execution_checkpoint(checkpoint) engine = OPCEngine(project_id="project-a") engine.store = store engine.company_executor = SimpleNamespace() engine._prepare_company_runtime_tasks_for_resume = AsyncMock( return_value=[runtime_task] ) engine._reset_company_executor_runtime_for_resume = AsyncMock() engine._clear_company_runtime_parent_stop_state = AsyncMock() handed_off = await engine._handoff_company_suspend_checkpoint( checkpoint, payload=dict(checkpoint.payload), parent_session_id="root-session", tasks=[runtime_task], ) still_cancelled = await store.get_task(anchor.id) resuming = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_interrupted"], statuses=["resuming"], ) assert handed_off is not None handed_off_tasks, driver_ownership = handed_off assert handed_off_tasks == [runtime_task] assert driver_ownership is not None driver_ownership.release() assert still_cancelled is not None assert still_cancelled.status == TaskStatus.CANCELLED assert len(resuming) == 1 await engine._complete_company_suspend_checkpoint_resume( checkpoint, parent_session_id="root-session", ) reopened = await store.get_task(anchor.id) persisted_checkpoints = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_interrupted"], statuses=["resolved"], ) assert reopened is not None and reopened.status == TaskStatus.IDLE assert len(persisted_checkpoints) == 1 finally: await store.close() @_async_test async def test_executor_failure_restores_pending_checkpoint_and_durable_holds( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: anchor = _ui_anchor(status=TaskStatus.CANCELLED) task = _runtime_task(task_id="failing-task", status=TaskStatus.RUNNING) work_item = DelegationWorkItem( work_item_id="failing-work-item", run_id="failing-run", cell_id="team::executor", role_id="executor", seat_id="seat::executor", title="Failing work", kind="execute", projection_id="execution", phase=Phase.RUNNING, claimed_by_role_runtime_session_id="role-runtime", claimed_by_seat_id="seat::executor", metadata={ "claimed_by_role_session_id": "role-session", "claimed_task_id": task.id, }, ) await store.save_task(anchor) await store.save_delegation_work_item(work_item) set_linked_work_item_id(task, work_item.work_item_id) await store.save_task(task) await store.link_work_item_runtime_task(work_item.work_item_id, task.id) engine = OPCEngine(project_id="project-a") engine.store = store suspended = await engine.suspend_company_runtime( origin_task_id=task.id, session_id="root-session", reason="user_stop", ) assert suspended is not None checkpoint = ( await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_suspended"], statuses=["pending"], ) )[0] engine.company_executor = SimpleNamespace( execute=AsyncMock(side_effect=RuntimeError("executor failed")) ) with pytest.raises(RuntimeError, match="executor failed"): await engine._resume_company_suspend_checkpoint(checkpoint, "continue") pending = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_suspended"], statuses=["pending"], ) refreshed_task = await store.get_task(task.id) refreshed_item = await store.get_delegation_work_item(work_item.work_item_id) refreshed_anchor = await store.get_task(anchor.id) assert len(pending) == 1 assert pending[0].payload["resume_state"] == "failed_during_execution" assert refreshed_task is not None assert refreshed_task.metadata["dispatch_hold"] == "company_runtime_suspended" assert refreshed_task.metadata["company_runtime_stop_state"] == "suspended" assert refreshed_item is not None assert refreshed_item.metadata["dispatch_hold"] == "company_runtime_suspended" assert refreshed_item.claimed_by_role_runtime_session_id == "" assert refreshed_item.claimed_by_seat_id == "" assert refreshed_anchor is not None assert refreshed_anchor.status == TaskStatus.CANCELLED finally: await store.close() @_async_test async def test_concurrent_resume_requests_execute_checkpoint_once(tmp_path: Path) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: await store.save_task(_ui_anchor()) task = _runtime_task(task_id="single-resume-task", status=TaskStatus.RUNNING) await store.save_task(task) engine = OPCEngine(project_id="project-a") engine.store = store await engine.suspend_company_runtime( origin_task_id=task.id, session_id="root-session", reason="user_stop", ) checkpoint = ( await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_suspended"], statuses=["pending"], ) )[0] execute_started = asyncio.Event() allow_execute = asyncio.Event() calls = 0 async def execute(_plan, _tasks): nonlocal calls calls += 1 execute_started.set() await allow_execute.wait() return "resumed once" engine.company_executor = SimpleNamespace(execute=execute) first = asyncio.create_task( engine._resume_company_suspend_checkpoint(checkpoint, "continue") ) await execute_started.wait() persisted = ( await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_suspended"], statuses=["resuming"], ) )[0] second = await engine._resume_company_suspend_checkpoint( persisted, "continue", ) allow_execute.set() first_result = await first assert calls == 1 assert "already being resumed" in second assert "resumed once" in first_result resolved = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_suspended"], statuses=["resolved"], ) assert len(resolved) == 1 finally: await store.close() @_async_test async def test_separate_controllers_atomically_claim_checkpoint_once( tmp_path: Path, ) -> None: db_path = tmp_path / "tasks.db" first_store = OPCStore(db_path) second_store = OPCStore(db_path) await first_store.initialize() try: await first_store.save_task(_ui_anchor()) task = _runtime_task( task_id="cross-controller-resume-task", status=TaskStatus.RUNNING, ) work_item = DelegationWorkItem( work_item_id="cross-controller-work-item", run_id="cross-controller-run", cell_id="team::executor", role_id="executor", seat_id="seat::executor", title="Cross-controller work", kind="execute", projection_id="execution", phase=Phase.RUNNING, ) await first_store.save_delegation_work_item(work_item) set_linked_work_item_id(task, work_item.work_item_id) await first_store.save_task(task) await first_store.link_work_item_runtime_task(work_item.work_item_id, task.id) setup_engine = OPCEngine(project_id="project-a") setup_engine.store = first_store await setup_engine.suspend_company_runtime( origin_task_id=task.id, session_id="root-session", reason="user_stop", ) await second_store.initialize() first_checkpoint = ( await first_store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_suspended"], statuses=["pending"], ) )[0] second_checkpoint = ( await second_store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_suspended"], statuses=["pending"], ) )[0] first_engine = OPCEngine(project_id="project-a") second_engine = OPCEngine(project_id="project-a") first_engine.store = first_store second_engine.store = second_store calls = 0 async def execute(_plan, _tasks): nonlocal calls calls += 1 return "claimed" first_engine.company_executor = SimpleNamespace(execute=execute) second_engine.company_executor = SimpleNamespace(execute=execute) # Hold both controllers after their independent pending reads so the # conditional UPDATE, rather than scheduling luck, decides ownership. both_read = asyncio.Event() read_count = 0 async def resolve_anchor(**_kwargs): nonlocal read_count read_count += 1 if read_count == 2: both_read.set() await both_read.wait() return "ui-anchor" first_engine._resolve_company_runtime_ui_anchor_task_id = resolve_anchor second_engine._resolve_company_runtime_ui_anchor_task_id = resolve_anchor results = await asyncio.gather( first_engine._resume_company_suspend_checkpoint( first_checkpoint, "continue", ), second_engine._resume_company_suspend_checkpoint( second_checkpoint, "continue", ), ) assert calls == 1 assert sum("already being resumed" in result for result in results) == 1 resolved = await first_store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_suspended"], statuses=["resolved"], ) assert len(resolved) == 1 finally: await second_store.close() await first_store.close() @_async_test async def test_runtime_without_ui_anchor_resumes_without_promoting_work_item( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: task = _runtime_task(task_id="headless-runtime-task", status=TaskStatus.RUNNING) await store.save_task(task) engine = OPCEngine(project_id="project-a") engine.store = store await engine.suspend_company_runtime( origin_task_id=task.id, session_id="root-session", reason="user_stop", ) checkpoint = ( await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_suspended"], statuses=["pending"], ) )[0] engine.company_executor = SimpleNamespace( execute=AsyncMock(return_value="headless resumed") ) result = await engine._resume_company_suspend_checkpoint( checkpoint, "continue", ) resolved = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_suspended"], statuses=["resolved"], ) refreshed = await store.get_task(task.id) assert "headless resumed" in result assert len(resolved) == 1 assert resolved[0].payload.get("ui_anchor_task_id") == "" assert refreshed is not None assert refreshed.status != TaskStatus.IDLE finally: await store.close() @_async_test async def test_canonical_suspend_wins_real_executor_result_commit_race( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: task = _runtime_task(task_id="racing-task", status=TaskStatus.PENDING) task.assigned_to = "executor" task.metadata.update({ "runtime_model": "multi_team_org", "work_item_role_id": "executor", "work_item_execution_strategy": "native", }) work_item = DelegationWorkItem( work_item_id="racing-work-item", run_id="racing-run", cell_id="team::executor", role_id="executor", seat_id="seat::executor", title="Racing work", kind="execute", projection_id="execution", phase=Phase.READY, ) await store.save_delegation_work_item(work_item) set_linked_work_item_id(task, work_item.work_item_id) await store.save_task(task) await store.link_work_item_runtime_task(work_item.work_item_id, task.id) run_started = asyncio.Event() allow_result = asyncio.Event() async def run_once(_task: Task): run_started.set() await allow_result.wait() return SimpleNamespace( status=TaskStatus.DONE, content="must not commit", artifacts={}, ) class OrgEngine: @staticmethod def get_role_for_work_item(_role_id, _tags): return SimpleNamespace( role_id="executor", preferred_external_agent=None, default_agent=None, tools=[], ) engine = OPCEngine(project_id="project-a") engine.store = store engine._run_task_once = run_once executor = CompanyWorkItemExecutor( org_engine=OrgEngine(), communication=None, approval_engine=SimpleNamespace(), memory=None, execute_task=engine._execute_task, save_task=store.save_task, store=store, ) execution = asyncio.create_task( executor._run_work_item(task, {"execution": task}) ) await run_started.wait() suspended = await engine.suspend_company_runtime( origin_task_id=task.id, session_id="root-session", reason="service_shutdown", checkpoint_type="company_runtime_interrupted", ) allow_result.set() with pytest.raises(asyncio.CancelledError): await execution checkpoints = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_interrupted"], statuses=["pending"], ) refreshed_task = await store.get_task(task.id) refreshed_item = await store.get_delegation_work_item(work_item.work_item_id) assert suspended is not None assert len(checkpoints) == 1 assert refreshed_task is not None assert refreshed_task.metadata["dispatch_hold"] == "company_runtime_suspended" assert refreshed_task.metadata["company_runtime_suspend_checkpoint_type"] == "company_runtime_interrupted" assert refreshed_task.result is None assert refreshed_item is not None assert refreshed_item.phase not in {Phase.APPROVED, Phase.FAILED, Phase.CANCELLED} assert refreshed_item.metadata["dispatch_hold"] == "company_runtime_suspended" finally: await store.close() @pytest.mark.parametrize("interrupt_kind", ["stop", "shutdown"]) @_async_test async def test_stop_or_shutdown_cannot_release_remaining_final_decider_work( tmp_path: Path, interrupt_kind: str, ) -> None: store = OPCStore(tmp_path / f"tasks-{interrupt_kind}.db") await store.initialize() registry = ActiveTaskRunRegistry() token = "" try: plan = CompanyWorkItemRuntimePlan( profile="corporate", projections=[ WorkItemProjectionSpec( projection_id="final", turn_type="deliver", title="Final decision", summary="Arbitrate the follow-up.", role_id="ceo", ), WorkItemProjectionSpec( projection_id="worker", turn_type="execute", title="Worker", summary="Continue implementation.", role_id="engineer", ), ], metadata={ "execution_model": "multi_team_org", "final_decider_role_id": "ceo", }, ) serialized_plan = serialize_company_work_item_plan(plan) await store.save_task(_ui_anchor()) final_task = Task( id="final-task", title="Final decision", session_id="final-session", parent_session_id="root-session", project_id="project-a", assigned_to="ceo", status=TaskStatus.RUNNING, metadata={ "work_item_runtime": True, "work_item_projection_id": "final", "company_work_item_plan": serialized_plan, }, ) worker_task = Task( id="worker-task", title="Worker", session_id="worker-session", parent_session_id="root-session", project_id="project-a", assigned_to="engineer", status=TaskStatus.RUNNING, metadata={ "work_item_runtime": True, "work_item_projection_id": "worker", "company_work_item_plan": serialized_plan, }, ) for task, work_item_id, projection, role in ( (final_task, "final-item", "final", "ceo"), (worker_task, "worker-item", "worker", "engineer"), ): await store.save_delegation_work_item( DelegationWorkItem( work_item_id=work_item_id, run_id="final-run", cell_id=f"team::{role}", role_id=role, seat_id=f"seat::{role}", title=task.title, kind="deliver" if role == "ceo" else "execute", projection_id=projection, phase=Phase.RUNNING, ) ) set_linked_work_item_id(task, work_item_id) await store.save_task(task) await store.link_work_item_runtime_task(work_item_id, task.id) engine = OPCEngine( project_id="project-a", active_task_run_registry=registry, owns_active_task_run_registry=True, ) engine.store = store engine.company_executor = SimpleNamespace(execute=AsyncMock()) await engine.suspend_company_runtime( origin_task_id=worker_task.id, session_id="root-session", reason="user_stop", ) checkpoint = ( await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_suspended"], statuses=["pending"], ) )[0] loaded = await engine._load_company_suspend_checkpoint_runtime(checkpoint) assert loaded is not None payload, parent_session_id, loaded_plan, tasks = loaded handoff = await engine._handoff_company_suspend_checkpoint( checkpoint, payload=payload, parent_session_id=parent_session_id, tasks=tasks, resume_task_ids={final_task.id}, ) assert handoff is not None resumed_tasks, initial_driver_ownership = handoff if initial_driver_ownership is not None: initial_driver_ownership.release() progress_checked = asyncio.Event() allow_remaining_handoff = asyncio.Event() async def final_progressed(_task_id: str) -> bool: progress_checked.set() await allow_remaining_handoff.wait() return True engine._company_followup_target_progressed = final_progressed continuation = asyncio.create_task( engine._resume_remaining_company_runtime_after_final_decider( checkpoint=checkpoint, plan=loaded_plan, tasks=resumed_tasks, payload=payload, parent_session_id=parent_session_id, final_decider_task_id=final_task.id, ) ) await progress_checked.wait() if interrupt_kind == "stop": await engine.suspend_company_runtime( origin_task_id=final_task.id, session_id="root-session", reason="user_stop", ) else: token = registry.register("project-a", final_task.id) await engine.prepare_active_company_runtimes_for_shutdown() allow_remaining_handoff.set() continuation_owned, continuation_result = await continuation assert not continuation_owned assert continuation_result is None engine.company_executor.execute.assert_not_awaited() pending = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_suspended"], statuses=["pending"], ) assert len(pending) == 1 for task_id, work_item_id in ( (final_task.id, "final-item"), (worker_task.id, "worker-item"), ): refreshed_task = await store.get_task(task_id) refreshed_item = await store.get_delegation_work_item(work_item_id) assert refreshed_task is not None assert refreshed_item is not None assert refreshed_task.metadata["dispatch_hold"] == "company_runtime_suspended" assert refreshed_item.metadata["dispatch_hold"] == "company_runtime_suspended" finally: if token: registry.unregister("project-a", final_task.id, token) await store.close() @_async_test async def test_final_decider_without_arbitration_keeps_multi_item_checkpoint_pending( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: plan = CompanyWorkItemRuntimePlan( profile="corporate", projections=[ WorkItemProjectionSpec( projection_id="final", turn_type="deliver", title="Final decision", summary="Arbitrate the follow-up.", role_id="ceo", ), WorkItemProjectionSpec( projection_id="worker", turn_type="execute", title="Worker", summary="Continue implementation.", role_id="engineer", ), ], metadata={ "execution_model": "multi_team_org", "final_decider_role_id": "ceo", }, ) serialized_plan = serialize_company_work_item_plan(plan) await store.save_task(_ui_anchor()) final_task = Task( id="final-task-no-arbitration", title="Final decision", session_id="final-session-no-arbitration", parent_session_id="root-session", project_id="project-a", assigned_to="ceo", status=TaskStatus.RUNNING, metadata={ "work_item_runtime": True, "work_item_projection_id": "final", "company_work_item_plan": serialized_plan, }, ) worker_task = Task( id="worker-task-still-held", title="Worker", session_id="worker-session-still-held", parent_session_id="root-session", project_id="project-a", assigned_to="engineer", status=TaskStatus.RUNNING, metadata={ "work_item_runtime": True, "work_item_projection_id": "worker", "company_work_item_plan": serialized_plan, }, ) for task, work_item_id, projection, role in ( (final_task, "final-item-no-arbitration", "final", "ceo"), (worker_task, "worker-item-still-held", "worker", "engineer"), ): await store.save_delegation_work_item( DelegationWorkItem( work_item_id=work_item_id, run_id="no-arbitration-run", cell_id=f"team::{role}", role_id=role, seat_id=f"seat::{role}", title=task.title, kind="deliver" if role == "ceo" else "execute", projection_id=projection, phase=Phase.RUNNING, ) ) set_linked_work_item_id(task, work_item_id) await store.save_task(task) await store.link_work_item_runtime_task(work_item_id, task.id) engine = OPCEngine(project_id="project-a") engine.store = store engine.company_executor = SimpleNamespace(execute=AsyncMock()) engine._resume_company_runtime_via_final_decider = AsyncMock( return_value="Final-decider turn returned without arbitration." ) suspended = await engine.suspend_company_runtime( origin_task_id=worker_task.id, session_id="root-session", reason="user_stop", ) assert suspended is not None active = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_suspended"], statuses=["pending"], ) assert len(active) == 1 checkpoint = active[0] response = await engine._resume_company_suspend_checkpoint_via_final_decider( checkpoint, "Continue and arbitrate the remaining work.", ) pending = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_suspended"], statuses=["pending"], ) resolved = await store.get_execution_checkpoints( project_id="project-a", session_id="root-session", checkpoint_types=["company_runtime_suspended"], statuses=["resolved"], ) assert len(pending) == 1 assert pending[0].checkpoint_id == checkpoint.checkpoint_id assert pending[0].payload["resume_state"] == "awaiting_final_decider_action" assert resolved == [] assert "durable arbitration action" in response assert "Resumed remaining company runtime" not in response engine.company_executor.execute.assert_not_awaited() for task_id, work_item_id in ( (final_task.id, "final-item-no-arbitration"), (worker_task.id, "worker-item-still-held"), ): refreshed_task = await store.get_task(task_id) refreshed_item = await store.get_delegation_work_item(work_item_id) assert refreshed_task is not None assert refreshed_item is not None assert refreshed_task.metadata["dispatch_hold"] == "company_runtime_suspended" assert refreshed_item.metadata["dispatch_hold"] == "company_runtime_suspended" finally: await store.close() @_async_test async def test_startup_does_not_replay_resolved_checkpoint_into_ui_anchor( tmp_path: Path, ) -> None: store = OPCStore(tmp_path / "tasks.db") await store.initialize() try: anchor = Task( id="crash-gap-anchor", title="Company chat", session_id="crash-gap-session", project_id="project-a", status=TaskStatus.CANCELLED, metadata={}, ) await store.save_task(anchor) checkpoint = ExecutionCheckpoint( project_id="project-a", session_id="crash-gap-session", checkpoint_type="company_runtime_interrupted", task_id="runtime-task", status="resolved", payload={ "resume_state": "handoff_complete", "ui_anchor_task_id": anchor.id, }, ) await store.save_execution_checkpoint(checkpoint) engine = OPCEngine(project_id="project-a") engine.store = store assert await engine._reconcile_interrupted_project_tasks() == 0 still_cancelled = await store.get_task(anchor.id) assert still_cancelled is not None and still_cancelled.status == TaskStatus.CANCELLED finally: await store.close()