Gitinbox/OpenOPC
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
9aed328d02e97f6b5dde561d443c47bdf6fb2e98
OpenOPC/tests/test_attempt_ledger.py
T
LZH-YS1998 9aed328d02 fix(company): add dispatch attempt ledger to brake infinite re-dispatch loops (#10) 
Work items whose execution kept dying without a durable verdict (crash
mid-dispatch, kill -9, cancel-before-harvest) were re-dispatched forever:
every exit path was responsible for remembering to write a terminal
phase, and any path that forgot left the card RUNNING and eligible again.

Replace that with structural accounting:

- claim CAS opens an attempt in the same UPDATE (attempt_seq+1,
  attempt_settled=false) so no dispatch can start unaccounted (store.py)
- transition_work_item becomes the settlement authority: every
  non-RUNNING transition settles the open attempt in the same write;
  crashed/interrupted outcomes accumulate streaks, clean outcomes reset
  them; claim release folds into the same write; settlement still lands
  when the phase write loses a race (work_item_transition.py)
- dispatcher refuses cards over the streak limits (crash>=3,
  interrupted>=5) in both is_dispatchable and _work_item_is_runnable,
  and a per-tick reconcile pass back-fills dead attempts as interrupted
  and terminalizes over-limit cards to FAILED with a visible
  blocked_reason (dispatch_hold quarantine if even that write fails)
  (phase.py, company_mode.py)
- crash exits now settle: cancellation unwind harvests coroutines that
  died on a real exception before discarding them, the crashed-item
  handler releases the claim and settles as crashed with a quarantine
  fallback, and the timeout path settles as crashed (company_mode.py)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-21 11:18:12 +08:00

556 lines
22 KiB
Python
Raw Blame History

"""Attempt-ledger settlement tests (issue #10 root fix).
Covers the structural anti-loop mechanism:
- the claim CAS opens a durable attempt (attempt_seq / attempt_settled)
- transition_work_item settles the attempt on every turn-boundary phase write
- crashed / interrupted streaks accumulate and the dispatcher refuses
over-limit cards (is_dispatchable + _work_item_is_runnable)
- the per-tick reconcile pass back-fills dead attempts and terminalizes
over-limit cards with a visible blocked_reason
- resume availability gate: a work item pinned to a disabled external agent
fails closed at resume-prep instead of crash-looping through dispatch
"""
from __future__ import annotations
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock
from opc.core.config import OPCConfig, RoleConfig
from opc.core.events import EventBus
from opc.core.models import (
DelegationRoleSession,
DelegationWorkItem,
Phase,
Task,
TaskStatus,
)
from opc.database.store import OPCStore
from opc.engine import OPCEngine
from opc.layer2_organization.communication import CommunicationManager
from opc.layer2_organization.company_mode import (
CompanyWorkItemExecutor,
serialize_company_work_item_runtime_plan,
)
from opc.layer2_organization.org_engine import OrgEngine
from opc.layer2_organization.org_work_item_planner import (
CompanyWorkItemRuntimePlan,
WorkItemProjectionSpec,
)
from opc.layer2_organization.phase import (
ATTEMPT_CRASH_STREAK_LIMIT,
ATTEMPT_INTERRUPTED_STREAK_LIMIT,
attempt_ledger_dispatch_block_reason,
has_open_attempt,
is_dispatchable,
)
from opc.layer2_organization.work_item_links import set_linked_work_item_id
from opc.layer2_organization.work_item_transition import (
settle_open_attempt_as_interrupted,
transition_work_item,
)
class AttemptLedgerStoreTests(unittest.IsolatedAsyncioTestCase):
async def _store(self) -> OPCStore:
tmpdir = tempfile.TemporaryDirectory()
self.addCleanup(tmpdir.cleanup)
store = OPCStore(Path(tmpdir.name) / "tasks.db")
await store.initialize()
self.addAsyncCleanup(store.close)
return store
async def _seed_item(
self,
store: OPCStore,
*,
work_item_id: str = "wi-1",
phase: Phase = Phase.READY,
metadata: dict | None = None,
) -> DelegationWorkItem:
item = DelegationWorkItem(
work_item_id=work_item_id,
run_id="run-1",
role_id="executor",
seat_id="seat-1",
title="Execution",
summary="Do the work.",
kind="execute",
projection_id="execution",
phase=phase,
metadata=dict(metadata or {}),
)
await store.save_delegation_work_item(item)
return item
async def _claim(self, store: OPCStore, work_item_id: str, phase: Phase) -> DelegationWorkItem:
claimed = await store.claim_delegation_work_item_if_dispatchable(
work_item_id,
expected_phase=phase,
role_runtime_session_id="role-sess-1",
seat_id="seat-1",
task_id="task-1",
)
assert claimed is not None, "claim CAS unexpectedly failed"
return claimed
async def test_claim_cas_opens_attempt(self) -> None:
store = await self._store()
await self._seed_item(store)
claimed = await self._claim(store, "wi-1", Phase.READY)
metadata = dict(claimed.metadata or {})
self.assertEqual(int(metadata.get("attempt_seq")), 1)
self.assertFalse(bool(metadata.get("attempt_settled")))
self.assertTrue(str(metadata.get("attempt_started_at", "")).strip())
self.assertTrue(has_open_attempt(metadata))
# Settle + release, then re-claim the orphaned RUNNING card: seq += 1.
await transition_work_item(
store,
"wi-1",
target_phase=Phase.READY,
reason="recovery",
release_claim=True,
)
settled = await store.get_delegation_work_item("wi-1")
assert settled is not None
self.assertTrue(bool(settled.metadata.get("attempt_settled")))
# transition folded the claim clear into the same write, but the claim
# mirror metadata keys are only dropped on terminal phases — clear them
# the way the resume/suspend paths do before re-claiming.
await store.update_delegation_work_item(
"wi-1",
metadata_updates={"claimed_by_role_session_id": "", "claimed_task_id": ""},
)
reclaimed = await self._claim(store, "wi-1", Phase.READY)
self.assertEqual(int(reclaimed.metadata.get("attempt_seq")), 2)
self.assertFalse(bool(reclaimed.metadata.get("attempt_settled")))
async def test_transition_settles_attempt_clean(self) -> None:
store = await self._store()
await self._seed_item(store)
await self._claim(store, "wi-1", Phase.READY)
await transition_work_item(
store,
"wi-1",
target_phase=Phase.AWAITING_MANAGER_REVIEW,
reason="turn_done",
)
item = await store.get_delegation_work_item("wi-1")
assert item is not None
metadata = dict(item.metadata or {})
self.assertTrue(bool(metadata.get("attempt_settled")))
self.assertEqual(metadata.get("attempt_outcome"), Phase.AWAITING_MANAGER_REVIEW.value)
self.assertEqual(int(metadata.get("attempt_crash_streak")), 0)
self.assertEqual(int(metadata.get("attempt_interrupted_streak")), 0)
self.assertFalse(has_open_attempt(metadata))
async def test_crash_streak_accumulates_and_blocks_dispatch(self) -> None:
store = await self._store()
await self._seed_item(store)
phase = Phase.READY
for round_index in range(ATTEMPT_CRASH_STREAK_LIMIT):
await self._claim(store, "wi-1", phase)
# Crash + recovery-exit back to READY (RUNNING → READY is the
# legal crash-recovery edge) with outcome=crashed.
await transition_work_item(
store,
"wi-1",
target_phase=Phase.READY,
reason="crash_recovery",
release_claim=True,
attempt_outcome="crashed",
)
await store.update_delegation_work_item(
"wi-1",
metadata_updates={"claimed_by_role_session_id": "", "claimed_task_id": ""},
)
item = await store.get_delegation_work_item("wi-1")
assert item is not None
self.assertEqual(
int(item.metadata.get("attempt_crash_streak")), round_index + 1
)
phase = item.phase
item = await store.get_delegation_work_item("wi-1")
assert item is not None
self.assertTrue(attempt_ledger_dispatch_block_reason(item.metadata))
self.assertFalse(is_dispatchable(item))
# A clean settle resets the streak and dispatch reopens.
await store.update_delegation_work_item(
"wi-1",
metadata_updates={
"attempt_crash_streak": 0,
},
)
item = await store.get_delegation_work_item("wi-1")
assert item is not None
self.assertTrue(is_dispatchable(item))
async def test_settle_interrupted_is_idempotent_and_blocks_at_limit(self) -> None:
store = await self._store()
await self._seed_item(store)
for round_index in range(ATTEMPT_INTERRUPTED_STREAK_LIMIT):
claimed = await self._claim(
store, "wi-1", Phase.READY if round_index == 0 else Phase.RUNNING
)
settled = await settle_open_attempt_as_interrupted(store, claimed)
self.assertTrue(settled)
# Second settle on the same attempt is a no-op.
self.assertFalse(await settle_open_attempt_as_interrupted(store, claimed))
item = await store.get_delegation_work_item("wi-1")
assert item is not None
self.assertEqual(
int(item.metadata.get("attempt_interrupted_streak")), round_index + 1
)
# Free the claim like the suspend/startup sweeps do.
await store.update_delegation_work_item(
"wi-1",
claimed_by_role_runtime_session_id="",
claimed_by_seat_id="",
metadata_updates={"claimed_by_role_session_id": "", "claimed_task_id": ""},
)
item = await store.get_delegation_work_item("wi-1")
assert item is not None
self.assertTrue(attempt_ledger_dispatch_block_reason(item.metadata))
self.assertFalse(is_dispatchable(item))
class AttemptLedgerReconcileTests(unittest.IsolatedAsyncioTestCase):
async def _executor(self) -> tuple[CompanyWorkItemExecutor, OPCStore]:
tmpdir = tempfile.TemporaryDirectory()
self.addCleanup(tmpdir.cleanup)
root = Path(tmpdir.name)
store = OPCStore(root / "tasks.db")
await store.initialize()
self.addAsyncCleanup(store.close)
config = OPCConfig()
config.org.company_profile = "custom"
config.org.final_decider_role_id = "executor"
config.org.roles = [
RoleConfig(id="executor", name="Executor", responsibility="Do work.", reports_to="owner"),
]
org_engine = OrgEngine(config, root)
communication = CommunicationManager(store, EventBus(), llm=None, org_engine=org_engine)
executor = CompanyWorkItemExecutor(
org_engine=org_engine,
communication=communication,
approval_engine=SimpleNamespace(),
memory=None,
execute_task=AsyncMock(),
save_task=store.save_task,
store=store,
llm=None,
)
return executor, store
async def test_reconcile_backfills_dead_attempt_and_terminalizes(self) -> None:
executor, store = await self._executor()
# A card whose owner died: open attempt, no claim, one interruption
# short of the limit — the reconcile pass must settle (streak hits the
# limit) and terminalize with a visible blocked_reason.
item = DelegationWorkItem(
work_item_id="wi-dead",
run_id="run-1",
role_id="executor",
seat_id="seat-1",
title="Doomed",
summary="Crash loops forever.",
kind="execute",
projection_id="doomed",
phase=Phase.RUNNING,
metadata={
"attempt_seq": ATTEMPT_INTERRUPTED_STREAK_LIMIT,
"attempt_settled": False,
"attempt_interrupted_streak": ATTEMPT_INTERRUPTED_STREAK_LIMIT - 1,
},
)
await store.save_delegation_work_item(item)
reconciled = await executor._reconcile_attempt_ledger([item], {})
refreshed = await store.get_delegation_work_item("wi-dead")
assert refreshed is not None
self.assertEqual(refreshed.phase, Phase.FAILED)
self.assertTrue(bool(refreshed.metadata.get("attempt_settled")))
self.assertEqual(refreshed.metadata.get("attempt_outcome"), "interrupted")
self.assertEqual(
int(refreshed.metadata.get("attempt_interrupted_streak")),
ATTEMPT_INTERRUPTED_STREAK_LIMIT,
)
self.assertIn("attempt ledger", str(refreshed.blocked_reason or ""))
self.assertEqual(len(reconciled), 1)
async def test_reconcile_leaves_live_and_healthy_items_alone(self) -> None:
executor, store = await self._executor()
healthy = DelegationWorkItem(
work_item_id="wi-healthy",
run_id="run-1",
role_id="executor",
seat_id="seat-1",
title="Healthy",
summary="Fine.",
kind="execute",
projection_id="healthy",
phase=Phase.READY,
metadata={},
)
await store.save_delegation_work_item(healthy)
# Live item: open attempt but currently claimed by this runtime.
live = DelegationWorkItem(
work_item_id="wi-live",
run_id="run-1",
role_id="executor",
seat_id="seat-1",
title="Live",
summary="Running now.",
kind="execute",
projection_id="live",
phase=Phase.RUNNING,
metadata={"attempt_seq": 1, "attempt_settled": False},
)
await store.save_delegation_work_item(live)
executor.runtime._claimed_work_item_ids.add("wi-live")
await executor._reconcile_attempt_ledger([healthy, live], {})
refreshed_live = await store.get_delegation_work_item("wi-live")
assert refreshed_live is not None
self.assertEqual(refreshed_live.phase, Phase.RUNNING)
self.assertFalse(bool(refreshed_live.metadata.get("attempt_settled")))
refreshed_healthy = await store.get_delegation_work_item("wi-healthy")
assert refreshed_healthy is not None
self.assertEqual(refreshed_healthy.phase, Phase.READY)
self.assertNotIn("attempt_settled", dict(refreshed_healthy.metadata or {}))
class ResumeAvailabilityGateTests(unittest.IsolatedAsyncioTestCase):
"""Fix-1: a resume pin to a disabled external agent fails closed."""
async def _store(self) -> OPCStore:
tmpdir = tempfile.TemporaryDirectory()
self.addCleanup(tmpdir.cleanup)
store = OPCStore(Path(tmpdir.name) / "tasks.db")
await store.initialize()
self.addAsyncCleanup(store.close)
return store
def _plan(self) -> CompanyWorkItemRuntimePlan:
return CompanyWorkItemRuntimePlan(
profile="corporate",
projections=[
WorkItemProjectionSpec(
projection_id="execution",
turn_type="execute",
title="Execution",
summary="Produce the main execution output.",
role_id="executor",
)
],
metadata={
"execution_model": "multi_team_org",
"runtime_model": "multi_team_org",
"final_decider_role_id": "executor",
"top_level_role_ids": ["executor"],
},
)
async def _seed(self, store: OPCStore, *, external_agent: str) -> Task:
plan = self._plan()
await store.save_delegation_role_session(
DelegationRoleSession(
role_session_id="role-runtime-1",
run_id="run-1",
project_id="proj1",
role_id="executor",
seat_id="seat-1",
)
)
await store.save_delegation_work_item(
DelegationWorkItem(
work_item_id="work-item-1",
run_id="run-1",
role_id="executor",
seat_id="seat-1",
title="Execution",
summary="Execute the project.",
kind="execute",
projection_id="execution",
phase=Phase.RUNNING,
claimed_by_role_runtime_session_id="role-runtime-1",
claimed_by_seat_id="seat-1",
metadata={"work_item_projection_id": "execution"},
)
)
task = Task(
id="execution-task",
title="Execution",
session_id="sess-child",
parent_session_id="sess-parent",
status=TaskStatus.RUNNING,
project_id="proj1",
assigned_to="executor",
assigned_external_agent=external_agent,
execution_lock=True,
metadata={
"company_profile": "corporate",
"execution_model": "multi_team_org",
"runtime_model": "multi_team_org",
"work_item_runtime": True,
"work_item_projection_id": "execution",
"delegation_run_id": "run-1",
"delegation_role_session_id": "role-runtime-1",
"selected_execution_agent": external_agent,
"company_work_item_plan": serialize_company_work_item_runtime_plan(plan),
},
)
set_linked_work_item_id(task, "work-item-1")
await store.save_task(task)
await store.link_work_item_runtime_task("work-item-1", "execution-task")
return task
def _engine(self, store: OPCStore, *, available: list[str]) -> OPCEngine:
engine = OPCEngine()
engine.project_id = "proj1"
engine.store = store
engine.adapter_registry = SimpleNamespace(
list_available=lambda: list(available),
)
return engine
async def test_resume_fails_closed_when_pinned_agent_unavailable(self) -> None:
store = await self._store()
task = await self._seed(store, external_agent="codex")
engine = self._engine(store, available=["opencode"]) # codex disabled
suspended = await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
self.assertIsNotNone(suspended)
executed: dict[str, list[Task]] = {}
class DummyCompanyExecutor:
async def execute(self, _plan, tasks: list[Task]) -> str:
executed["tasks"] = tasks
return "runtime resumed"
engine.company_executor = DummyCompanyExecutor()
response = await engine._maybe_resume_checkpoint(
"continue",
"sess-parent",
reply_metadata={"ui_force_resume": True},
)
self.assertIsNotNone(response)
refreshed_item = await store.get_delegation_work_item("work-item-1")
assert refreshed_item is not None
self.assertEqual(refreshed_item.phase, Phase.FAILED)
self.assertIn("codex", str(refreshed_item.blocked_reason or ""))
refreshed_task = await store.get_task(task.id)
assert refreshed_task is not None
self.assertEqual(refreshed_task.status, TaskStatus.FAILED)
self.assertEqual(
refreshed_task.metadata.get("resume_unavailable_external_agent"),
"codex",
)
# The runtime still executed (the rest of the org resumes normally).
self.assertIn("tasks", executed)
async def test_plain_message_after_gate_failure_converges_without_revival(self) -> None:
"""A plain text follow-up (final-decider routing path) on a run whose
decider card failed terminally must drain the checkpoint and must not
clobber the FAILED task back to PENDING (InvalidPhaseTransition crash
found by the issue #10 end-to-end reproduction)."""
store = await self._store()
task = await self._seed(store, external_agent="codex")
engine = self._engine(store, available=["opencode"]) # codex disabled
await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
class DummyCompanyExecutor:
async def execute(self, _plan, tasks: list[Task]) -> str:
return "runtime resumed"
engine.company_executor = DummyCompanyExecutor()
# First resume: gate fails the codex-pinned decider card closed.
await engine._maybe_resume_checkpoint(
"continue",
"sess-parent",
reply_metadata={"ui_force_resume": True},
)
refreshed_item = await store.get_delegation_work_item("work-item-1")
assert refreshed_item is not None
self.assertEqual(refreshed_item.phase, Phase.FAILED)
# Re-suspend cannot happen (run is terminal) — but if a pending
# suspend checkpoint still exists, a plain message must converge
# instead of raising InvalidPhaseTransition. Route a plain message
# through the checkpoint machinery when present, else assert the
# terminal state simply holds.
response = await engine._maybe_resume_checkpoint("重跑", "sess-parent")
self.assertNotIsInstance(response, Exception)
refreshed_item = await store.get_delegation_work_item("work-item-1")
assert refreshed_item is not None
self.assertEqual(refreshed_item.phase, Phase.FAILED)
refreshed_task = await store.get_task(task.id)
assert refreshed_task is not None
self.assertEqual(
refreshed_task.status,
TaskStatus.FAILED,
"follow-up routing must not clobber the FAILED task projection",
)
remaining = await store.get_pending_checkpoints(
project_id="proj1",
session_id="sess-parent",
)
self.assertEqual(
[c.checkpoint_type for c in remaining],
[],
"suspend checkpoint must drain instead of bouncing back to pending",
)
async def test_resume_proceeds_when_pinned_agent_available(self) -> None:
store = await self._store()
task = await self._seed(store, external_agent="codex")
engine = self._engine(store, available=["codex", "opencode"])
suspended = await engine.suspend_company_runtime(
origin_task_id=task.id,
session_id="sess-parent",
reason="user_stop",
)
self.assertIsNotNone(suspended)
class DummyCompanyExecutor:
async def execute(self, _plan, tasks: list[Task]) -> str:
return "runtime resumed"
engine.company_executor = DummyCompanyExecutor()
await engine._maybe_resume_checkpoint(
"continue",
"sess-parent",
reply_metadata={"ui_force_resume": True},
)
refreshed_item = await store.get_delegation_work_item("work-item-1")
assert refreshed_item is not None
self.assertEqual(refreshed_item.phase, Phase.RUNNING)
refreshed_task = await store.get_task(task.id)
assert refreshed_task is not None
self.assertEqual(refreshed_task.status, TaskStatus.RUNNING)
self.assertEqual(
refreshed_task.metadata.get("_company_runtime_resume_execution_agent_pin", {}).get(
"selected_execution_agent"
),
"codex",
)
if __name__ == "__main__":
unittest.main()
Reference in New Issue View Git Blame Copy Permalink