fix(company): add dispatch attempt ledger to brake infinite re-dispatch loops (#10)

Work items whose execution kept dying without a durable verdict (crash
mid-dispatch, kill -9, cancel-before-harvest) were re-dispatched forever:
every exit path was responsible for remembering to write a terminal
phase, and any path that forgot left the card RUNNING and eligible again.

Replace that with structural accounting:

- claim CAS opens an attempt in the same UPDATE (attempt_seq+1,
  attempt_settled=false) so no dispatch can start unaccounted (store.py)
- transition_work_item becomes the settlement authority: every
  non-RUNNING transition settles the open attempt in the same write;
  crashed/interrupted outcomes accumulate streaks, clean outcomes reset
  them; claim release folds into the same write; settlement still lands
  when the phase write loses a race (work_item_transition.py)
- dispatcher refuses cards over the streak limits (crash>=3,
  interrupted>=5) in both is_dispatchable and _work_item_is_runnable,
  and a per-tick reconcile pass back-fills dead attempts as interrupted
  and terminalizes over-limit cards to FAILED with a visible
  blocked_reason (dispatch_hold quarantine if even that write fails)
  (phase.py, company_mode.py)
- crash exits now settle: cancellation unwind harvests coroutines that
  died on a real exception before discarding them, the crashed-item
  handler releases the claim and settles as crashed with a quarantine
  fallback, and the timeout path settles as crashed (company_mode.py)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
LZH-YS1998
2026-07-21 11:18:12 +08:00
parent eb934cf2bd
commit 9aed328d02
6 changed files with 968 additions and 26 deletions
+53
View File
@@ -385,6 +385,57 @@ def is_orphaned(item: Any) -> bool:
return not claim
# ── Attempt ledger ───────────────────────────────────────────────────────
#
# Every durable claim opens an "attempt" on the work item (stamped by the
# claim CAS in store.claim_delegation_work_item_if_dispatchable) and every
# turn-boundary transition through work_item_transition.transition_work_item
# settles it with an outcome. The dispatcher refuses to start attempt N+1
# for cards whose ledger shows a run of consecutive crashed / interrupted
# attempts — this is the structural brake that makes "a crash path forgot
# (or failed) to write FAILED" converge instead of re-dispatching forever
# (issue #10: restart → resume → deterministic crash → re-dispatch loop).
ATTEMPT_CRASH_STREAK_LIMIT = 3
ATTEMPT_INTERRUPTED_STREAK_LIMIT = 5
def _attempt_ledger_int(metadata: Mapping[str, Any], key: str) -> int:
try:
return int(metadata.get(key, 0) or 0)
except (TypeError, ValueError):
return 0
def attempt_ledger_dispatch_block_reason(metadata: Mapping[str, Any] | None) -> str:
"""Non-empty human-readable reason when the attempt ledger forbids
dispatching this card again; empty string when dispatch is allowed."""
if not isinstance(metadata, Mapping):
return ""
crash_streak = _attempt_ledger_int(metadata, "attempt_crash_streak")
if crash_streak >= ATTEMPT_CRASH_STREAK_LIMIT:
return (
f"attempt ledger: {crash_streak} consecutive crashed attempts "
f"(limit {ATTEMPT_CRASH_STREAK_LIMIT})"
)
interrupted_streak = _attempt_ledger_int(metadata, "attempt_interrupted_streak")
if interrupted_streak >= ATTEMPT_INTERRUPTED_STREAK_LIMIT:
return (
f"attempt ledger: {interrupted_streak} consecutive interrupted attempts "
f"(limit {ATTEMPT_INTERRUPTED_STREAK_LIMIT})"
)
return ""
def has_open_attempt(metadata: Mapping[str, Any] | None) -> bool:
"""True when a claim opened an attempt that was never settled."""
if not isinstance(metadata, Mapping):
return False
if _attempt_ledger_int(metadata, "attempt_seq") <= 0:
return False
return not bool(metadata.get("attempt_settled", True))
def is_dispatchable(item: Any) -> bool:
"""Combined check used by the dispatcher: pick this card on next tick?
@@ -405,6 +456,8 @@ def is_dispatchable(item: Any) -> bool:
return False
if isinstance(metadata, dict) and str(metadata.get("dispatch_hold", "") or "").strip():
return False
if attempt_ledger_dispatch_block_reason(metadata):
return False
return is_runnable(item.phase) or is_orphaned(item)