fix(company): add dispatch attempt ledger to brake infinite re-dispatch loops (#10)
Work items whose execution kept dying without a durable verdict (crash mid-dispatch, kill -9, cancel-before-harvest) were re-dispatched forever: every exit path was responsible for remembering to write a terminal phase, and any path that forgot left the card RUNNING and eligible again. Replace that with structural accounting: - claim CAS opens an attempt in the same UPDATE (attempt_seq+1, attempt_settled=false) so no dispatch can start unaccounted (store.py) - transition_work_item becomes the settlement authority: every non-RUNNING transition settles the open attempt in the same write; crashed/interrupted outcomes accumulate streaks, clean outcomes reset them; claim release folds into the same write; settlement still lands when the phase write loses a race (work_item_transition.py) - dispatcher refuses cards over the streak limits (crash>=3, interrupted>=5) in both is_dispatchable and _work_item_is_runnable, and a per-tick reconcile pass back-fills dead attempts as interrupted and terminalizes over-limit cards to FAILED with a visible blocked_reason (dispatch_hold quarantine if even that write fails) (phase.py, company_mode.py) - crash exits now settle: cancellation unwind harvests coroutines that died on a real exception before discarding them, the crashed-item handler releases the claim and settles as crashed with a quarantine fallback, and the timeout path settles as crashed (company_mode.py) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -385,6 +385,57 @@ def is_orphaned(item: Any) -> bool:
|
||||
return not claim
|
||||
|
||||
|
||||
# ── Attempt ledger ───────────────────────────────────────────────────────
|
||||
#
|
||||
# Every durable claim opens an "attempt" on the work item (stamped by the
|
||||
# claim CAS in store.claim_delegation_work_item_if_dispatchable) and every
|
||||
# turn-boundary transition through work_item_transition.transition_work_item
|
||||
# settles it with an outcome. The dispatcher refuses to start attempt N+1
|
||||
# for cards whose ledger shows a run of consecutive crashed / interrupted
|
||||
# attempts — this is the structural brake that makes "a crash path forgot
|
||||
# (or failed) to write FAILED" converge instead of re-dispatching forever
|
||||
# (issue #10: restart → resume → deterministic crash → re-dispatch loop).
|
||||
|
||||
ATTEMPT_CRASH_STREAK_LIMIT = 3
|
||||
ATTEMPT_INTERRUPTED_STREAK_LIMIT = 5
|
||||
|
||||
|
||||
def _attempt_ledger_int(metadata: Mapping[str, Any], key: str) -> int:
|
||||
try:
|
||||
return int(metadata.get(key, 0) or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def attempt_ledger_dispatch_block_reason(metadata: Mapping[str, Any] | None) -> str:
|
||||
"""Non-empty human-readable reason when the attempt ledger forbids
|
||||
dispatching this card again; empty string when dispatch is allowed."""
|
||||
if not isinstance(metadata, Mapping):
|
||||
return ""
|
||||
crash_streak = _attempt_ledger_int(metadata, "attempt_crash_streak")
|
||||
if crash_streak >= ATTEMPT_CRASH_STREAK_LIMIT:
|
||||
return (
|
||||
f"attempt ledger: {crash_streak} consecutive crashed attempts "
|
||||
f"(limit {ATTEMPT_CRASH_STREAK_LIMIT})"
|
||||
)
|
||||
interrupted_streak = _attempt_ledger_int(metadata, "attempt_interrupted_streak")
|
||||
if interrupted_streak >= ATTEMPT_INTERRUPTED_STREAK_LIMIT:
|
||||
return (
|
||||
f"attempt ledger: {interrupted_streak} consecutive interrupted attempts "
|
||||
f"(limit {ATTEMPT_INTERRUPTED_STREAK_LIMIT})"
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
def has_open_attempt(metadata: Mapping[str, Any] | None) -> bool:
|
||||
"""True when a claim opened an attempt that was never settled."""
|
||||
if not isinstance(metadata, Mapping):
|
||||
return False
|
||||
if _attempt_ledger_int(metadata, "attempt_seq") <= 0:
|
||||
return False
|
||||
return not bool(metadata.get("attempt_settled", True))
|
||||
|
||||
|
||||
def is_dispatchable(item: Any) -> bool:
|
||||
"""Combined check used by the dispatcher: pick this card on next tick?
|
||||
|
||||
@@ -405,6 +456,8 @@ def is_dispatchable(item: Any) -> bool:
|
||||
return False
|
||||
if isinstance(metadata, dict) and str(metadata.get("dispatch_hold", "") or "").strip():
|
||||
return False
|
||||
if attempt_ledger_dispatch_block_reason(metadata):
|
||||
return False
|
||||
return is_runnable(item.phase) or is_orphaned(item)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user