fix(company): add dispatch attempt ledger to brake infinite re-dispatch loops (#10)

Work items whose execution kept dying without a durable verdict (crash
mid-dispatch, kill -9, cancel-before-harvest) were re-dispatched forever:
every exit path was responsible for remembering to write a terminal
phase, and any path that forgot left the card RUNNING and eligible again.

Replace that with structural accounting:

- claim CAS opens an attempt in the same UPDATE (attempt_seq+1,
  attempt_settled=false) so no dispatch can start unaccounted (store.py)
- transition_work_item becomes the settlement authority: every
  non-RUNNING transition settles the open attempt in the same write;
  crashed/interrupted outcomes accumulate streaks, clean outcomes reset
  them; claim release folds into the same write; settlement still lands
  when the phase write loses a race (work_item_transition.py)
- dispatcher refuses cards over the streak limits (crash>=3,
  interrupted>=5) in both is_dispatchable and _work_item_is_runnable,
  and a per-tick reconcile pass back-fills dead attempts as interrupted
  and terminalizes over-limit cards to FAILED with a visible
  blocked_reason (dispatch_hold quarantine if even that write fails)
  (phase.py, company_mode.py)
- crash exits now settle: cancellation unwind harvests coroutines that
  died on a real exception before discarding them, the crashed-item
  handler releases the claim and settles as crashed with a quarantine
  fallback, and the timeout path settles as crashed (company_mode.py)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
LZH-YS1998
2026-07-21 11:18:12 +08:00
parent eb934cf2bd
commit 9aed328d02
6 changed files with 968 additions and 26 deletions
+133 -18
View File
@@ -34,6 +34,7 @@ from opc.layer2_organization.phase import (
DONE_PHASES,
InvalidPhaseTransition,
coerce_phase,
has_open_attempt,
phase_for_task_status,
task_status_for_phase,
validate_transition,
@@ -43,6 +44,54 @@ from opc.layer2_organization.work_item_identity import work_item_identity_payloa
from opc.layer2_organization.work_item_runtime import is_work_item_runtime_metadata
def build_attempt_settlement_updates(
current_metadata: dict[str, Any] | None,
*,
outcome: str,
) -> dict[str, Any]:
"""Compute the metadata delta that settles the currently-open attempt.
``outcome`` semantics:
* ``"crashed"`` — the attempt died on an exception/timeout; increments
``attempt_crash_streak`` (consecutive crashes).
* ``"interrupted"`` — the owning process/coroutine went away without a
verdict (kill, cancel-before-harvest); increments
``attempt_interrupted_streak``.
* anything else — a clean turn boundary (approved/failed-by-review/
waiting/park/...); resets both streaks.
Returns ``{}`` when there is no open attempt to settle, so callers can
merge unconditionally.
"""
metadata = dict(current_metadata or {})
if not has_open_attempt(metadata):
return {}
def _streak(key: str) -> int:
try:
return int(metadata.get(key, 0) or 0)
except (TypeError, ValueError):
return 0
outcome_clean = str(outcome or "").strip() or "settled"
if outcome_clean == "crashed":
crash_streak = _streak("attempt_crash_streak") + 1
interrupted_streak = _streak("attempt_interrupted_streak")
elif outcome_clean == "interrupted":
crash_streak = _streak("attempt_crash_streak")
interrupted_streak = _streak("attempt_interrupted_streak") + 1
else:
crash_streak = 0
interrupted_streak = 0
return {
"attempt_settled": True,
"attempt_outcome": outcome_clean,
"attempt_settled_at": datetime.now().isoformat(),
"attempt_crash_streak": crash_streak,
"attempt_interrupted_streak": interrupted_streak,
}
async def transition_work_item(
store: Any,
work_item_id: str,
@@ -52,6 +101,7 @@ async def transition_work_item(
summary: str | None = None,
metadata_updates: dict[str, Any] | None = None,
release_claim: bool = False,
attempt_outcome: str | None = None,
) -> DelegationWorkItem | None:
"""Transition a work item to ``target_phase``.
@@ -89,41 +139,104 @@ async def transition_work_item(
reason_clean = str(reason or "").strip()
if reason_clean:
merged["last_transition_reason"] = reason_clean
# Attempt settlement: any transition to a non-RUNNING phase is a turn
# boundary for the claim that opened the current attempt (the claim CAS
# stamped attempt_settled=false). Fold the settlement delta into the SAME
# write as the phase change so an attempt can never end without its
# ledger entry — the structural guarantee behind the dispatcher's
# crash/interrupted streak brake (see phase.attempt_ledger_dispatch_
# block_reason).
settlement: dict[str, Any] = {}
current_item = None
if phase != Phase.RUNNING and hasattr(store, "get_delegation_work_item"):
try:
current_item = await store.get_delegation_work_item(work_item_id)
except Exception:
current_item = None
if current_item is not None:
settlement = build_attempt_settlement_updates(
dict(getattr(current_item, "metadata", {}) or {}),
outcome=attempt_outcome or phase.value,
)
if settlement:
merged = {**settlement, **merged}
kwargs: dict[str, Any] = {
"phase": phase,
"metadata_updates": merged,
}
if summary is not None:
kwargs["summary"] = summary
# Phase transition first, claim release second (when requested). The
# old rationale for this ordering was "sync_member_session_hook reads
# item.claimed_by_role_runtime_session_id"; Phase B removed that hook
# and moved the unpark to the dispatcher's per-tick rehydrate pass,
# but the two-step ordering is kept so downstream listeners that
# still inspect the claim (audit logs, kanban projections) see a
# consistent before/after.
if release_claim:
# Fold claim release into the same write as the phase change: the
# legacy two-call sequence could commit the phase and then fail the
# release, stranding a dead claim. Terminal phases additionally drop
# the metadata claim mirror keys so the row can never satisfy a
# future claim-CAS predicate by accident.
kwargs["claimed_by_role_runtime_session_id"] = ""
kwargs["claimed_by_seat_id"] = ""
if phase in DONE_PHASES:
merged.setdefault("claimed_by_role_session_id", "")
merged.setdefault("claimed_task_id", "")
try:
result = await store.update_delegation_work_item(work_item_id, **kwargs)
except InvalidPhaseTransition:
# The phase write lost a race (or the caller is out of date). The
# attempt settlement must still land — losing it is exactly the
# "crash without accounting" gap that produced endless re-dispatch.
if settlement:
try:
await store.update_delegation_work_item(
work_item_id,
metadata_updates=settlement,
)
except Exception:
logger.opt(exception=True).warning(
f"transition_work_item: settlement fallback failed wid={work_item_id}"
)
raise
except Exception:
logger.opt(exception=True).warning(
f"transition_work_item failed wid={work_item_id} "
f"target={phase.value} reason={reason_clean}"
)
raise
if release_claim and result is not None:
try:
result = await store.update_delegation_work_item(
work_item_id,
claimed_by_role_runtime_session_id="",
claimed_by_seat_id="",
)
except Exception:
logger.opt(exception=True).warning(
f"transition_work_item: claim release failed wid={work_item_id}"
)
return result
async def settle_open_attempt_as_interrupted(
store: Any,
work_item: DelegationWorkItem,
) -> bool:
"""Back-fill the ledger for an attempt whose owner died without settling.
Metadata-only write (no phase change): the card keeps whatever phase the
crash left it in — recovery semantics stay untouched — but the attempt
stops being invisible. Called by the dispatcher's per-tick reconcile pass
for cards with an open attempt and no live in-process owner. Returns True
when a settlement write was issued.
"""
if store is None or not hasattr(store, "update_delegation_work_item"):
return False
metadata = dict(getattr(work_item, "metadata", {}) or {})
settlement = build_attempt_settlement_updates(metadata, outcome="interrupted")
if not settlement:
return False
try:
updated = await store.update_delegation_work_item(
work_item.work_item_id,
metadata_updates=settlement,
)
except Exception:
logger.opt(exception=True).warning(
"settle_open_attempt_as_interrupted failed "
f"wid={getattr(work_item, 'work_item_id', '')}"
)
return False
if updated is not None:
work_item.metadata = dict(updated.metadata or {})
return True
def _fallback_status_for(
target_status_or_phase: TaskStatus | Phase | str,
task: Task,
@@ -162,6 +275,7 @@ async def transition_work_item_from_task(
metadata_updates: dict[str, Any] | None = None,
release_claim: bool = False,
require_work_item: bool = False,
attempt_outcome: str | None = None,
) -> bool:
"""Task bridge helper: transition a work item when the caller holds a Task.
@@ -306,6 +420,7 @@ async def transition_work_item_from_task(
summary=summary,
metadata_updates=back_ref,
release_claim=release_claim,
attempt_outcome=attempt_outcome,
)
except InvalidPhaseTransition:
# Defensive: state-machine validation at the store layer can also