fix(company): add dispatch attempt ledger to brake infinite re-dispatch loops (#10)

Work items whose execution kept dying without a durable verdict (crash
mid-dispatch, kill -9, cancel-before-harvest) were re-dispatched forever:
every exit path was responsible for remembering to write a terminal
phase, and any path that forgot left the card RUNNING and eligible again.

Replace that with structural accounting:

- claim CAS opens an attempt in the same UPDATE (attempt_seq+1,
  attempt_settled=false) so no dispatch can start unaccounted (store.py)
- transition_work_item becomes the settlement authority: every
  non-RUNNING transition settles the open attempt in the same write;
  crashed/interrupted outcomes accumulate streaks, clean outcomes reset
  them; claim release folds into the same write; settlement still lands
  when the phase write loses a race (work_item_transition.py)
- dispatcher refuses cards over the streak limits (crash>=3,
  interrupted>=5) in both is_dispatchable and _work_item_is_runnable,
  and a per-tick reconcile pass back-fills dead attempts as interrupted
  and terminalizes over-limit cards to FAILED with a visible
  blocked_reason (dispatch_hold quarantine if even that write fails)
  (phase.py, company_mode.py)
- crash exits now settle: cancellation unwind harvests coroutines that
  died on a real exception before discarding them, the crashed-item
  handler releases the claim and settles as crashed with a quarantine
  fallback, and the timeout path settles as crashed (company_mode.py)

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
LZH-YS1998
2026-07-21 11:18:12 +08:00
parent eb934cf2bd
commit 9aed328d02
6 changed files with 968 additions and 26 deletions
+9 -1
View File
@@ -5497,7 +5497,14 @@ class OPCStore:
COALESCE(NULLIF(metadata, ''), '{}'),
'$.claimed_by_role_session_id', ?,
'$.claimed_task_id', ?,
'$.claimed_work_item_revision', ?
'$.claimed_work_item_revision', ?,
'$.attempt_seq', COALESCE(
CAST(json_extract(metadata, '$.attempt_seq') AS INTEGER),
0
) + 1,
'$.attempt_settled', json('false'),
'$.attempt_outcome', '',
'$.attempt_started_at', ?
),
updated_at = ?
WHERE work_item_id = ?
@@ -5521,6 +5528,7 @@ class OPCStore:
claimed_task_id,
int(work_item_revision or 0),
updated_at.isoformat(),
updated_at.isoformat(),
str(work_item_id or "").strip(),
phase.value,
int(work_item_revision or 0),
+207 -7
View File
@@ -51,6 +51,9 @@ from opc.layer2_organization.phase import (
IN_PROGRESS_PHASES,
IN_REVIEW_PHASES,
TODO_PHASES,
InvalidPhaseTransition,
attempt_ledger_dispatch_block_reason,
has_open_attempt,
is_dispatchable,
is_orphaned,
is_report_execution_work_item_metadata,
@@ -111,7 +114,9 @@ from opc.layer2_organization.work_item_transition import (
has_pending_settlement_release,
normalize_dependency_work_item_ids,
refresh_dependents_for_run,
settle_open_attempt_as_interrupted,
settled_failure_dependency_ids,
transition_work_item,
transition_work_item_from_task,
)
from opc.layer2_organization.work_item_identity import (
@@ -2671,6 +2676,13 @@ class CompanyWorkItemExecutor:
report_execution_work_item = is_report_execution_work_item_metadata(metadata)
if str(metadata.get("dispatch_hold", "") or "").strip():
return False
# Attempt-ledger brake: cards whose recent attempts keep crashing or
# keep getting interrupted must not be re-enqueued — this check also
# covers the review/report `or phase == RUNNING` arms below that
# bypass is_dispatchable. The per-tick ledger reconcile pass
# terminalizes such cards with a visible blocked_reason.
if attempt_ledger_dispatch_block_reason(metadata):
return False
# Hidden auxiliary cards (review / report) are still runnable —
# they are the kanban-push primitives the dispatcher schedules.
# Worker work items marked hidden for any other reason stay
@@ -2828,6 +2840,124 @@ class CompanyWorkItemExecutor:
return True
return True
async def _reconcile_attempt_ledger(
self,
work_items: list[DelegationWorkItem],
active_work_item_tasks: dict[
"asyncio.Task[TaskResult | None]",
tuple["CompanyMemberSession", Task],
],
) -> list[DelegationWorkItem]:
"""Per-tick attempt-ledger reconcile: the structural anti-loop pass.
Two responsibilities, both driven by durable state instead of any
code path "remembering" to behave:
1. **Back-fill dead attempts.** A card whose claim opened an attempt
(``attempt_settled == False``) but has no live owner in THIS
dispatcher (not in ``active_work_item_tasks`` / claimed sets) had
its coroutine or process die without a verdict kill -9, cancel
before harvest, or a settlement write that never landed. Settle it
as ``interrupted`` so the ledger sees it.
2. **Terminalize over-limit cards.** When the ledger shows a run of
consecutive crashed/interrupted attempts over the limit, transition
the card to FAILED with a visible ``blocked_reason`` (quarantine
via ``dispatch_hold`` if even that write fails). Eligibility checks
(``is_dispatchable`` / ``_work_item_is_runnable``) independently
refuse such cards, so either write path converges the loop.
"""
if not work_items or self.store is None:
return work_items
in_flight_ids: set[str] = {
linked_work_item_id_for_task(claimed_task)
for _member, claimed_task in active_work_item_tasks.values()
}
in_flight_ids.discard("")
in_flight_ids.update(
str(item) for item in getattr(self.runtime, "_claimed_work_item_ids", set()) or set()
)
reconciled: list[DelegationWorkItem] = []
for work_item in work_items:
phase = getattr(work_item, "phase", None)
work_item_id = str(getattr(work_item, "work_item_id", "") or "").strip()
if not work_item_id or phase in DONE_PHASES:
reconciled.append(work_item)
continue
metadata = dict(getattr(work_item, "metadata", {}) or {})
if has_open_attempt(metadata) and work_item_id not in in_flight_ids:
if await settle_open_attempt_as_interrupted(self.store, work_item):
metadata = dict(getattr(work_item, "metadata", {}) or {})
logger.info(
"[attempt_ledger] settled dead attempt as interrupted "
"work_item={} phase={} interrupted_streak={}",
work_item_id,
getattr(phase, "value", phase),
metadata.get("attempt_interrupted_streak"),
)
block_reason = attempt_ledger_dispatch_block_reason(metadata)
if not block_reason:
reconciled.append(work_item)
continue
summary_text = (
f"Work item quarantined by the attempt ledger: {block_reason}. "
"Its recent execution attempts kept dying without a durable verdict; "
"manual triage (rework or new card) is required."
)
try:
updated = await transition_work_item(
self.store,
work_item_id,
target_phase=Phase.FAILED,
reason="attempt_ledger_limit",
summary=summary_text,
metadata_updates={"attempt_ledger_block_reason": block_reason},
release_claim=True,
)
if updated is not None:
work_item = updated
try:
await self.store.update_delegation_work_item(
work_item_id,
blocked_reason=block_reason,
)
except Exception:
logger.opt(exception=True).debug(
"[attempt_ledger] blocked_reason write failed for {}", work_item_id
)
await self._emit_progress(
f"[Company:{projection_id_for_work_item(work_item)}] {summary_text}"
)
except InvalidPhaseTransition:
# A concurrent writer moved the card; re-read and keep going —
# eligibility checks still refuse it while the ledger is over
# the limit.
logger.opt(exception=True).debug(
"[attempt_ledger] FAILED terminalize lost a phase race for {}",
work_item_id,
)
except Exception:
logger.opt(exception=True).warning(
"[attempt_ledger] FAILED terminalize did not land for {}; "
"quarantining via dispatch_hold",
work_item_id,
)
try:
await self.store.update_delegation_work_item(
work_item_id,
blocked_reason=block_reason,
metadata_updates={
"dispatch_hold": "attempt_ledger_quarantine",
"attempt_ledger_block_reason": block_reason,
},
)
except Exception:
logger.opt(exception=True).error(
"[attempt_ledger] quarantine hold write also failed for {}",
work_item_id,
)
reconciled.append(work_item)
return reconciled
def _task_effective_projection_spec(self, task: Task) -> WorkItemProjectionSpec:
projection = self._projection_spec_for_task(task)
if projection is not None:
@@ -4528,6 +4658,10 @@ class CompanyWorkItemExecutor:
await self._try_unpark_blocking_comms(parked)
await self.runtime.refresh_inbox_state(tasks)
work_items = await self._load_delegation_work_items(tasks)
work_items = await self._reconcile_attempt_ledger(
work_items,
active_work_item_tasks,
)
work_items = await self._refresh_ready_work_items(work_items, tasks=tasks)
tasks = await self._materialize_work_item_tasks(tasks, work_items)
self._active_tasks = tasks
@@ -4690,6 +4824,42 @@ class CompanyWorkItemExecutor:
self._schedule_kanban_notification()
except asyncio.CancelledError:
claimed_pairs = list(active_work_item_tasks.values())
# Coroutines that already died on a real exception must settle
# their attempt BEFORE this turn unwinds. The old gather(...,
# return_exceptions=True) below silently discarded them, leaving
# the card RUNNING with an open attempt — the exact gap that let
# a deterministic crash replay forever across suspend/resume
# cycles (issue #10). Cancellation of live coroutines is still a
# legitimate suspend; only genuine crashes are harvested here.
crashed_pairs: list[tuple[CompanyMemberSession, Task, BaseException]] = []
for work_item_task, session_task in list(active_work_item_tasks.items()):
if not work_item_task.done() or work_item_task.cancelled():
continue
crash_exc = work_item_task.exception()
if crash_exc is None or isinstance(crash_exc, asyncio.CancelledError):
continue
crashed_member_session, crashed_task = session_task
crashed_pairs.append((crashed_member_session, crashed_task, crash_exc))
for crashed_member_session, crashed_task, crash_exc in crashed_pairs:
try:
await asyncio.shield(
self._handle_claimed_work_item_exception(
crashed_member_session,
crashed_task,
crash_exc,
)
)
except asyncio.CancelledError:
logger.warning(
"company runtime cancellation: crashed work item settlement "
"for task={} continues shielded in background",
crashed_task.id,
)
except Exception:
logger.opt(exception=True).error(
"company runtime cancellation: failed to settle crashed work item task={}",
crashed_task.id,
)
for work_item_task in list(active_work_item_tasks.keys()):
if not work_item_task.done():
work_item_task.cancel()
@@ -4957,13 +5127,42 @@ class CompanyWorkItemExecutor:
"artifacts": dict(failure_result.artifacts or {}),
}
# Phase A: phase write first → hook projects task.status=FAILED
# onto the DB row and syncs our local task.status too.
await transition_work_item_from_task(
self.store, task,
target_status_or_phase=Phase.FAILED,
reason="claimed_work_item_exception",
summary=summary or None,
)
# onto the DB row and syncs our local task.status too. The
# attempt_outcome="crashed" settlement rides the same write so
# the dispatcher's crash-streak brake has durable accounting.
# If even this write fails, quarantine the card via a plain
# metadata hold — dispatch_hold blocks is_dispatchable without
# needing phase-machine legality — so a store hiccup can never
# convert a crash into an endless re-dispatch loop.
try:
await transition_work_item_from_task(
self.store, task,
target_status_or_phase=Phase.FAILED,
reason="claimed_work_item_exception",
summary=summary or None,
release_claim=True,
attempt_outcome="crashed",
)
except Exception as transition_exc:
logger.opt(exception=transition_exc).error(
f"[Company:{projection_id}] FAILED transition for crashed work item "
"did not land; quarantining via dispatch_hold"
)
if work_item_id and self.store is not None and hasattr(self.store, "update_delegation_work_item"):
try:
await self.store.update_delegation_work_item(
work_item_id,
blocked_reason=summary,
metadata_updates={
"dispatch_hold": "crash_quarantine",
"crash_quarantine_at": datetime.now().isoformat(),
"crash_quarantine_error": str(exc),
},
)
except Exception:
logger.opt(exception=True).error(
f"[Company:{projection_id}] crash quarantine write also failed"
)
try:
await self.runtime.complete_claim(member_session, task, result=failure_result)
except Exception as cleanup_exc:
@@ -5428,6 +5627,7 @@ class CompanyWorkItemExecutor:
self.store, task,
target_status_or_phase=Phase.FAILED,
reason="work_item_timeout",
attempt_outcome="crashed",
)
await self.save_task(task)
return TaskResult(status=TaskStatus.FAILED, content=f"Work item timed out after {self.work_item_timeout}s.")
@@ -187,6 +187,17 @@ _WORK_ITEM_FIELDS: tuple[MetadataFieldSpec, ...] = (
_spec("self_evolution_patch", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
_spec("self_evolution_completed_at", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
_spec("self_evolution_error", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
# Attempt ledger (dispatch-attempt accounting; see phase.py). Opened by the
# claim CAS, settled by transition_work_item — the dispatcher's structural
# brake against crash/interrupted re-dispatch loops.
_spec("attempt_seq", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
_spec("attempt_settled", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
_spec("attempt_outcome", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
_spec("attempt_started_at", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
_spec("attempt_settled_at", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
_spec("attempt_crash_streak", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
_spec("attempt_interrupted_streak", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
_spec("attempt_ledger_block_reason", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
)
_RUNTIME_TASK_FIELDS: tuple[MetadataFieldSpec, ...] = (
+53
View File
@@ -385,6 +385,57 @@ def is_orphaned(item: Any) -> bool:
return not claim
# ── Attempt ledger ───────────────────────────────────────────────────────
#
# Every durable claim opens an "attempt" on the work item (stamped by the
# claim CAS in store.claim_delegation_work_item_if_dispatchable) and every
# turn-boundary transition through work_item_transition.transition_work_item
# settles it with an outcome. The dispatcher refuses to start attempt N+1
# for cards whose ledger shows a run of consecutive crashed / interrupted
# attempts — this is the structural brake that makes "a crash path forgot
# (or failed) to write FAILED" converge instead of re-dispatching forever
# (issue #10: restart → resume → deterministic crash → re-dispatch loop).
ATTEMPT_CRASH_STREAK_LIMIT = 3
ATTEMPT_INTERRUPTED_STREAK_LIMIT = 5
def _attempt_ledger_int(metadata: Mapping[str, Any], key: str) -> int:
try:
return int(metadata.get(key, 0) or 0)
except (TypeError, ValueError):
return 0
def attempt_ledger_dispatch_block_reason(metadata: Mapping[str, Any] | None) -> str:
"""Non-empty human-readable reason when the attempt ledger forbids
dispatching this card again; empty string when dispatch is allowed."""
if not isinstance(metadata, Mapping):
return ""
crash_streak = _attempt_ledger_int(metadata, "attempt_crash_streak")
if crash_streak >= ATTEMPT_CRASH_STREAK_LIMIT:
return (
f"attempt ledger: {crash_streak} consecutive crashed attempts "
f"(limit {ATTEMPT_CRASH_STREAK_LIMIT})"
)
interrupted_streak = _attempt_ledger_int(metadata, "attempt_interrupted_streak")
if interrupted_streak >= ATTEMPT_INTERRUPTED_STREAK_LIMIT:
return (
f"attempt ledger: {interrupted_streak} consecutive interrupted attempts "
f"(limit {ATTEMPT_INTERRUPTED_STREAK_LIMIT})"
)
return ""
def has_open_attempt(metadata: Mapping[str, Any] | None) -> bool:
"""True when a claim opened an attempt that was never settled."""
if not isinstance(metadata, Mapping):
return False
if _attempt_ledger_int(metadata, "attempt_seq") <= 0:
return False
return not bool(metadata.get("attempt_settled", True))
def is_dispatchable(item: Any) -> bool:
"""Combined check used by the dispatcher: pick this card on next tick?
@@ -405,6 +456,8 @@ def is_dispatchable(item: Any) -> bool:
return False
if isinstance(metadata, dict) and str(metadata.get("dispatch_hold", "") or "").strip():
return False
if attempt_ledger_dispatch_block_reason(metadata):
return False
return is_runnable(item.phase) or is_orphaned(item)
+133 -18
View File
@@ -34,6 +34,7 @@ from opc.layer2_organization.phase import (
DONE_PHASES,
InvalidPhaseTransition,
coerce_phase,
has_open_attempt,
phase_for_task_status,
task_status_for_phase,
validate_transition,
@@ -43,6 +44,54 @@ from opc.layer2_organization.work_item_identity import work_item_identity_payloa
from opc.layer2_organization.work_item_runtime import is_work_item_runtime_metadata
def build_attempt_settlement_updates(
current_metadata: dict[str, Any] | None,
*,
outcome: str,
) -> dict[str, Any]:
"""Compute the metadata delta that settles the currently-open attempt.
``outcome`` semantics:
* ``"crashed"`` — the attempt died on an exception/timeout; increments
``attempt_crash_streak`` (consecutive crashes).
* ``"interrupted"`` — the owning process/coroutine went away without a
verdict (kill, cancel-before-harvest); increments
``attempt_interrupted_streak``.
* anything else — a clean turn boundary (approved/failed-by-review/
waiting/park/...); resets both streaks.
Returns ``{}`` when there is no open attempt to settle, so callers can
merge unconditionally.
"""
metadata = dict(current_metadata or {})
if not has_open_attempt(metadata):
return {}
def _streak(key: str) -> int:
try:
return int(metadata.get(key, 0) or 0)
except (TypeError, ValueError):
return 0
outcome_clean = str(outcome or "").strip() or "settled"
if outcome_clean == "crashed":
crash_streak = _streak("attempt_crash_streak") + 1
interrupted_streak = _streak("attempt_interrupted_streak")
elif outcome_clean == "interrupted":
crash_streak = _streak("attempt_crash_streak")
interrupted_streak = _streak("attempt_interrupted_streak") + 1
else:
crash_streak = 0
interrupted_streak = 0
return {
"attempt_settled": True,
"attempt_outcome": outcome_clean,
"attempt_settled_at": datetime.now().isoformat(),
"attempt_crash_streak": crash_streak,
"attempt_interrupted_streak": interrupted_streak,
}
async def transition_work_item(
store: Any,
work_item_id: str,
@@ -52,6 +101,7 @@ async def transition_work_item(
summary: str | None = None,
metadata_updates: dict[str, Any] | None = None,
release_claim: bool = False,
attempt_outcome: str | None = None,
) -> DelegationWorkItem | None:
"""Transition a work item to ``target_phase``.
@@ -89,41 +139,104 @@ async def transition_work_item(
reason_clean = str(reason or "").strip()
if reason_clean:
merged["last_transition_reason"] = reason_clean
# Attempt settlement: any transition to a non-RUNNING phase is a turn
# boundary for the claim that opened the current attempt (the claim CAS
# stamped attempt_settled=false). Fold the settlement delta into the SAME
# write as the phase change so an attempt can never end without its
# ledger entry — the structural guarantee behind the dispatcher's
# crash/interrupted streak brake (see phase.attempt_ledger_dispatch_
# block_reason).
settlement: dict[str, Any] = {}
current_item = None
if phase != Phase.RUNNING and hasattr(store, "get_delegation_work_item"):
try:
current_item = await store.get_delegation_work_item(work_item_id)
except Exception:
current_item = None
if current_item is not None:
settlement = build_attempt_settlement_updates(
dict(getattr(current_item, "metadata", {}) or {}),
outcome=attempt_outcome or phase.value,
)
if settlement:
merged = {**settlement, **merged}
kwargs: dict[str, Any] = {
"phase": phase,
"metadata_updates": merged,
}
if summary is not None:
kwargs["summary"] = summary
# Phase transition first, claim release second (when requested). The
# old rationale for this ordering was "sync_member_session_hook reads
# item.claimed_by_role_runtime_session_id"; Phase B removed that hook
# and moved the unpark to the dispatcher's per-tick rehydrate pass,
# but the two-step ordering is kept so downstream listeners that
# still inspect the claim (audit logs, kanban projections) see a
# consistent before/after.
if release_claim:
# Fold claim release into the same write as the phase change: the
# legacy two-call sequence could commit the phase and then fail the
# release, stranding a dead claim. Terminal phases additionally drop
# the metadata claim mirror keys so the row can never satisfy a
# future claim-CAS predicate by accident.
kwargs["claimed_by_role_runtime_session_id"] = ""
kwargs["claimed_by_seat_id"] = ""
if phase in DONE_PHASES:
merged.setdefault("claimed_by_role_session_id", "")
merged.setdefault("claimed_task_id", "")
try:
result = await store.update_delegation_work_item(work_item_id, **kwargs)
except InvalidPhaseTransition:
# The phase write lost a race (or the caller is out of date). The
# attempt settlement must still land — losing it is exactly the
# "crash without accounting" gap that produced endless re-dispatch.
if settlement:
try:
await store.update_delegation_work_item(
work_item_id,
metadata_updates=settlement,
)
except Exception:
logger.opt(exception=True).warning(
f"transition_work_item: settlement fallback failed wid={work_item_id}"
)
raise
except Exception:
logger.opt(exception=True).warning(
f"transition_work_item failed wid={work_item_id} "
f"target={phase.value} reason={reason_clean}"
)
raise
if release_claim and result is not None:
try:
result = await store.update_delegation_work_item(
work_item_id,
claimed_by_role_runtime_session_id="",
claimed_by_seat_id="",
)
except Exception:
logger.opt(exception=True).warning(
f"transition_work_item: claim release failed wid={work_item_id}"
)
return result
async def settle_open_attempt_as_interrupted(
store: Any,
work_item: DelegationWorkItem,
) -> bool:
"""Back-fill the ledger for an attempt whose owner died without settling.
Metadata-only write (no phase change): the card keeps whatever phase the
crash left it in — recovery semantics stay untouched — but the attempt
stops being invisible. Called by the dispatcher's per-tick reconcile pass
for cards with an open attempt and no live in-process owner. Returns True
when a settlement write was issued.
"""
if store is None or not hasattr(store, "update_delegation_work_item"):
return False
metadata = dict(getattr(work_item, "metadata", {}) or {})
settlement = build_attempt_settlement_updates(metadata, outcome="interrupted")
if not settlement:
return False
try:
updated = await store.update_delegation_work_item(
work_item.work_item_id,
metadata_updates=settlement,
)
except Exception:
logger.opt(exception=True).warning(
"settle_open_attempt_as_interrupted failed "
f"wid={getattr(work_item, 'work_item_id', '')}"
)
return False
if updated is not None:
work_item.metadata = dict(updated.metadata or {})
return True
def _fallback_status_for(
target_status_or_phase: TaskStatus | Phase | str,
task: Task,
@@ -162,6 +275,7 @@ async def transition_work_item_from_task(
metadata_updates: dict[str, Any] | None = None,
release_claim: bool = False,
require_work_item: bool = False,
attempt_outcome: str | None = None,
) -> bool:
"""Task bridge helper: transition a work item when the caller holds a Task.
@@ -306,6 +420,7 @@ async def transition_work_item_from_task(
summary=summary,
metadata_updates=back_ref,
release_claim=release_claim,
attempt_outcome=attempt_outcome,
)
except InvalidPhaseTransition:
# Defensive: state-machine validation at the store layer can also