fix(company): add dispatch attempt ledger to brake infinite re-dispatch loops (#10)
Work items whose execution kept dying without a durable verdict (crash mid-dispatch, kill -9, cancel-before-harvest) were re-dispatched forever: every exit path was responsible for remembering to write a terminal phase, and any path that forgot left the card RUNNING and eligible again. Replace that with structural accounting: - claim CAS opens an attempt in the same UPDATE (attempt_seq+1, attempt_settled=false) so no dispatch can start unaccounted (store.py) - transition_work_item becomes the settlement authority: every non-RUNNING transition settles the open attempt in the same write; crashed/interrupted outcomes accumulate streaks, clean outcomes reset them; claim release folds into the same write; settlement still lands when the phase write loses a race (work_item_transition.py) - dispatcher refuses cards over the streak limits (crash>=3, interrupted>=5) in both is_dispatchable and _work_item_is_runnable, and a per-tick reconcile pass back-fills dead attempts as interrupted and terminalizes over-limit cards to FAILED with a visible blocked_reason (dispatch_hold quarantine if even that write fails) (phase.py, company_mode.py) - crash exits now settle: cancellation unwind harvests coroutines that died on a real exception before discarding them, the crashed-item handler releases the claim and settles as crashed with a quarantine fallback, and the timeout path settles as crashed (company_mode.py) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -5497,7 +5497,14 @@ class OPCStore:
|
||||
COALESCE(NULLIF(metadata, ''), '{}'),
|
||||
'$.claimed_by_role_session_id', ?,
|
||||
'$.claimed_task_id', ?,
|
||||
'$.claimed_work_item_revision', ?
|
||||
'$.claimed_work_item_revision', ?,
|
||||
'$.attempt_seq', COALESCE(
|
||||
CAST(json_extract(metadata, '$.attempt_seq') AS INTEGER),
|
||||
0
|
||||
) + 1,
|
||||
'$.attempt_settled', json('false'),
|
||||
'$.attempt_outcome', '',
|
||||
'$.attempt_started_at', ?
|
||||
),
|
||||
updated_at = ?
|
||||
WHERE work_item_id = ?
|
||||
@@ -5521,6 +5528,7 @@ class OPCStore:
|
||||
claimed_task_id,
|
||||
int(work_item_revision or 0),
|
||||
updated_at.isoformat(),
|
||||
updated_at.isoformat(),
|
||||
str(work_item_id or "").strip(),
|
||||
phase.value,
|
||||
int(work_item_revision or 0),
|
||||
|
||||
@@ -51,6 +51,9 @@ from opc.layer2_organization.phase import (
|
||||
IN_PROGRESS_PHASES,
|
||||
IN_REVIEW_PHASES,
|
||||
TODO_PHASES,
|
||||
InvalidPhaseTransition,
|
||||
attempt_ledger_dispatch_block_reason,
|
||||
has_open_attempt,
|
||||
is_dispatchable,
|
||||
is_orphaned,
|
||||
is_report_execution_work_item_metadata,
|
||||
@@ -111,7 +114,9 @@ from opc.layer2_organization.work_item_transition import (
|
||||
has_pending_settlement_release,
|
||||
normalize_dependency_work_item_ids,
|
||||
refresh_dependents_for_run,
|
||||
settle_open_attempt_as_interrupted,
|
||||
settled_failure_dependency_ids,
|
||||
transition_work_item,
|
||||
transition_work_item_from_task,
|
||||
)
|
||||
from opc.layer2_organization.work_item_identity import (
|
||||
@@ -2671,6 +2676,13 @@ class CompanyWorkItemExecutor:
|
||||
report_execution_work_item = is_report_execution_work_item_metadata(metadata)
|
||||
if str(metadata.get("dispatch_hold", "") or "").strip():
|
||||
return False
|
||||
# Attempt-ledger brake: cards whose recent attempts keep crashing or
|
||||
# keep getting interrupted must not be re-enqueued — this check also
|
||||
# covers the review/report `or phase == RUNNING` arms below that
|
||||
# bypass is_dispatchable. The per-tick ledger reconcile pass
|
||||
# terminalizes such cards with a visible blocked_reason.
|
||||
if attempt_ledger_dispatch_block_reason(metadata):
|
||||
return False
|
||||
# Hidden auxiliary cards (review / report) are still runnable —
|
||||
# they are the kanban-push primitives the dispatcher schedules.
|
||||
# Worker work items marked hidden for any other reason stay
|
||||
@@ -2828,6 +2840,124 @@ class CompanyWorkItemExecutor:
|
||||
return True
|
||||
return True
|
||||
|
||||
async def _reconcile_attempt_ledger(
|
||||
self,
|
||||
work_items: list[DelegationWorkItem],
|
||||
active_work_item_tasks: dict[
|
||||
"asyncio.Task[TaskResult | None]",
|
||||
tuple["CompanyMemberSession", Task],
|
||||
],
|
||||
) -> list[DelegationWorkItem]:
|
||||
"""Per-tick attempt-ledger reconcile: the structural anti-loop pass.
|
||||
|
||||
Two responsibilities, both driven by durable state instead of any
|
||||
code path "remembering" to behave:
|
||||
|
||||
1. **Back-fill dead attempts.** A card whose claim opened an attempt
|
||||
(``attempt_settled == False``) but has no live owner in THIS
|
||||
dispatcher (not in ``active_work_item_tasks`` / claimed sets) had
|
||||
its coroutine or process die without a verdict — kill -9, cancel
|
||||
before harvest, or a settlement write that never landed. Settle it
|
||||
as ``interrupted`` so the ledger sees it.
|
||||
2. **Terminalize over-limit cards.** When the ledger shows a run of
|
||||
consecutive crashed/interrupted attempts over the limit, transition
|
||||
the card to FAILED with a visible ``blocked_reason`` (quarantine
|
||||
via ``dispatch_hold`` if even that write fails). Eligibility checks
|
||||
(``is_dispatchable`` / ``_work_item_is_runnable``) independently
|
||||
refuse such cards, so either write path converges the loop.
|
||||
"""
|
||||
if not work_items or self.store is None:
|
||||
return work_items
|
||||
in_flight_ids: set[str] = {
|
||||
linked_work_item_id_for_task(claimed_task)
|
||||
for _member, claimed_task in active_work_item_tasks.values()
|
||||
}
|
||||
in_flight_ids.discard("")
|
||||
in_flight_ids.update(
|
||||
str(item) for item in getattr(self.runtime, "_claimed_work_item_ids", set()) or set()
|
||||
)
|
||||
reconciled: list[DelegationWorkItem] = []
|
||||
for work_item in work_items:
|
||||
phase = getattr(work_item, "phase", None)
|
||||
work_item_id = str(getattr(work_item, "work_item_id", "") or "").strip()
|
||||
if not work_item_id or phase in DONE_PHASES:
|
||||
reconciled.append(work_item)
|
||||
continue
|
||||
metadata = dict(getattr(work_item, "metadata", {}) or {})
|
||||
if has_open_attempt(metadata) and work_item_id not in in_flight_ids:
|
||||
if await settle_open_attempt_as_interrupted(self.store, work_item):
|
||||
metadata = dict(getattr(work_item, "metadata", {}) or {})
|
||||
logger.info(
|
||||
"[attempt_ledger] settled dead attempt as interrupted "
|
||||
"work_item={} phase={} interrupted_streak={}",
|
||||
work_item_id,
|
||||
getattr(phase, "value", phase),
|
||||
metadata.get("attempt_interrupted_streak"),
|
||||
)
|
||||
block_reason = attempt_ledger_dispatch_block_reason(metadata)
|
||||
if not block_reason:
|
||||
reconciled.append(work_item)
|
||||
continue
|
||||
summary_text = (
|
||||
f"Work item quarantined by the attempt ledger: {block_reason}. "
|
||||
"Its recent execution attempts kept dying without a durable verdict; "
|
||||
"manual triage (rework or new card) is required."
|
||||
)
|
||||
try:
|
||||
updated = await transition_work_item(
|
||||
self.store,
|
||||
work_item_id,
|
||||
target_phase=Phase.FAILED,
|
||||
reason="attempt_ledger_limit",
|
||||
summary=summary_text,
|
||||
metadata_updates={"attempt_ledger_block_reason": block_reason},
|
||||
release_claim=True,
|
||||
)
|
||||
if updated is not None:
|
||||
work_item = updated
|
||||
try:
|
||||
await self.store.update_delegation_work_item(
|
||||
work_item_id,
|
||||
blocked_reason=block_reason,
|
||||
)
|
||||
except Exception:
|
||||
logger.opt(exception=True).debug(
|
||||
"[attempt_ledger] blocked_reason write failed for {}", work_item_id
|
||||
)
|
||||
await self._emit_progress(
|
||||
f"[Company:{projection_id_for_work_item(work_item)}] {summary_text}"
|
||||
)
|
||||
except InvalidPhaseTransition:
|
||||
# A concurrent writer moved the card; re-read and keep going —
|
||||
# eligibility checks still refuse it while the ledger is over
|
||||
# the limit.
|
||||
logger.opt(exception=True).debug(
|
||||
"[attempt_ledger] FAILED terminalize lost a phase race for {}",
|
||||
work_item_id,
|
||||
)
|
||||
except Exception:
|
||||
logger.opt(exception=True).warning(
|
||||
"[attempt_ledger] FAILED terminalize did not land for {}; "
|
||||
"quarantining via dispatch_hold",
|
||||
work_item_id,
|
||||
)
|
||||
try:
|
||||
await self.store.update_delegation_work_item(
|
||||
work_item_id,
|
||||
blocked_reason=block_reason,
|
||||
metadata_updates={
|
||||
"dispatch_hold": "attempt_ledger_quarantine",
|
||||
"attempt_ledger_block_reason": block_reason,
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
logger.opt(exception=True).error(
|
||||
"[attempt_ledger] quarantine hold write also failed for {}",
|
||||
work_item_id,
|
||||
)
|
||||
reconciled.append(work_item)
|
||||
return reconciled
|
||||
|
||||
def _task_effective_projection_spec(self, task: Task) -> WorkItemProjectionSpec:
|
||||
projection = self._projection_spec_for_task(task)
|
||||
if projection is not None:
|
||||
@@ -4528,6 +4658,10 @@ class CompanyWorkItemExecutor:
|
||||
await self._try_unpark_blocking_comms(parked)
|
||||
await self.runtime.refresh_inbox_state(tasks)
|
||||
work_items = await self._load_delegation_work_items(tasks)
|
||||
work_items = await self._reconcile_attempt_ledger(
|
||||
work_items,
|
||||
active_work_item_tasks,
|
||||
)
|
||||
work_items = await self._refresh_ready_work_items(work_items, tasks=tasks)
|
||||
tasks = await self._materialize_work_item_tasks(tasks, work_items)
|
||||
self._active_tasks = tasks
|
||||
@@ -4690,6 +4824,42 @@ class CompanyWorkItemExecutor:
|
||||
self._schedule_kanban_notification()
|
||||
except asyncio.CancelledError:
|
||||
claimed_pairs = list(active_work_item_tasks.values())
|
||||
# Coroutines that already died on a real exception must settle
|
||||
# their attempt BEFORE this turn unwinds. The old gather(...,
|
||||
# return_exceptions=True) below silently discarded them, leaving
|
||||
# the card RUNNING with an open attempt — the exact gap that let
|
||||
# a deterministic crash replay forever across suspend/resume
|
||||
# cycles (issue #10). Cancellation of live coroutines is still a
|
||||
# legitimate suspend; only genuine crashes are harvested here.
|
||||
crashed_pairs: list[tuple[CompanyMemberSession, Task, BaseException]] = []
|
||||
for work_item_task, session_task in list(active_work_item_tasks.items()):
|
||||
if not work_item_task.done() or work_item_task.cancelled():
|
||||
continue
|
||||
crash_exc = work_item_task.exception()
|
||||
if crash_exc is None or isinstance(crash_exc, asyncio.CancelledError):
|
||||
continue
|
||||
crashed_member_session, crashed_task = session_task
|
||||
crashed_pairs.append((crashed_member_session, crashed_task, crash_exc))
|
||||
for crashed_member_session, crashed_task, crash_exc in crashed_pairs:
|
||||
try:
|
||||
await asyncio.shield(
|
||||
self._handle_claimed_work_item_exception(
|
||||
crashed_member_session,
|
||||
crashed_task,
|
||||
crash_exc,
|
||||
)
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
logger.warning(
|
||||
"company runtime cancellation: crashed work item settlement "
|
||||
"for task={} continues shielded in background",
|
||||
crashed_task.id,
|
||||
)
|
||||
except Exception:
|
||||
logger.opt(exception=True).error(
|
||||
"company runtime cancellation: failed to settle crashed work item task={}",
|
||||
crashed_task.id,
|
||||
)
|
||||
for work_item_task in list(active_work_item_tasks.keys()):
|
||||
if not work_item_task.done():
|
||||
work_item_task.cancel()
|
||||
@@ -4957,13 +5127,42 @@ class CompanyWorkItemExecutor:
|
||||
"artifacts": dict(failure_result.artifacts or {}),
|
||||
}
|
||||
# Phase A: phase write first → hook projects task.status=FAILED
|
||||
# onto the DB row and syncs our local task.status too.
|
||||
await transition_work_item_from_task(
|
||||
self.store, task,
|
||||
target_status_or_phase=Phase.FAILED,
|
||||
reason="claimed_work_item_exception",
|
||||
summary=summary or None,
|
||||
)
|
||||
# onto the DB row and syncs our local task.status too. The
|
||||
# attempt_outcome="crashed" settlement rides the same write so
|
||||
# the dispatcher's crash-streak brake has durable accounting.
|
||||
# If even this write fails, quarantine the card via a plain
|
||||
# metadata hold — dispatch_hold blocks is_dispatchable without
|
||||
# needing phase-machine legality — so a store hiccup can never
|
||||
# convert a crash into an endless re-dispatch loop.
|
||||
try:
|
||||
await transition_work_item_from_task(
|
||||
self.store, task,
|
||||
target_status_or_phase=Phase.FAILED,
|
||||
reason="claimed_work_item_exception",
|
||||
summary=summary or None,
|
||||
release_claim=True,
|
||||
attempt_outcome="crashed",
|
||||
)
|
||||
except Exception as transition_exc:
|
||||
logger.opt(exception=transition_exc).error(
|
||||
f"[Company:{projection_id}] FAILED transition for crashed work item "
|
||||
"did not land; quarantining via dispatch_hold"
|
||||
)
|
||||
if work_item_id and self.store is not None and hasattr(self.store, "update_delegation_work_item"):
|
||||
try:
|
||||
await self.store.update_delegation_work_item(
|
||||
work_item_id,
|
||||
blocked_reason=summary,
|
||||
metadata_updates={
|
||||
"dispatch_hold": "crash_quarantine",
|
||||
"crash_quarantine_at": datetime.now().isoformat(),
|
||||
"crash_quarantine_error": str(exc),
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
logger.opt(exception=True).error(
|
||||
f"[Company:{projection_id}] crash quarantine write also failed"
|
||||
)
|
||||
try:
|
||||
await self.runtime.complete_claim(member_session, task, result=failure_result)
|
||||
except Exception as cleanup_exc:
|
||||
@@ -5428,6 +5627,7 @@ class CompanyWorkItemExecutor:
|
||||
self.store, task,
|
||||
target_status_or_phase=Phase.FAILED,
|
||||
reason="work_item_timeout",
|
||||
attempt_outcome="crashed",
|
||||
)
|
||||
await self.save_task(task)
|
||||
return TaskResult(status=TaskStatus.FAILED, content=f"Work item timed out after {self.work_item_timeout}s.")
|
||||
|
||||
@@ -187,6 +187,17 @@ _WORK_ITEM_FIELDS: tuple[MetadataFieldSpec, ...] = (
|
||||
_spec("self_evolution_patch", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
|
||||
_spec("self_evolution_completed_at", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
|
||||
_spec("self_evolution_error", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
|
||||
# Attempt ledger (dispatch-attempt accounting; see phase.py). Opened by the
|
||||
# claim CAS, settled by transition_work_item — the dispatcher's structural
|
||||
# brake against crash/interrupted re-dispatch loops.
|
||||
_spec("attempt_seq", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
|
||||
_spec("attempt_settled", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
|
||||
_spec("attempt_outcome", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
|
||||
_spec("attempt_started_at", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
|
||||
_spec("attempt_settled_at", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
|
||||
_spec("attempt_crash_streak", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
|
||||
_spec("attempt_interrupted_streak", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
|
||||
_spec("attempt_ledger_block_reason", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
|
||||
)
|
||||
|
||||
_RUNTIME_TASK_FIELDS: tuple[MetadataFieldSpec, ...] = (
|
||||
|
||||
@@ -385,6 +385,57 @@ def is_orphaned(item: Any) -> bool:
|
||||
return not claim
|
||||
|
||||
|
||||
# ── Attempt ledger ───────────────────────────────────────────────────────
|
||||
#
|
||||
# Every durable claim opens an "attempt" on the work item (stamped by the
|
||||
# claim CAS in store.claim_delegation_work_item_if_dispatchable) and every
|
||||
# turn-boundary transition through work_item_transition.transition_work_item
|
||||
# settles it with an outcome. The dispatcher refuses to start attempt N+1
|
||||
# for cards whose ledger shows a run of consecutive crashed / interrupted
|
||||
# attempts — this is the structural brake that makes "a crash path forgot
|
||||
# (or failed) to write FAILED" converge instead of re-dispatching forever
|
||||
# (issue #10: restart → resume → deterministic crash → re-dispatch loop).
|
||||
|
||||
ATTEMPT_CRASH_STREAK_LIMIT = 3
|
||||
ATTEMPT_INTERRUPTED_STREAK_LIMIT = 5
|
||||
|
||||
|
||||
def _attempt_ledger_int(metadata: Mapping[str, Any], key: str) -> int:
|
||||
try:
|
||||
return int(metadata.get(key, 0) or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def attempt_ledger_dispatch_block_reason(metadata: Mapping[str, Any] | None) -> str:
|
||||
"""Non-empty human-readable reason when the attempt ledger forbids
|
||||
dispatching this card again; empty string when dispatch is allowed."""
|
||||
if not isinstance(metadata, Mapping):
|
||||
return ""
|
||||
crash_streak = _attempt_ledger_int(metadata, "attempt_crash_streak")
|
||||
if crash_streak >= ATTEMPT_CRASH_STREAK_LIMIT:
|
||||
return (
|
||||
f"attempt ledger: {crash_streak} consecutive crashed attempts "
|
||||
f"(limit {ATTEMPT_CRASH_STREAK_LIMIT})"
|
||||
)
|
||||
interrupted_streak = _attempt_ledger_int(metadata, "attempt_interrupted_streak")
|
||||
if interrupted_streak >= ATTEMPT_INTERRUPTED_STREAK_LIMIT:
|
||||
return (
|
||||
f"attempt ledger: {interrupted_streak} consecutive interrupted attempts "
|
||||
f"(limit {ATTEMPT_INTERRUPTED_STREAK_LIMIT})"
|
||||
)
|
||||
return ""
|
||||
|
||||
|
||||
def has_open_attempt(metadata: Mapping[str, Any] | None) -> bool:
|
||||
"""True when a claim opened an attempt that was never settled."""
|
||||
if not isinstance(metadata, Mapping):
|
||||
return False
|
||||
if _attempt_ledger_int(metadata, "attempt_seq") <= 0:
|
||||
return False
|
||||
return not bool(metadata.get("attempt_settled", True))
|
||||
|
||||
|
||||
def is_dispatchable(item: Any) -> bool:
|
||||
"""Combined check used by the dispatcher: pick this card on next tick?
|
||||
|
||||
@@ -405,6 +456,8 @@ def is_dispatchable(item: Any) -> bool:
|
||||
return False
|
||||
if isinstance(metadata, dict) and str(metadata.get("dispatch_hold", "") or "").strip():
|
||||
return False
|
||||
if attempt_ledger_dispatch_block_reason(metadata):
|
||||
return False
|
||||
return is_runnable(item.phase) or is_orphaned(item)
|
||||
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ from opc.layer2_organization.phase import (
|
||||
DONE_PHASES,
|
||||
InvalidPhaseTransition,
|
||||
coerce_phase,
|
||||
has_open_attempt,
|
||||
phase_for_task_status,
|
||||
task_status_for_phase,
|
||||
validate_transition,
|
||||
@@ -43,6 +44,54 @@ from opc.layer2_organization.work_item_identity import work_item_identity_payloa
|
||||
from opc.layer2_organization.work_item_runtime import is_work_item_runtime_metadata
|
||||
|
||||
|
||||
def build_attempt_settlement_updates(
|
||||
current_metadata: dict[str, Any] | None,
|
||||
*,
|
||||
outcome: str,
|
||||
) -> dict[str, Any]:
|
||||
"""Compute the metadata delta that settles the currently-open attempt.
|
||||
|
||||
``outcome`` semantics:
|
||||
* ``"crashed"`` — the attempt died on an exception/timeout; increments
|
||||
``attempt_crash_streak`` (consecutive crashes).
|
||||
* ``"interrupted"`` — the owning process/coroutine went away without a
|
||||
verdict (kill, cancel-before-harvest); increments
|
||||
``attempt_interrupted_streak``.
|
||||
* anything else — a clean turn boundary (approved/failed-by-review/
|
||||
waiting/park/...); resets both streaks.
|
||||
|
||||
Returns ``{}`` when there is no open attempt to settle, so callers can
|
||||
merge unconditionally.
|
||||
"""
|
||||
metadata = dict(current_metadata or {})
|
||||
if not has_open_attempt(metadata):
|
||||
return {}
|
||||
|
||||
def _streak(key: str) -> int:
|
||||
try:
|
||||
return int(metadata.get(key, 0) or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
|
||||
outcome_clean = str(outcome or "").strip() or "settled"
|
||||
if outcome_clean == "crashed":
|
||||
crash_streak = _streak("attempt_crash_streak") + 1
|
||||
interrupted_streak = _streak("attempt_interrupted_streak")
|
||||
elif outcome_clean == "interrupted":
|
||||
crash_streak = _streak("attempt_crash_streak")
|
||||
interrupted_streak = _streak("attempt_interrupted_streak") + 1
|
||||
else:
|
||||
crash_streak = 0
|
||||
interrupted_streak = 0
|
||||
return {
|
||||
"attempt_settled": True,
|
||||
"attempt_outcome": outcome_clean,
|
||||
"attempt_settled_at": datetime.now().isoformat(),
|
||||
"attempt_crash_streak": crash_streak,
|
||||
"attempt_interrupted_streak": interrupted_streak,
|
||||
}
|
||||
|
||||
|
||||
async def transition_work_item(
|
||||
store: Any,
|
||||
work_item_id: str,
|
||||
@@ -52,6 +101,7 @@ async def transition_work_item(
|
||||
summary: str | None = None,
|
||||
metadata_updates: dict[str, Any] | None = None,
|
||||
release_claim: bool = False,
|
||||
attempt_outcome: str | None = None,
|
||||
) -> DelegationWorkItem | None:
|
||||
"""Transition a work item to ``target_phase``.
|
||||
|
||||
@@ -89,41 +139,104 @@ async def transition_work_item(
|
||||
reason_clean = str(reason or "").strip()
|
||||
if reason_clean:
|
||||
merged["last_transition_reason"] = reason_clean
|
||||
# Attempt settlement: any transition to a non-RUNNING phase is a turn
|
||||
# boundary for the claim that opened the current attempt (the claim CAS
|
||||
# stamped attempt_settled=false). Fold the settlement delta into the SAME
|
||||
# write as the phase change so an attempt can never end without its
|
||||
# ledger entry — the structural guarantee behind the dispatcher's
|
||||
# crash/interrupted streak brake (see phase.attempt_ledger_dispatch_
|
||||
# block_reason).
|
||||
settlement: dict[str, Any] = {}
|
||||
current_item = None
|
||||
if phase != Phase.RUNNING and hasattr(store, "get_delegation_work_item"):
|
||||
try:
|
||||
current_item = await store.get_delegation_work_item(work_item_id)
|
||||
except Exception:
|
||||
current_item = None
|
||||
if current_item is not None:
|
||||
settlement = build_attempt_settlement_updates(
|
||||
dict(getattr(current_item, "metadata", {}) or {}),
|
||||
outcome=attempt_outcome or phase.value,
|
||||
)
|
||||
if settlement:
|
||||
merged = {**settlement, **merged}
|
||||
kwargs: dict[str, Any] = {
|
||||
"phase": phase,
|
||||
"metadata_updates": merged,
|
||||
}
|
||||
if summary is not None:
|
||||
kwargs["summary"] = summary
|
||||
# Phase transition first, claim release second (when requested). The
|
||||
# old rationale for this ordering was "sync_member_session_hook reads
|
||||
# item.claimed_by_role_runtime_session_id"; Phase B removed that hook
|
||||
# and moved the unpark to the dispatcher's per-tick rehydrate pass,
|
||||
# but the two-step ordering is kept so downstream listeners that
|
||||
# still inspect the claim (audit logs, kanban projections) see a
|
||||
# consistent before/after.
|
||||
if release_claim:
|
||||
# Fold claim release into the same write as the phase change: the
|
||||
# legacy two-call sequence could commit the phase and then fail the
|
||||
# release, stranding a dead claim. Terminal phases additionally drop
|
||||
# the metadata claim mirror keys so the row can never satisfy a
|
||||
# future claim-CAS predicate by accident.
|
||||
kwargs["claimed_by_role_runtime_session_id"] = ""
|
||||
kwargs["claimed_by_seat_id"] = ""
|
||||
if phase in DONE_PHASES:
|
||||
merged.setdefault("claimed_by_role_session_id", "")
|
||||
merged.setdefault("claimed_task_id", "")
|
||||
try:
|
||||
result = await store.update_delegation_work_item(work_item_id, **kwargs)
|
||||
except InvalidPhaseTransition:
|
||||
# The phase write lost a race (or the caller is out of date). The
|
||||
# attempt settlement must still land — losing it is exactly the
|
||||
# "crash without accounting" gap that produced endless re-dispatch.
|
||||
if settlement:
|
||||
try:
|
||||
await store.update_delegation_work_item(
|
||||
work_item_id,
|
||||
metadata_updates=settlement,
|
||||
)
|
||||
except Exception:
|
||||
logger.opt(exception=True).warning(
|
||||
f"transition_work_item: settlement fallback failed wid={work_item_id}"
|
||||
)
|
||||
raise
|
||||
except Exception:
|
||||
logger.opt(exception=True).warning(
|
||||
f"transition_work_item failed wid={work_item_id} "
|
||||
f"target={phase.value} reason={reason_clean}"
|
||||
)
|
||||
raise
|
||||
if release_claim and result is not None:
|
||||
try:
|
||||
result = await store.update_delegation_work_item(
|
||||
work_item_id,
|
||||
claimed_by_role_runtime_session_id="",
|
||||
claimed_by_seat_id="",
|
||||
)
|
||||
except Exception:
|
||||
logger.opt(exception=True).warning(
|
||||
f"transition_work_item: claim release failed wid={work_item_id}"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def settle_open_attempt_as_interrupted(
|
||||
store: Any,
|
||||
work_item: DelegationWorkItem,
|
||||
) -> bool:
|
||||
"""Back-fill the ledger for an attempt whose owner died without settling.
|
||||
|
||||
Metadata-only write (no phase change): the card keeps whatever phase the
|
||||
crash left it in — recovery semantics stay untouched — but the attempt
|
||||
stops being invisible. Called by the dispatcher's per-tick reconcile pass
|
||||
for cards with an open attempt and no live in-process owner. Returns True
|
||||
when a settlement write was issued.
|
||||
"""
|
||||
if store is None or not hasattr(store, "update_delegation_work_item"):
|
||||
return False
|
||||
metadata = dict(getattr(work_item, "metadata", {}) or {})
|
||||
settlement = build_attempt_settlement_updates(metadata, outcome="interrupted")
|
||||
if not settlement:
|
||||
return False
|
||||
try:
|
||||
updated = await store.update_delegation_work_item(
|
||||
work_item.work_item_id,
|
||||
metadata_updates=settlement,
|
||||
)
|
||||
except Exception:
|
||||
logger.opt(exception=True).warning(
|
||||
"settle_open_attempt_as_interrupted failed "
|
||||
f"wid={getattr(work_item, 'work_item_id', '')}"
|
||||
)
|
||||
return False
|
||||
if updated is not None:
|
||||
work_item.metadata = dict(updated.metadata or {})
|
||||
return True
|
||||
|
||||
|
||||
def _fallback_status_for(
|
||||
target_status_or_phase: TaskStatus | Phase | str,
|
||||
task: Task,
|
||||
@@ -162,6 +275,7 @@ async def transition_work_item_from_task(
|
||||
metadata_updates: dict[str, Any] | None = None,
|
||||
release_claim: bool = False,
|
||||
require_work_item: bool = False,
|
||||
attempt_outcome: str | None = None,
|
||||
) -> bool:
|
||||
"""Task bridge helper: transition a work item when the caller holds a Task.
|
||||
|
||||
@@ -306,6 +420,7 @@ async def transition_work_item_from_task(
|
||||
summary=summary,
|
||||
metadata_updates=back_ref,
|
||||
release_claim=release_claim,
|
||||
attempt_outcome=attempt_outcome,
|
||||
)
|
||||
except InvalidPhaseTransition:
|
||||
# Defensive: state-machine validation at the store layer can also
|
||||
|
||||
Reference in New Issue
Block a user