fix(company): make delegation review lifecycle durable

This commit is contained in:
LZH-YS1998
2026-07-14 10:52:22 +08:00
parent 4bc18dcd27
commit 5e02364eb4
19 changed files with 3150 additions and 1087 deletions
+151 -17
View File
@@ -76,7 +76,7 @@ from opc.layer2_organization.phase import (
InvalidPhaseTransition,
TODO_PHASES,
coerce_phase,
is_resumable_after_claim_release,
is_stale_claim_releasable,
is_terminal,
kanban_column,
on_phase_transition,
@@ -4600,9 +4600,9 @@ class OPCStore:
We only touch in-flight phases (RUNNING / WAITING_FOR_* /
PAUSED / NEEDS_ATTENTION / AWAITING_*) so we never disturb
terminal cards. The phase itself is left alone; the sweep only
clears the claim. The dispatcher's ``is_dispatchable`` check
recognises a non-terminal card with no claim as eligible for
re-pick on the next tick.
clears the claim. Active execution can subsequently be re-picked;
passive AWAITING_* parents remain non-dispatchable while their
report/review chain (or human) advances them.
"""
if self._db is None:
return 0
@@ -4619,7 +4619,7 @@ class OPCStore:
phase = coerce_phase(phase_str)
except (TypeError, ValueError):
continue
if not is_resumable_after_claim_release(phase):
if not is_stale_claim_releasable(phase):
continue
metadata = _json_loads(metadata_json, {})
metadata["claimed_by_role_session_id"] = ""
@@ -5102,13 +5102,26 @@ class OPCStore:
return total
async def save_delegation_work_item(self, item: DelegationWorkItem) -> None:
async def save_delegation_work_item(
self,
item: DelegationWorkItem,
) -> None:
await self._write_delegation_work_item(item, if_absent=False)
async def _write_delegation_work_item(
self,
item: DelegationWorkItem,
*,
if_absent: bool,
) -> bool:
# Single-source-of-truth gate: every write — whether it goes through
# update_delegation_work_item or directly mutates `item.phase` and
# then calls save — passes through validate_transition. Skipping the
# validation requires a separate code path; there is no way to write
# an invalid phase by accident.
existing = await self.get_delegation_work_item(item.work_item_id)
if if_absent and existing is not None:
return False
previous_phase = existing.phase if existing is not None else None
validate_transition(previous_phase, item.phase)
item.metadata = dict(item.metadata or {})
@@ -5127,16 +5140,10 @@ class OPCStore:
# may want to know "what changed".
target_phase = item.phase
db = self._require_db()
await db.execute(
"""INSERT INTO delegation_work_items
(work_item_id, run_id, cell_id, team_instance_id, team_id, role_id, seat_id, seat_state_id,
role_runtime_session_id, parent_work_item_id, source_role_id, source_seat_id, title, summary,
kind, projection_id, phase, batch_id, batch_index,
deliverable_summary, blocked_reason, handoff_status, continuation_source, manager_role_id,
manager_seat_id, claimed_by_role_runtime_session_id, claimed_by_seat_id, metadata,
created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(work_item_id) DO UPDATE SET
conflict_action = (
"DO NOTHING"
if if_absent
else """DO UPDATE SET
run_id=excluded.run_id,
cell_id=excluded.cell_id,
team_instance_id=excluded.team_instance_id,
@@ -5165,7 +5172,18 @@ class OPCStore:
claimed_by_seat_id=excluded.claimed_by_seat_id,
metadata=excluded.metadata,
created_at=excluded.created_at,
updated_at=excluded.updated_at""",
updated_at=excluded.updated_at"""
)
cursor = await db.execute(
f"""INSERT INTO delegation_work_items
(work_item_id, run_id, cell_id, team_instance_id, team_id, role_id, seat_id, seat_state_id,
role_runtime_session_id, parent_work_item_id, source_role_id, source_seat_id, title, summary,
kind, projection_id, phase, batch_id, batch_index,
deliverable_summary, blocked_reason, handoff_status, continuation_source, manager_role_id,
manager_seat_id, claimed_by_role_runtime_session_id, claimed_by_seat_id, metadata,
created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
ON CONFLICT(work_item_id) {conflict_action}""",
(
item.work_item_id,
item.run_id,
@@ -5200,6 +5218,8 @@ class OPCStore:
),
)
await db.commit()
if if_absent and not (getattr(cursor, "rowcount", 0) or 0):
return False
# D2 hook fire — propagate phase change to dependent layers
# (task.status, role_session.status, dispatcher wake, etc.). All
# writes to delegation_work_items pass through here, so this is
@@ -5208,6 +5228,20 @@ class OPCStore:
await on_phase_transition(previous_phase, target_phase, item, store=self)
except Exception: # never let hook failures break the write
logger.opt(exception=True).debug("on_phase_transition raised at top level")
return True
async def insert_delegation_work_item_if_absent(
self,
item: DelegationWorkItem,
) -> bool:
"""Atomically create a WorkItem without overwriting a concurrent claim.
Auxiliary report/review IDs are deterministic. A read-before-write
check is insufficient because another dispatcher can create and claim
the same card between those operations; the conflict decision must be
made by SQLite in the insert statement itself.
"""
return await self._write_delegation_work_item(item, if_absent=True)
async def list_delegation_work_items(
self,
@@ -5475,6 +5509,106 @@ class OPCStore:
await self.save_delegation_work_item(item)
return item
async def apply_delegation_review_resolution(
self,
work_item_id: str,
*,
source_report_work_item_id: str,
target_phase: Phase | str,
blocked_reason: str,
metadata_updates: dict[str, Any],
) -> DelegationWorkItem | None:
"""Atomically apply a manager verdict to its exact report generation.
The phase predicate and latest-applied-report predicate live in the
same SQLite UPDATE as the child phase + applied-stamp write. This
prevents a late manager turn from crossing an AWAITING_HUMAN
transition or approving an older report after a newer report landed.
``updated_at`` provides optimistic metadata concurrency; a few retries
preserve unrelated same-phase updates without weakening either guard.
"""
target = coerce_phase(target_phase)
validate_transition(Phase.AWAITING_MANAGER_REVIEW, target)
expected_source = str(source_report_work_item_id or "").strip()
db = self._require_db()
for _attempt in range(3):
item = await self.get_delegation_work_item(work_item_id)
if item is None or item.phase != Phase.AWAITING_MANAGER_REVIEW:
return None
metadata = dict(item.metadata or {})
metadata.update(dict(metadata_updates or {}))
if (
self._metadata_has_work_item_projection_identity(metadata)
or str(item.projection_id or "").strip()
or str(item.kind or "").strip()
):
metadata, _ = migrate_work_item_projection_metadata(
metadata,
projection_id_fallback=str(
item.projection_id or item.work_item_id or ""
).strip(),
turn_type_fallback=str(item.kind or "").strip(),
)
previous_updated_at = item.updated_at.isoformat()
updated_at = datetime.now()
cursor = await db.execute(
"""UPDATE delegation_work_items
SET phase = ?, blocked_reason = ?, metadata = ?, updated_at = ?
WHERE work_item_id = ?
AND phase = ?
AND updated_at = ?
AND COALESCE((
SELECT report.work_item_id
FROM delegation_work_items AS report
WHERE report.parent_work_item_id = ?
AND report.kind = 'report'
AND json_extract(
report.metadata,
'$.report_target_work_item_id'
) = ?
AND json_extract(
report.metadata,
'$.report_card_outcome'
) = 'applied'
ORDER BY report.batch_index DESC,
report.created_at DESC,
report.work_item_id DESC
LIMIT 1
), '') = ?""",
(
target.value,
str(blocked_reason or ""),
_json_dumps(metadata),
updated_at.isoformat(),
work_item_id,
Phase.AWAITING_MANAGER_REVIEW.value,
previous_updated_at,
work_item_id,
work_item_id,
expected_source,
),
)
await db.commit()
if not (getattr(cursor, "rowcount", 0) or 0):
continue
persisted = await self.get_delegation_work_item(work_item_id)
if persisted is None:
return None
try:
await on_phase_transition(
Phase.AWAITING_MANAGER_REVIEW,
target,
persisted,
store=self,
)
except Exception:
logger.opt(exception=True).debug(
"on_phase_transition raised after review-resolution CAS"
)
return persisted
return None
async def reopen_approved_delegation_work_item_for_rework(
self,
work_item_id: str,
+3 -15
View File
@@ -117,6 +117,7 @@ from opc.layer2_organization.session_scoping import (
is_top_level_company_session,
task_session_scope_id,
)
from opc.layer2_organization.turn_mode import reset_manager_dispatch_turn_metadata
from opc.layer2_organization.seat_executor import EngineSeatExecutor
from opc.layer2_organization.work_item_runtime import (
is_work_item_runtime_metadata,
@@ -6945,11 +6946,7 @@ class OPCEngine:
task.metadata.pop("delegation_pending_work_item_ids", None)
task.metadata.pop("delegated_children_pending", None)
task.metadata.pop("delegation_wait_for_work_item_ids", None)
task.metadata.pop("manager_board_mutation_performed", None)
task.metadata.pop("manager_board_modified_work_item_ids", None)
task.metadata.pop("manager_board_deleted_work_item_ids", None)
task.metadata.pop("manager_no_delegation_justification", None)
task.metadata.pop("no_delegation_justification", None)
task.metadata = reset_manager_dispatch_turn_metadata(task.metadata)
progress = list(task.metadata.get("progress_log", []) or [])
progress.append(f"Company follow-up routed to final decider ({resume_source}): {reply}")
task.metadata["progress_log"] = progress[-20:]
@@ -10211,16 +10208,7 @@ class OPCEngine:
return True
if str(task_metadata.get("manager_no_delegation_justification", "") or "").strip():
return True
work_item_id = linked_work_item_id_for_task(task)
if not work_item_id or not hasattr(self.store, "get_delegation_work_item"):
return False
work_item = await self.store.get_delegation_work_item(work_item_id)
if work_item is None:
return False
metadata = dict(getattr(work_item, "metadata", {}) or {})
if bool(metadata.get("manager_board_mutation_performed", False)):
return True
if str(metadata.get("manager_no_delegation_justification", "") or "").strip():
if str(task_metadata.get("manager_dispatch_guard_unresolved", "") or "").strip():
return True
return False
File diff suppressed because it is too large Load Diff
@@ -131,6 +131,10 @@ _WORK_ITEM_FIELDS: tuple[MetadataFieldSpec, ...] = (
_spec("review_attempt", MetadataOwner.WORK_ITEM, allowed_locations=("work_item", "task_execution_copy"), legacy_fallback=True, migration_policy="work_item_wins"),
_spec("review_attempt_count", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), legacy_fallback=True, migration_policy="work_item_wins"),
_spec("review_target_work_item_id", MetadataOwner.WORK_ITEM, allowed_locations=("work_item", "task_execution_copy"), legacy_fallback=True, migration_policy="work_item_wins"),
_spec("review_source_report_work_item_id", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
_spec("review_resolution", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
_spec("review_resolution_state", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
_spec("review_resolution_applied_work_item_id", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
_spec("review_target_worker_task_id", MetadataOwner.WORK_ITEM, allowed_locations=("work_item", "task_execution_copy"), legacy_fallback=True, migration_policy="work_item_wins"),
_spec("review_target_worker_role_id", MetadataOwner.WORK_ITEM, allowed_locations=("work_item", "task_execution_copy"), legacy_fallback=True, migration_policy="work_item_wins"),
_spec("review_target_worker_seat_id", MetadataOwner.WORK_ITEM, allowed_locations=("work_item", "task_execution_copy"), legacy_fallback=True, migration_policy="work_item_wins"),
+51 -23
View File
@@ -49,7 +49,9 @@ __all__ = [
"phase_for_task_status",
"coerce_phase",
"is_review_execution_work_item_metadata",
"is_runtime_auxiliary_work_item",
"should_hide_work_item_from_company_kanban",
"is_stale_claim_releasable",
"is_resumable_after_claim_release",
"is_orphaned",
"is_dispatchable",
@@ -146,6 +148,27 @@ def is_report_execution_work_item_metadata(metadata: Mapping[str, Any] | None) -
return work_kind == "report" and bool(str(data.get("report_target_work_item_id", "") or "").strip())
def is_runtime_auxiliary_work_item(value_or_metadata: Any) -> bool:
"""True for attention, report, and review runtime helper cards.
Accepts either a work-item-like object, a serialized work item containing
a ``metadata`` mapping, or the metadata mapping itself. Keeping this
identity check below the company runtime gives lifecycle and board code a
single dependency-free definition of a non-business child.
"""
if isinstance(value_or_metadata, Mapping):
nested = value_or_metadata.get("metadata")
metadata = nested if isinstance(nested, Mapping) else value_or_metadata
else:
metadata = getattr(value_or_metadata, "metadata", None)
data = dict(metadata or {}) if isinstance(metadata, Mapping) else {}
return (
bool(data.get("attention_work_item", False))
or is_report_execution_work_item_metadata(data)
or is_review_execution_work_item_metadata(data)
)
def should_hide_work_item_from_company_kanban(metadata: Mapping[str, Any] | None) -> bool:
"""True when the kanban UI should not display this work item."""
data = dict(metadata or {})
@@ -316,40 +339,45 @@ def is_runnable(phase: Phase) -> bool:
return phase in RUNNABLE_PHASES
# Phases whose runtime claim, when released as stale (process restart, crashed
# session), allow the dispatcher to re-pick the card. Without this set, a card
# that was actively running when the process died becomes a zombie: phase still
# says RUNNING / WAITING_FOR_*, but no session is alive to make progress.
_RESUMABLE_AFTER_STALE_CLAIM: frozenset[Phase] = frozenset({
Phase.RUNNING,
Phase.WAITING_FOR_PEER,
Phase.WAITING_FOR_CHILDREN,
Phase.PAUSED,
Phase.NEEDS_ATTENTION,
Phase.AWAITING_MANAGER_REVIEW,
Phase.AWAITING_HUMAN,
})
# A process restart invalidates every persisted runtime claim, including claims
# on passive review states. Releasing a dead claim and allowing the original
# worker to execute again are intentionally separate decisions: review parents
# must lose their dead claim but remain passive while their report/review
# auxiliaries resume.
_STALE_CLAIM_RELEASABLE_PHASES: frozenset[Phase] = (
IN_PROGRESS_PHASES | IN_REVIEW_PHASES
)
_RESUMABLE_AFTER_CLAIM_RELEASE_PHASES: frozenset[Phase] = IN_PROGRESS_PHASES
def is_stale_claim_releasable(phase: Phase) -> bool:
"""Whether startup recovery may clear a dead runtime claim.
This includes passive review/human-wait states so a crashed resident
session cannot retain ownership forever. It does *not* imply that the
original work item may be dispatched again.
"""
return phase in _STALE_CLAIM_RELEASABLE_PHASES
def is_resumable_after_claim_release(phase: Phase) -> bool:
"""True iff a stale claim on this phase can be released and the card
re-picked up by the dispatcher (rather than left as a zombie).
"""Whether the original worker may resume after its claim is released.
Used by the periodic stale-claim sweeper. Every in-flight phase must
return True here — the invariant test in
test_phase_state_machine_invariants.py enforces this.
Active execution phases can be re-picked after a crash. Passive
``AWAITING_*`` parents cannot: their report/review chain (or human) owns
forward progress.
"""
return phase in _RESUMABLE_AFTER_STALE_CLAIM
return phase in _RESUMABLE_AFTER_CLAIM_RELEASE_PHASES
def is_orphaned(item: Any) -> bool:
"""A work item is orphaned when its phase says 'in flight' but no
runtime session currently holds a claim on it.
Typical scenario: the process that owned the claim died (restart,
crash). On startup the stale-claim sweeper clears the claim
metadata; this function then lets the dispatcher re-pick the card
on the next tick, eliminating zombie work items (Bug C).
Typical scenario: the process that owned an execution claim died. On
startup the stale-claim sweeper clears the claim metadata; this function
then lets the dispatcher re-pick active execution, but never a passive
review parent.
"""
if not is_resumable_after_claim_release(item.phase):
return False
+12
View File
@@ -31,6 +31,7 @@ from opc.core.models import Phase, TaskStatus
from opc.layer2_organization.phase import (
DONE_PHASES,
RUNNABLE_PHASES,
is_stale_claim_releasable,
register_phase_transition_hook,
task_status_for_phase,
)
@@ -365,6 +366,17 @@ def _active_focus_id(session: Any, work_item_by_id: dict[str, Any]) -> str:
return ""
if getattr(focused_item, "phase", None) in DONE_PHASES:
return ""
phase = getattr(focused_item, "phase", None)
if is_stale_claim_releasable(phase):
claim = str(
getattr(focused_item, "claimed_by_role_runtime_session_id", "") or ""
).strip()
if not claim:
# On a live transition the focused item retains its claim. After
# restart the startup sweep removes that dead-process claim; the
# persisted session focus must stop blocking the orphaned item (or
# a passive parent's report/review helper) from being dispatched.
return ""
return focused
+23
View File
@@ -30,6 +30,29 @@ from typing import Any, Mapping
from opc.core.models import Phase
MANAGER_DISPATCH_TURN_METADATA_KEYS: tuple[str, ...] = (
"manager_board_mutation_performed",
"manager_board_modified_work_item_ids",
"manager_board_deleted_work_item_ids",
"manager_no_delegation_justification",
"no_delegation_justification",
"manager_dispatch_guard_unresolved",
)
def reset_manager_dispatch_turn_metadata(metadata: Mapping[str, Any] | None) -> dict[str, Any]:
"""Return mutable metadata with prior manager-turn outcomes removed.
These keys describe what happened in one agent turn. They must not be
carried into a retry, rework turn, or user follow-up; durable board state
(dependencies and child mutation revisions) is intentionally untouched.
"""
result = dict(metadata or {})
for key in MANAGER_DISPATCH_TURN_METADATA_KEYS:
result.pop(key, None)
return result
class TurnMode(str, Enum):
EXECUTE = "execute"
DELEGATE = "delegate"
+5 -17
View File
@@ -155,25 +155,13 @@ def is_delivery_turn(value_or_metadata: Any) -> bool:
def is_manager_reviewable_turn(value_or_metadata: Any) -> bool:
"""Return True when a finished WorkItem should enter manager review flow.
"""Return the turn type's default manager-review policy.
An explicit persisted ``turn_output_kind`` wins over the turn-type
default: delegation-kind turns (dispatch/intake/plan) are review-exempt
only because their deliverable is normally a child card set that gets
reviewed per child. When such a turn completed with the manager's own
work product instead, the done-transition stamps
``turn_output_kind=self_produced`` on the WorkItem and that output is
reviewable like any execute turn — every consumer of this predicate
(DONE routing, report spawn, report completion, recovery scans) follows
automatically.
Dispatch/intake/plan are normally board-producing turns and therefore
exempt. The executor handles the one dynamic exception — a current turn
that produced no board mutation — before it writes the authoritative
WorkItem phase. Downstream review plumbing follows that phase directly.
"""
metadata: Mapping[str, Any] | None = None
if isinstance(value_or_metadata, Mapping):
metadata = value_or_metadata
elif hasattr(value_or_metadata, "metadata"):
metadata = getattr(value_or_metadata, "metadata", None) or {}
if metadata and str(metadata.get("turn_output_kind", "") or "").strip().lower() == "self_produced":
return True
turn_type = _turn_type_for_value(value_or_metadata, fallback="")
if not turn_type:
return False
+5
View File
@@ -2230,6 +2230,11 @@ def create_collaboration_tools(
task.metadata = dict(task.metadata)
task.metadata["delegation_wait_for_work_item_ids"] = parent_dependency_ids
task.metadata["manager_board_parent_work_item_id"] = parent_work_item_id
# Reusing an existing scope-key match is not a current-turn board
# mutation. Only newly persisted business children make this
# attempt a delegated-board completion.
if resolved_pending_items:
task.metadata["manager_board_mutation_performed"] = True
if attention_work_item_id:
task.metadata["attention_business_parent_work_item_id"] = parent_work_item_id
task.metadata["attention_work_item_id"] = attention_work_item_id
+252 -33
View File
@@ -967,6 +967,7 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
)
self.assertEqual(issues, [])
self.assertTrue(self.task.metadata.get("manager_board_mutation_performed"))
async def test_manager_dispatch_guard_accepts_existing_child_mutation(self) -> None:
await self.store.save_delegation_work_item(
@@ -1012,6 +1013,114 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
)
self.assertEqual(issues, [])
self.assertTrue(self.task.metadata.get("manager_board_mutation_performed"))
async def test_manager_dispatch_guard_accepts_existing_child_deletion(self) -> None:
await self.store.save_delegation_work_item(
DelegationWorkItem(
work_item_id="cto-child-item",
run_id="run-1",
cell_id="team::ceo",
team_instance_id="team-instance::run-1::team::ceo",
team_id="team::ceo",
role_id="cto",
seat_id="seat::team::ceo::cto",
seat_state_id="seat-state::run-1::seat::team::ceo::cto",
role_runtime_session_id="role-runtime::run-1::seat::team::ceo::cto",
parent_work_item_id="ceo-work-item",
title="Obsolete CTO Work",
summary="Remove this obsolete branch.",
kind="execute",
projection_id="cto-child-item",
phase=Phase.RUNNING,
manager_role_id="ceo",
manager_seat_id="seat::team::ceo::ceo",
metadata={
"work_item_runtime": True,
"runtime_model": "multi_team_org",
"manager_mutation_revision": 0,
},
)
)
before = await self.executor._snapshot_manager_dispatch_state(self.task)
await self.store.amend_delegation_work_item(
"cto-child-item",
metadata_set={
"manager_mutation_revision": 1,
"manager_mutation_action": "delete",
"deleted_by_manager_tool": True,
"hidden_from_company_kanban": True,
"upstream_visibility": "hidden",
},
)
issues = await self.executor._enforce_manager_dispatch_guard(
self.task,
TaskResult(status=TaskStatus.DONE, content="Deleted the obsolete child work item."),
before_state=before,
)
self.assertEqual(issues, [])
self.assertTrue(self.task.metadata.get("manager_board_mutation_performed"))
async def test_historical_dependency_does_not_satisfy_current_turn_guard(self) -> None:
await self.store.save_delegation_work_item(
DelegationWorkItem(
work_item_id="historical-child",
run_id="run-1",
parent_work_item_id="ceo-work-item",
role_id="cto",
seat_id="seat::team::ceo::cto",
title="Historical child",
summary="Created in an earlier turn.",
kind="execute",
projection_id="historical-child",
phase=Phase.APPROVED,
metadata={"work_item_runtime": True, "runtime_model": "multi_team_org"},
)
)
self.task.metadata["delegation_wait_for_work_item_ids"] = ["historical-child"]
before = await self.executor._snapshot_manager_dispatch_state(self.task)
issues = await self.executor._enforce_manager_dispatch_guard(
self.task,
TaskResult(status=TaskStatus.DONE, content="Handled this new request directly."),
before_state=before,
)
self.assertEqual(len(issues), 1)
self.assertFalse(self.task.metadata.get("manager_board_mutation_performed", False))
async def test_attention_child_created_during_turn_does_not_satisfy_dispatch_guard(self) -> None:
before = await self.executor._snapshot_manager_dispatch_state(self.task)
await self.store.save_delegation_work_item(
DelegationWorkItem(
work_item_id="ceo-attention-item",
run_id="run-1",
parent_work_item_id="ceo-work-item",
role_id="ceo",
seat_id="seat::team::ceo::ceo",
title="CEO attention",
summary="Runtime wake-up wrapper, not delegated business work.",
kind="monitor",
projection_id="ceo-attention-item",
phase=Phase.READY,
metadata={
"work_item_runtime": True,
"runtime_model": "multi_team_org",
"attention_work_item": True,
},
)
)
issues = await self.executor._enforce_manager_dispatch_guard(
self.task,
TaskResult(status=TaskStatus.DONE, content="Handled this request directly."),
before_state=before,
)
self.assertEqual(len(issues), 1)
self.assertFalse(self.task.metadata.get("manager_board_mutation_performed", False))
async def test_manager_dispatch_guard_exhaustion_accepts_turn_instead_of_failing(self) -> None:
# Dispatch is a soft constraint: when the guard reminders run out,
@@ -1029,6 +1138,11 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
# per-turn reset must clear it, otherwise the guard would
# silently pass without reminders.
"manager_board_mutation_performed": True,
"manager_board_modified_work_item_ids": ["stale-modified-child"],
"manager_board_deleted_work_item_ids": ["stale-deleted-child"],
"manager_no_delegation_justification": "stale reason from an earlier turn",
"no_delegation_justification": "another stale reason",
"manager_dispatch_guard_unresolved": "stale unresolved guard",
},
)
task.status = TaskStatus.PENDING
@@ -1054,13 +1168,22 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
# card waits for CEO review and a live report card drives it.
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
self.assertEqual(work_item.phase, Phase.AWAITING_MANAGER_REVIEW)
self.assertEqual(
str((work_item.metadata or {}).get("turn_output_kind", "")), "self_produced"
self.assertNotIn("manager_board_mutation_performed", task.metadata)
self.assertNotIn("manager_board_modified_work_item_ids", task.metadata)
self.assertNotIn("manager_board_deleted_work_item_ids", task.metadata)
self.assertNotIn("manager_no_delegation_justification", task.metadata)
self.assertNotIn("no_delegation_justification", task.metadata)
self.assertNotIn("stale unresolved guard", task.metadata["manager_dispatch_guard_unresolved"])
dispatch_evidence = dict(
dict((work_item.metadata or {}).get("review_evidence", {}) or {}).get(
"manager_dispatch", {}
)
self.assertEqual(
str((work_item.metadata or {}).get("turn_output_source", "")),
"dispatch_guard_exhausted",
or {}
)
self.assertEqual(dispatch_evidence.get("outcome"), "self_produced")
self.assertEqual(dispatch_evidence.get("source"), "dispatch_guard_exhausted")
self.assertNotIn("turn_output_kind", work_item.metadata or {})
self.assertNotIn("turn_output_source", work_item.metadata or {})
report_cards = await self._aux_cards_targeting("cto-dispatch-item", "report_target_work_item_id")
self.assertEqual(len(report_cards), 1)
self.assertNotIn(report_cards[0].phase, DONE_PHASES)
@@ -1131,15 +1254,29 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(phase, Phase.AWAITING_MANAGER_REVIEW)
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
self.assertEqual(work_item.phase, Phase.AWAITING_MANAGER_REVIEW)
self.assertEqual(
str((work_item.metadata or {}).get("turn_output_source", "")), "justified"
dispatch_evidence = dict(
dict((work_item.metadata or {}).get("review_evidence", {}) or {}).get(
"manager_dispatch", {}
)
or {}
)
self.assertEqual(dispatch_evidence.get("outcome"), "self_produced")
self.assertEqual(dispatch_evidence.get("source"), "justified")
self.assertEqual(dispatch_evidence.get("note"), "single-seat scoping decision")
self.assertNotIn("turn_output_kind", work_item.metadata or {})
self.assertNotIn("turn_output_source", work_item.metadata or {})
report_cards = await self._aux_cards_targeting("cto-dispatch-item", "report_target_work_item_id")
self.assertEqual(len(report_cards), 1)
report_card = report_cards[0]
self.assertNotIn(report_card.phase, DONE_PHASES)
# Drive the report turn to completion — the review card must appear.
# Production dispatch claims READY → RUNNING before materializing
# the Task; this direct lifecycle test must model that claim.
await self.store.update_delegation_work_item(
report_card.work_item_id,
phase=Phase.RUNNING,
)
report_task = Task(
id="cto-report-task",
title=report_card.title,
@@ -1165,16 +1302,26 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(len(review_cards), 1)
self.assertNotIn(review_cards[0].phase, DONE_PHASES)
self.assertEqual(str(review_cards[0].role_id or ""), "ceo")
review_dispatch_evidence = dict(
dict((review_cards[0].metadata or {}).get("review_evidence", {}) or {}).get(
"manager_dispatch", {}
)
or {}
)
self.assertEqual(review_dispatch_evidence.get("source"), "justified")
async def test_dispatch_turn_that_delegated_keeps_auto_approve(self) -> None:
# Normal delegation flow: a live child card exists in the store, so
# the dispatch exemption applies (children carry the reviewable
# output) — classification comes from store ground truth, not from
# transient task markers.
task = await self._make_cto_dispatch_task()
async def test_historical_business_child_does_not_exempt_current_self_output(self) -> None:
# A child created by a previous attempt is durable board state, not
# proof that this completion delegated its output. The current direct
# work product still needs the manager review chain.
task = await self._make_cto_dispatch_task(
metadata_extra={
"manager_no_delegation_justification": "This new request is a direct architecture decision."
}
)
await self.store.save_delegation_work_item(
DelegationWorkItem(
work_item_id="cto-delegated-child",
work_item_id="cto-historical-child",
run_id="run-1",
cell_id="team::ceo",
team_instance_id="team-instance::run-1::team::ceo",
@@ -1184,26 +1331,93 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
seat_state_id="seat-state::run-1::seat::team::ceo::cto",
role_runtime_session_id="role-runtime::run-1::seat::team::ceo::cto",
parent_work_item_id="cto-dispatch-item",
title="Delegated child",
summary="Build the feature.",
title="Historical delegated child",
summary="Completed in a previous attempt.",
kind="execute",
projection_id="cto-delegated-child",
phase=Phase.READY,
projection_id="cto-historical-child",
phase=Phase.APPROVED,
manager_role_id="cto",
manager_seat_id="seat::team::ceo::cto",
metadata={"work_item_runtime": True, "runtime_model": "multi_team_org"},
)
)
phase = await self.executor._apply_done_transition(
task,
result=TaskResult(
status=TaskStatus.DONE,
content="Made the new architecture decision directly.",
),
)
self.assertEqual(phase, Phase.AWAITING_MANAGER_REVIEW)
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
report_cards = await self._aux_cards_targeting("cto-dispatch-item", "report_target_work_item_id")
self.assertEqual(len(report_cards), 1)
dispatch_evidence = dict(
dict((work_item.metadata or {}).get("review_evidence", {}) or {}).get(
"manager_dispatch", {}
)
or {}
)
self.assertEqual(dispatch_evidence.get("outcome"), "self_produced")
async def test_attention_aux_does_not_exempt_current_self_output(self) -> None:
task = await self._make_cto_dispatch_task(
metadata_extra={
"manager_no_delegation_justification": "This request requires a direct architecture decision."
}
)
await self.store.save_delegation_work_item(
DelegationWorkItem(
work_item_id="cto-attention-item",
run_id="run-1",
parent_work_item_id="cto-dispatch-item",
role_id="cto",
seat_id="seat::team::ceo::cto",
title="CTO attention",
summary="Runtime wake-up wrapper, not delegated business work.",
kind="monitor",
projection_id="cto-attention-item",
phase=Phase.APPROVED,
metadata={
"work_item_runtime": True,
"runtime_model": "multi_team_org",
"attention_work_item": True,
},
)
)
phase = await self.executor._apply_done_transition(
task,
result=TaskResult(
status=TaskStatus.DONE,
content="Made the architecture decision directly.",
),
)
self.assertEqual(phase, Phase.AWAITING_MANAGER_REVIEW)
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
self.assertEqual(work_item.phase, Phase.AWAITING_MANAGER_REVIEW)
self.assertEqual(
len(await self._aux_cards_targeting("cto-dispatch-item", "report_target_work_item_id")),
1,
)
async def test_current_turn_business_board_mutation_keeps_dispatch_auto_approve(self) -> None:
task = await self._make_cto_dispatch_task(
metadata_extra={"manager_board_mutation_performed": True}
)
phase = await self.executor._apply_done_transition(
task, result=TaskResult(status=TaskStatus.DONE, content="Delegated to the team."),
)
self.assertEqual(phase, Phase.APPROVED)
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
self.assertEqual(
str((work_item.metadata or {}).get("turn_output_kind", "")), "delegated"
)
self.assertEqual(work_item.phase, Phase.APPROVED)
self.assertNotIn("turn_output_kind", work_item.metadata or {})
self.assertNotIn("turn_output_source", work_item.metadata or {})
report_cards = await self._aux_cards_targeting("cto-dispatch-item", "report_target_work_item_id")
self.assertEqual(report_cards, [])
@@ -1219,15 +1433,15 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(phase, Phase.AWAITING_MANAGER_REVIEW)
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
self.assertEqual(
str((work_item.metadata or {}).get("turn_output_kind", "")), "self_produced"
)
self.assertEqual(work_item.phase, Phase.AWAITING_MANAGER_REVIEW)
self.assertNotIn("turn_output_kind", work_item.metadata or {})
self.assertNotIn("turn_output_source", work_item.metadata or {})
async def test_top_seat_self_produced_falls_back_to_auto_approve(self) -> None:
# The CEO has no manager to review: the existing no-reviewer
# fallback auto-approves instead of stranding the card.
self.task.status = TaskStatus.DONE
self.task.metadata["work_kind"] = "dispatch"
self.task.metadata["work_kind"] = "intake"
self.task.metadata["manager_dispatch_guard_unresolved"] = "no children"
await self.store.update_delegation_work_item("ceo-work-item", phase=Phase.RUNNING)
@@ -1236,20 +1450,25 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
)
self.assertEqual(phase, Phase.APPROVED)
work_item = await self.store.get_delegation_work_item("ceo-work-item")
self.assertEqual(work_item.phase, Phase.APPROVED)
self.assertEqual(
await self._aux_cards_targeting("ceo-work-item", "report_target_work_item_id"),
[],
)
self.assertEqual(
await self._aux_cards_targeting("ceo-work-item", "review_target_work_item_id"),
[],
)
async def test_reconcile_rebuilds_missing_report_card(self) -> None:
# Legacy shape: card already parked in AWAITING_MANAGER_REVIEW with
# the self_produced marker but no live report/review card (the
# historical spawn refused dispatch parents). The dispatcher-tick
# reconcile must rebuild the report card idempotently.
# Current-state crash shape: phase was durably written but the process
# stopped before its report card was saved. Phase alone is the
# authoritative recovery fact; no output-kind marker is required.
await self._make_cto_dispatch_task()
await self.store.update_delegation_work_item(
"cto-dispatch-item",
phase=Phase.AWAITING_MANAGER_REVIEW,
metadata_updates={
"turn_output_kind": "self_produced",
"turn_output_source": "dispatch_guard_exhausted",
},
)
run_items = await self.store.list_delegation_work_items("run-1")
+10 -1
View File
@@ -1067,12 +1067,21 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
self.assertTrue(await engine._company_followup_target_progressed("task-ceo"))
task.metadata.pop("manager_no_delegation_justification", None)
task.metadata["manager_dispatch_guard_unresolved"] = (
"Soft dispatch constraint exhausted; accept the top-seat output."
)
await store.save_task(task)
self.assertTrue(await engine._company_followup_target_progressed("task-ceo"))
task.metadata.pop("manager_dispatch_guard_unresolved", None)
await store.save_task(task)
await store.update_delegation_work_item(
"wi-ceo",
metadata_updates={"manager_board_mutation_performed": True},
)
self.assertTrue(await engine._company_followup_target_progressed("task-ceo"))
# Dispatch outcome markers are attempt-scoped Task state. A stale
# WorkItem copy must not make a later follow-up look progressed.
self.assertFalse(await engine._company_followup_target_progressed("task-ceo"))
async def test_final_decider_followup_keeps_dispatch_turn_mode_with_existing_children(self) -> None:
runtime = CompanyRuntime(org_engine=None, communication=None, store=None)
+5 -1
View File
@@ -380,7 +380,11 @@ class ReviewWorkItemLifecycleTests(unittest.IsolatedAsyncioTestCase):
}
await store.save_delegation_work_item(child_work_item)
await executor._close_review_work_item_for_work_item("wi-child", outcome=Phase.FAILED.value)
await executor._persist_terminal_review_card(
review_work_item_id,
phase=Phase.CANCELLED,
outcome=Phase.FAILED.value,
)
refreshed_review = await store.get_delegation_work_item(review_work_item_id)
assert refreshed_review is not None
self.assertEqual(refreshed_review.phase, Phase.CANCELLED)
+34
View File
@@ -83,6 +83,40 @@ class MetadataOwnershipMatrixTests(unittest.TestCase):
self.assertNotIn("progress_log", copied)
self.assertIn("current_turn_mode", EXECUTION_COPY_KEYS)
def test_review_source_report_link_is_work_item_only_not_execution_copy(self) -> None:
key = "review_source_report_work_item_id"
item = _work_item(
{
key: "report::wi-1::v2",
"review_target_work_item_id": "wi-1",
}
)
self.assertEqual(metadata_owner_for_key(key), MetadataOwner.WORK_ITEM)
self.assertTrue(is_work_item_owned_key(key))
self.assertNotIn(key, EXECUTION_COPY_KEYS)
copied = copy_work_item_execution_metadata(item)
self.assertNotIn(key, copied)
self.assertEqual(copied["review_target_work_item_id"], "wi-1")
task = Task(id="task-1", metadata={key: "report::wi-1::v2"})
removed = strip_disallowed_work_item_metadata_from_runtime_task(task)
self.assertIn(key, removed)
self.assertNotIn(key, task.metadata)
for journal_key in (
"review_resolution",
"review_resolution_state",
"review_resolution_applied_work_item_id",
):
self.assertEqual(
metadata_owner_for_key(journal_key),
MetadataOwner.WORK_ITEM,
)
self.assertNotIn(journal_key, EXECUTION_COPY_KEYS)
def test_validate_metadata_ownership_reports_task_only_work_item_field(self) -> None:
item = _work_item({})
task = Task(
+45 -13
View File
@@ -30,7 +30,10 @@ from opc.layer2_organization.phase import (
TERMINAL_PHASES,
TODO_PHASES,
coerce_phase,
is_resumable_after_claim_release,
is_runnable,
is_runtime_auxiliary_work_item,
is_stale_claim_releasable,
is_terminal,
kanban_column,
validate_transition,
@@ -253,20 +256,49 @@ def test_review_work_item_id_per_attempt_is_unique() -> None:
# ── H: in-flight phase recoverability after stale claim release ───────────
def test_inflight_phase_recoverable_after_claim_release() -> None:
"""Every in-flight phase (RUNNING / WAITING_FOR_*) must be re-claimable
once the previous claim is released (e.g. after process restart).
def test_inflight_claims_are_releasable_but_only_execution_is_resumable() -> None:
"""Claim cleanup and worker re-execution are separate invariants.
Without this invariant, any restart leaves in-flight cards as zombies
that the dispatcher refuses to pick up. The recovery path is provided
by `is_resumable_after_claim_release` defined in `phase.py`.
All in-flight claims belong to the dead process and must be released on
restart. Active execution may then resume; passive review/human parents
must wait for their auxiliary or external actor instead of running twice.
"""
from opc.layer2_organization.phase import is_resumable_after_claim_release
inflight = IN_PROGRESS_PHASES | IN_REVIEW_PHASES
not_recoverable = [p.value for p in inflight if not is_resumable_after_claim_release(p)]
assert not not_recoverable, (
f"in-flight phases that cannot recover after a stale claim is "
f"released: {not_recoverable}. After process restart these cards "
f"would become permanent zombies."
not_releasable = [p.value for p in inflight if not is_stale_claim_releasable(p)]
assert not not_releasable, (
f"in-flight phases whose dead claims cannot be released: {not_releasable}"
)
not_resumable = [
p.value for p in IN_PROGRESS_PHASES
if not is_resumable_after_claim_release(p)
]
assert not not_resumable, (
f"active execution phases that cannot resume: {not_resumable}"
)
incorrectly_resumable = [
p.value for p in IN_REVIEW_PHASES
if is_resumable_after_claim_release(p)
]
assert not incorrectly_resumable, (
f"passive review phases incorrectly resume worker execution: "
f"{incorrectly_resumable}"
)
@pytest.mark.parametrize(
"value",
[
{"attention_work_item": True},
{"report_execution_work_item": True},
{"review_execution_work_item": True},
{"work_kind": "report", "report_target_work_item_id": "parent"},
{"work_kind": "review", "review_target_work_item_id": "parent"},
{"metadata": {"attention_work_item": True}},
],
)
def test_runtime_auxiliary_identity_is_canonical(value: dict[str, object]) -> None:
assert is_runtime_auxiliary_work_item(value)
def test_business_work_item_is_not_runtime_auxiliary() -> None:
assert not is_runtime_auxiliary_work_item({"work_kind": "execute"})
+19 -4
View File
@@ -214,9 +214,7 @@ class RefreshDependentsForRunTests(unittest.IsolatedAsyncioTestCase):
executor = self._executor()
executor._active_tasks = [manager_task]
created = await executor._materialize_follow_up_work_items(
manager_task,
TaskResult(
follow_up_result = TaskResult(
status=TaskStatus.DONE,
content="Create a PPT deck.",
artifacts={
@@ -230,7 +228,10 @@ class RefreshDependentsForRunTests(unittest.IsolatedAsyncioTestCase):
}
]
},
),
)
created = await executor._materialize_follow_up_work_items(
manager_task,
follow_up_result,
)
self.assertEqual(len(created), 1)
@@ -239,6 +240,20 @@ class RefreshDependentsForRunTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(follow.phase, Phase.READY)
self.assertEqual(parent_after.phase, Phase.WAITING_FOR_CHILDREN)
# Re-emitting the same dedupe key reuses board state; it is not a
# current-turn creation signal for the dispatch guard.
reused = await executor._materialize_follow_up_work_items(
manager_task,
follow_up_result,
)
self.assertEqual(reused, [])
follow_ups = [
item
for item in await self.store.list_delegation_work_items("run-materialize")
if (item.metadata or {}).get("follow_up_dedupe_key")
]
self.assertEqual(len(follow_ups), 1)
async def test_parent_wakes_when_last_child_approved(self) -> None:
"""The canonical app12 fix: parent in WAITING_FOR_CHILDREN with a
stale claim unblocks to RUNNING and releases the claim when all
+174 -1
View File
@@ -228,6 +228,18 @@ class IsDispatchableQueueFilterTests(unittest.TestCase):
)
self.assertFalse(is_dispatchable(item))
def test_passive_review_parent_without_claim_is_not_dispatchable(self) -> None:
for phase in (Phase.AWAITING_MANAGER_REVIEW, Phase.AWAITING_HUMAN):
with self.subTest(phase=phase):
item = DelegationWorkItem(
work_item_id=f"wi-{phase.value}",
run_id="r", cell_id="c", role_id="cto",
seat_id="seat", manager_role_id="ceo",
manager_seat_id="seat::ceo", title="t",
phase=phase,
)
self.assertFalse(is_dispatchable(item))
# ── enqueue_session_work_on_runnable_hook ────────────────────────────────
@@ -467,7 +479,10 @@ class SerialQueueReconcilerTests(_StoreFixture):
async def test_reconcile_preserves_valid_marker_for_busy_session(self) -> None:
self.store.role_serial_queue_enabled = True
sid = await self._seed_session(focused="wi-active", pending=["wi-next"])
await self._save_item(wid="wi-active", sid=sid, phase=Phase.RUNNING)
active = await self._save_item(wid="wi-active", sid=sid, phase=Phase.RUNNING)
active.claimed_by_role_runtime_session_id = sid
active.claimed_by_seat_id = "seat"
await self.store.save_delegation_work_item(active)
await self._save_item(wid="wi-next", sid=sid, marker=True)
result = await reconcile_role_serial_queues(self.store, "r")
@@ -479,6 +494,64 @@ class SerialQueueReconcilerTests(_StoreFixture):
self.assertEqual(item.metadata.get("queued_behind_session"), sid)
self.assertFalse(is_dispatchable(item))
async def test_reconcile_promotes_aux_behind_unclaimed_review_focus(self) -> None:
self.store.role_serial_queue_enabled = True
sid = await self._seed_session(
focused="wi-parent",
pending=["wi-report"],
status="running",
)
await self._save_item(
wid="wi-parent",
sid=sid,
phase=Phase.AWAITING_MANAGER_REVIEW,
)
report = await self._save_item(wid="wi-report", sid=sid, marker=True)
report.metadata.update({
"report_execution_work_item": True,
"report_target_work_item_id": "wi-parent",
})
await self.store.save_delegation_work_item(report)
result = await reconcile_role_serial_queues(self.store, "r")
self.assertIn(sid, result["cleared_focus_session_ids"])
self.assertIn("wi-report", result["promoted_work_item_ids"])
session = await self.store.get_delegation_role_session(sid)
self.assertEqual(session.focused_work_item_id, "")
self.assertEqual(session.pending_work_item_ids, [])
report_after = await self.store.get_delegation_work_item("wi-report")
self.assertNotIn("queued_behind_session", report_after.metadata)
self.assertTrue(is_dispatchable(report_after))
async def test_reconcile_preserves_live_claimed_review_focus(self) -> None:
self.store.role_serial_queue_enabled = True
sid = await self._seed_session(
focused="wi-parent",
pending=["wi-report"],
status="running",
)
parent = await self._save_item(
wid="wi-parent",
sid=sid,
phase=Phase.AWAITING_MANAGER_REVIEW,
)
parent.claimed_by_role_runtime_session_id = sid
parent.claimed_by_seat_id = "seat"
await self.store.save_delegation_work_item(parent)
await self._save_item(wid="wi-report", sid=sid, marker=True)
result = await reconcile_role_serial_queues(self.store, "r")
self.assertEqual(result["cleared_focus_session_ids"], [])
self.assertEqual(result["promoted_work_item_ids"], [])
session = await self.store.get_delegation_role_session(sid)
self.assertEqual(session.focused_work_item_id, "wi-parent")
self.assertEqual(session.pending_work_item_ids, ["wi-report"])
report = await self.store.get_delegation_work_item("wi-report")
self.assertEqual(report.metadata.get("queued_behind_session"), sid)
self.assertFalse(is_dispatchable(report))
async def test_reconcile_prunes_dead_entries_and_promotes_one_head(self) -> None:
self.store.role_serial_queue_enabled = True
sid = "role-runtime::r::cto"
@@ -559,6 +632,106 @@ class SerialQueueReconcilerTests(_StoreFixture):
self.assertEqual(repaired.status, TaskStatus.DONE)
class StartupClaimSweepTests(_StoreFixture):
async def test_restart_clears_dead_running_focus_so_auxiliary_can_resume(self) -> None:
sid = "role-runtime::r::cto"
report_id = "report::wi-parent::v1"
await self.store.save_delegation_role_session(
DelegationRoleSession(
role_session_id=sid,
run_id="r",
role_id="cto",
focused_work_item_id=report_id,
status="running",
)
)
await self.store.save_delegation_work_item(
DelegationWorkItem(
work_item_id=report_id,
run_id="r",
cell_id="c",
role_id="cto",
seat_id="seat",
role_runtime_session_id=sid,
manager_role_id="ceo",
manager_seat_id="seat::ceo",
title="report",
kind="report",
phase=Phase.RUNNING,
claimed_by_role_runtime_session_id=sid,
claimed_by_seat_id="seat",
metadata={
"report_execution_work_item": True,
"report_target_work_item_id": "wi-parent",
"claimed_by_role_session_id": sid,
"claimed_task_id": "task-report",
},
)
)
db_path = self.store.db_path
await self.store.close()
self.store = OPCStore(db_path=db_path)
await self.store.initialize()
self.store.role_serial_queue_enabled = True
report = await self.store.get_delegation_work_item(report_id)
session_before = await self.store.get_delegation_role_session(sid)
self.assertEqual(report.claimed_by_role_runtime_session_id, "")
self.assertTrue(is_dispatchable(report))
self.assertEqual(session_before.focused_work_item_id, report_id)
self.assertEqual(session_before.status, "running")
result = await reconcile_role_serial_queues(self.store, "r")
session_after = await self.store.get_delegation_role_session(sid)
self.assertIn(sid, result["cleared_focus_session_ids"])
self.assertEqual(session_after.focused_work_item_id, "")
self.assertEqual(session_after.status, "idle")
self.assertTrue(is_dispatchable(await self.store.get_delegation_work_item(report_id)))
async def test_restart_releases_review_claim_without_dispatching_parent(self) -> None:
parent = DelegationWorkItem(
work_item_id="wi-parent",
run_id="r", cell_id="c", role_id="cto", seat_id="seat",
manager_role_id="ceo", manager_seat_id="seat::ceo", title="parent",
phase=Phase.AWAITING_MANAGER_REVIEW,
claimed_by_role_runtime_session_id="dead-role-session",
claimed_by_seat_id="seat",
metadata={
"claimed_by_role_session_id": "dead-role-session",
"claimed_task_id": "task-parent",
},
)
report = DelegationWorkItem(
work_item_id="wi-report",
run_id="r", cell_id="c", role_id="cto", seat_id="seat",
manager_role_id="ceo", manager_seat_id="seat::ceo", title="report",
phase=Phase.READY,
metadata={
"report_execution_work_item": True,
"report_target_work_item_id": "wi-parent",
},
)
await self.store.save_delegation_work_item(parent)
await self.store.save_delegation_work_item(report)
db_path = self.store.db_path
await self.store.close()
self.store = OPCStore(db_path=db_path)
await self.store.initialize()
parent_after = await self.store.get_delegation_work_item("wi-parent")
report_after = await self.store.get_delegation_work_item("wi-report")
self.assertEqual(parent_after.claimed_by_role_runtime_session_id, "")
self.assertEqual(parent_after.claimed_by_seat_id, "")
self.assertEqual(parent_after.metadata.get("claimed_by_role_session_id"), "")
self.assertEqual(parent_after.metadata.get("claimed_task_id"), "")
self.assertTrue(parent_after.metadata.get("claim_swept_at"))
self.assertFalse(is_dispatchable(parent_after))
self.assertTrue(is_dispatchable(report_after))
# ── Config loader picks up the feature flag ──────────────────────────────
+125
View File
@@ -29,6 +29,7 @@ from opc.layer2_organization.communication import CommunicationManager
from opc.layer2_organization.company_mode import (
CompanyWorkItemExecutor,
MAX_VERDICT_PARSE_RETRIES,
report_work_item_id_for_attempt,
review_work_item_id_for_attempt,
)
from opc.layer2_organization.org_engine import OrgEngine
@@ -238,6 +239,130 @@ class VerdictParseRetryTests(unittest.IsolatedAsyncioTestCase):
finally:
await store.close()
async def test_retry_card_save_failure_stays_in_review_and_reconciles(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
store = OPCStore(root / "tasks.db")
await store.initialize()
try:
executor = _build_executor(store, _make_org_engine(root))
child, review_id_v1 = _build_review_setup(store)
report_id = report_work_item_id_for_attempt("wi-child", 1)
child.metadata = {
**dict(child.metadata or {}),
"report_attempt_count": 1,
}
report = DelegationWorkItem(
work_item_id=report_id,
run_id="run-1",
cell_id="team::cto",
team_id="team::cto",
role_id="engineer",
seat_id="seat::team::cto::engineer",
manager_role_id="cto",
manager_seat_id="seat::team::cto::cto",
parent_work_item_id="wi-child",
title="Report #1: Build feature",
summary="Durable worker report.",
kind="report",
projection_id=report_id,
phase=Phase.APPROVED,
batch_index=1,
metadata={
"runtime_model": "multi_team_org",
"work_kind": "report",
"report_execution_work_item": True,
"report_attempt": 1,
"report_target_work_item_id": "wi-child",
"report_card_outcome": "applied",
"completion_report": "Durable worker report.",
"review_evidence": {
"completion_summary": "Durable worker report."
},
},
)
review = _make_review_card(review_card_id=review_id_v1)
review.metadata = {
**dict(review.metadata or {}),
"review_source_report_work_item_id": report_id,
}
await store.save_delegation_work_item(child)
await store.save_delegation_work_item(report)
await store.save_delegation_work_item(review)
original_insert = store.insert_delegation_work_item_if_absent
review_id_v2 = review_work_item_id_for_attempt("wi-child", 2)
injected = False
async def fail_first_retry_card_insert(
item: DelegationWorkItem,
) -> bool:
nonlocal injected
if not injected and item.work_item_id == review_id_v2:
injected = True
raise RuntimeError("injected review retry save failure")
return await original_insert(item)
store.insert_delegation_work_item_if_absent = AsyncMock(
side_effect=fail_first_retry_card_insert
)
review_task = _make_review_task(
review_card_id=review_id_v1,
structured_verdict={"unparseable": True},
)
try:
await executor._finalize_review_work_item(review_task)
finally:
store.insert_delegation_work_item_if_absent = original_insert
self.assertTrue(injected)
child_after_failure = await store.get_delegation_work_item(
"wi-child"
)
failed_review = await store.get_delegation_work_item(review_id_v1)
self.assertEqual(
child_after_failure.phase,
Phase.AWAITING_MANAGER_REVIEW,
)
self.assertFalse(
child_after_failure.metadata.get(
"review_verdict_parse_failed_auto_done", False
)
)
self.assertEqual(failed_review.phase, Phase.CANCELLED)
self.assertEqual(
failed_review.metadata.get("review_work_item_outcome"),
"verdict_parse_failed",
)
self.assertIsNone(
await store.get_delegation_work_item(review_id_v2)
)
run_items = await store.list_delegation_work_items("run-1")
await executor._reconcile_missing_review_chain(run_items)
await executor._reconcile_missing_review_chain(
await store.list_delegation_work_items("run-1")
)
retry = await store.get_delegation_work_item(review_id_v2)
self.assertIsNotNone(retry)
self.assertEqual(retry.phase, Phase.READY)
self.assertEqual(
retry.metadata.get("review_source_report_work_item_id"),
report_id,
)
self.assertEqual(
retry.metadata.get("review_retry_reason"),
"verdict_parse_failed",
)
self.assertIsNone(
await store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
)
finally:
await store.close()
async def test_retry_preserves_review_owner_when_runtime_task_lost_owner_metadata(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
+4 -41
View File
@@ -18,7 +18,6 @@ from __future__ import annotations
import tempfile
import unittest
from datetime import datetime
from pathlib import Path
from unittest.mock import MagicMock
@@ -599,45 +598,6 @@ class DeliveryCardPhaseSyncTests(unittest.IsolatedAsyncioTestCase):
self.assertFalse(any(item.kind == "report" for item in items))
self.assertFalse(any(item.kind == "review" for item in items))
async def test_repair_stuck_aggregate_review_item_unblocks_dependents(self) -> None:
executor = self._executor()
stuck = DelegationWorkItem(
work_item_id="aggregate-stuck",
run_id="r",
cell_id="c",
role_id="cto",
seat_id="seat-cto",
manager_role_id="ceo",
manager_seat_id="seat-ceo",
title="Stuck aggregate",
kind="execute",
phase=Phase.AWAITING_MANAGER_REVIEW,
metadata={"work_kind": "synthesize", "work_item_turn_type": "aggregate"},
)
dependent = DelegationWorkItem(
work_item_id="final-qa",
run_id="r",
cell_id="c",
role_id="coo",
seat_id="seat-coo",
manager_role_id="ceo",
manager_seat_id="seat-ceo",
title="Final QA",
kind="review",
phase=Phase.WAITING_DEPENDENCIES,
metadata={"dependency_work_item_ids": ["aggregate-stuck"]},
)
await self.store.save_delegation_work_item(stuck)
await self.store.save_delegation_work_item(dependent)
work_items = await self.store.list_delegation_work_items("r")
await executor._repair_stuck_aggregate_review_items(work_items)
repaired = await self.store.get_delegation_work_item("aggregate-stuck")
unblocked = await self.store.get_delegation_work_item("final-qa")
self.assertEqual(repaired.phase, Phase.APPROVED)
self.assertEqual(unblocked.phase, Phase.READY)
async def test_stale_report_for_delivery_parent_closes_without_review_card(self) -> None:
executor = self._executor()
parent = DelegationWorkItem(
@@ -697,7 +657,10 @@ class DeliveryCardPhaseSyncTests(unittest.IsolatedAsyncioTestCase):
closed = await self.store.get_delegation_work_item(report.work_item_id)
self.assertEqual(closed.phase, Phase.APPROVED)
self.assertEqual(closed.metadata["report_card_outcome"], "non_reviewable_parent")
self.assertEqual(
closed.metadata["report_card_outcome"],
"parent_not_awaiting_review",
)
items = await self.store.list_delegation_work_items("r")
self.assertFalse(any(str(item.work_item_id).startswith("review::deliver-parent") for item in items))
+962
View File
@@ -32,6 +32,7 @@ from opc.layer2_organization.company_mode import (
review_work_item_id_for_attempt,
)
from opc.layer2_organization.org_engine import OrgEngine
from opc.layer2_organization.phase import DONE_PHASES
from opc.layer2_organization.work_item_links import set_linked_work_item_id
@@ -201,6 +202,10 @@ class WorkerExecuteDoneSpawnsReportTests(unittest.IsolatedAsyncioTestCase):
worker_task = _build_worker_task()
worker_task.metadata = dict(worker_task.metadata or {})
worker_task.metadata["work_kind"] = "dispatch"
# Delegation is attempt-scoped. The child row alone may be
# historical, so mirror the tool's current-turn mutation
# marker instead of asking DONE routing to infer from rows.
worker_task.metadata["manager_board_mutation_performed"] = True
await store.save_task(worker_task)
await executor._apply_done_transition(
@@ -396,6 +401,963 @@ class ReportTurnDoneSpawnsReviewTests(unittest.IsolatedAsyncioTestCase):
await store.close()
class ReviewChainRecoveryTests(unittest.IsolatedAsyncioTestCase):
"""Crash boundaries use persisted auxiliary cards as their journal.
The attempt counters on the parent are only lookup caches: a crash can
happen after an auxiliary card commits but before its counter does. The
report card also has to contain the completed report before review-card
creation is attempted, so reconciliation can resume without a live Task.
"""
async def asyncSetUp(self) -> None:
self._tmpdir = tempfile.TemporaryDirectory()
self.root = Path(self._tmpdir.name)
self.store = OPCStore(self.root / "tasks.db")
await self.store.initialize()
self.executor = _build_executor(self.store, _make_org_engine(self.root))
async def asyncTearDown(self) -> None:
await self.store.close()
self._tmpdir.cleanup()
async def _save_awaiting_parent(self) -> DelegationWorkItem:
parent = _build_child_work_item()
parent.phase = Phase.AWAITING_MANAGER_REVIEW
parent.metadata = {
**dict(parent.metadata or {}),
"review_owner_role_id": "cto",
"review_owner_seat_id": "seat::team::cto::cto",
"completion_report": "execute-turn fallback",
}
await self.store.save_delegation_work_item(parent)
return parent
async def _run_reconcile(self) -> list[DelegationWorkItem]:
items = await self.store.list_delegation_work_items("run-1")
return await self.executor._reconcile_missing_review_chain(items)
async def _auxiliary_cards(self) -> list[DelegationWorkItem]:
return [
item
for item in await self.store.list_delegation_work_items("run-1")
if str((item.metadata or {}).get("report_target_work_item_id", "") or "").strip()
== "wi-child"
or str((item.metadata or {}).get("review_target_work_item_id", "") or "").strip()
== "wi-child"
]
async def _setup_running_report(
self,
) -> tuple[str, Task]:
"""Create the real execute->report handoff and claim its report card."""
await self.store.save_delegation_work_item(_build_child_work_item())
worker_task = _build_worker_task()
await self.executor._apply_done_transition(
worker_task,
result=TaskResult(status=TaskStatus.DONE, content="execute fallback"),
)
report_id = report_work_item_id_for_attempt("wi-child", 1)
role_session_id = "role-runtime::run-1::engineer"
await self.store.update_delegation_work_item(
report_id,
phase=Phase.RUNNING,
claimed_by_role_runtime_session_id=role_session_id,
claimed_by_seat_id="seat::team::cto::engineer",
metadata_updates={
"claimed_by_role_session_id": role_session_id,
"claimed_task_id": "task-report-1",
},
)
report_task = ReportTurnDoneSpawnsReviewTests()._report_turn_task(
report_card_id=report_id,
target_work_item_id="wi-child",
)
return report_id, report_task
async def _setup_running_review(
self,
*,
verdict: dict[str, object],
) -> tuple[str, str, Task]:
"""Create and claim a review card using the production report path."""
report_id, report_task = await self._setup_running_report()
await self.executor._apply_report_done_transition(
report_task,
result=TaskResult(status=TaskStatus.DONE, content="durable report v1"),
)
review_id = review_work_item_id_for_attempt("wi-child", 1)
role_session_id = "role-runtime::run-1::cto"
await self.store.update_delegation_work_item(
review_id,
phase=Phase.RUNNING,
claimed_by_role_runtime_session_id=role_session_id,
claimed_by_seat_id="seat::team::cto::cto",
metadata_updates={
"claimed_by_role_session_id": role_session_id,
"claimed_task_id": "task-review-1",
},
)
review_item = await self.store.get_delegation_work_item(review_id)
self.assertIsNotNone(review_item)
review_task = Task(
id="task-review-1",
title="Review #1: Build feature",
project_id="proj1",
session_id="session-cto",
parent_session_id="session-root",
assigned_to="cto",
status=TaskStatus.DONE,
metadata={
**dict(review_item.metadata or {}),
"execution_mode": "company_mode",
"runtime_model": "multi_team_org",
"delegation_run_id": "run-1",
"work_item_runtime": True,
"review_execution_work_item": True,
"structured_review_verdict": verdict,
},
)
set_linked_work_item_id(review_task, review_id)
return report_id, review_id, review_task
async def test_insert_if_absent_preserves_claimed_deterministic_aux_card(self) -> None:
cases = (
(
"report",
report_work_item_id_for_attempt("wi-child", 1),
"engineer",
"seat::team::cto::engineer",
{"report_target_work_item_id": "wi-child", "report_attempt": 1},
),
(
"review",
review_work_item_id_for_attempt("wi-child", 1),
"cto",
"seat::team::cto::cto",
{"review_target_work_item_id": "wi-child", "review_attempt": 1},
),
)
for kind, work_item_id, role_id, seat_id, target_metadata in cases:
with self.subTest(kind=kind):
ready = DelegationWorkItem(
work_item_id=work_item_id,
run_id="run-1",
cell_id="team::cto",
team_id="team::cto",
role_id=role_id,
seat_id=seat_id,
parent_work_item_id="wi-child",
title=f"{kind.title()} attempt 1",
kind=kind,
projection_id=work_item_id,
phase=Phase.READY,
batch_index=1,
metadata={
"work_kind": kind,
**target_metadata,
"persisted_sentinel": f"original-{kind}",
},
)
self.assertTrue(
await self.store.insert_delegation_work_item_if_absent(ready)
)
role_session_id = f"role-runtime::run-1::{role_id}"
claimed_task_id = f"task-{kind}-1"
await self.store.update_delegation_work_item(
work_item_id,
phase=Phase.RUNNING,
claimed_by_role_runtime_session_id=role_session_id,
claimed_by_seat_id=seat_id,
metadata_updates={
"claimed_by_role_session_id": role_session_id,
"claimed_task_id": claimed_task_id,
},
)
competing_ready = DelegationWorkItem(
work_item_id=work_item_id,
run_id="run-1",
cell_id="team::cto",
team_id="team::cto",
role_id=role_id,
seat_id=seat_id,
parent_work_item_id="wi-child",
title=f"Competing {kind} attempt 1",
kind=kind,
projection_id=work_item_id,
phase=Phase.READY,
batch_index=1,
metadata={
"work_kind": kind,
**target_metadata,
"persisted_sentinel": f"competing-{kind}",
},
)
inserted = await self.store.insert_delegation_work_item_if_absent(
competing_ready
)
persisted = await self.store.get_delegation_work_item(work_item_id)
self.assertFalse(inserted)
self.assertIsNotNone(persisted)
self.assertEqual(persisted.phase, Phase.RUNNING)
self.assertEqual(
persisted.claimed_by_role_runtime_session_id,
role_session_id,
)
self.assertEqual(persisted.claimed_by_seat_id, seat_id)
self.assertEqual(
persisted.metadata.get("claimed_by_role_session_id"),
role_session_id,
)
self.assertEqual(
persisted.metadata.get("claimed_task_id"), claimed_task_id
)
self.assertEqual(
persisted.metadata.get("persisted_sentinel"),
f"original-{kind}",
)
async def test_report_terminal_write_failure_releases_claim_for_retry(self) -> None:
report_id, report_task = await self._setup_running_report()
original_update = self.store.update_delegation_work_item
injected = False
async def fail_first_terminal_report_write(work_item_id: str, **kwargs):
nonlocal injected
metadata_updates = dict(kwargs.get("metadata_updates") or {})
if (
not injected
and work_item_id == report_id
and kwargs.get("phase") == Phase.APPROVED
and metadata_updates.get("report_card_outcome") == "applied"
):
injected = True
raise RuntimeError("injected terminal report journal failure")
return await original_update(work_item_id, **kwargs)
self.store.update_delegation_work_item = AsyncMock(
side_effect=fail_first_terminal_report_write
)
try:
await self.executor._apply_report_done_transition(
report_task,
result=TaskResult(status=TaskStatus.DONE, content="volatile report"),
)
finally:
self.store.update_delegation_work_item = original_update
self.assertTrue(injected)
parent = await self.store.get_delegation_work_item("wi-child")
report = await self.store.get_delegation_work_item(report_id)
self.assertEqual(parent.phase, Phase.AWAITING_MANAGER_REVIEW)
self.assertEqual(report.phase, Phase.RUNNING)
self.assertEqual(report.claimed_by_role_runtime_session_id, "")
self.assertEqual(report.claimed_by_seat_id, "")
self.assertEqual(report.metadata.get("claimed_by_role_session_id"), "")
self.assertEqual(report.metadata.get("claimed_task_id"), "")
self.assertTrue(
CompanyWorkItemExecutor._work_item_is_runnable(
report,
{"wi-child": parent, report_id: report},
)
)
self.assertIsNone(
await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 1)
)
)
async def test_review_terminal_write_failure_keeps_parent_reviewable(self) -> None:
report_id, review_id, review_task = await self._setup_running_review(
verdict={
"label": "reject",
"summary": "needs rework",
"blocking_issues": ["fix the defect"],
"followups": [],
}
)
original_update = self.store.update_delegation_work_item
injected = False
async def fail_first_terminal_review_write(work_item_id: str, **kwargs):
nonlocal injected
if (
not injected
and work_item_id == review_id
and kwargs.get("phase") in DONE_PHASES
):
injected = True
raise RuntimeError("injected terminal review journal failure")
return await original_update(work_item_id, **kwargs)
self.store.update_delegation_work_item = AsyncMock(
side_effect=fail_first_terminal_review_write
)
try:
await self.executor._finalize_review_work_item(review_task)
finally:
self.store.update_delegation_work_item = original_update
self.assertTrue(injected)
parent = await self.store.get_delegation_work_item("wi-child")
review = await self.store.get_delegation_work_item(review_id)
self.assertEqual(parent.phase, Phase.AWAITING_MANAGER_REVIEW)
self.assertEqual(review.phase, Phase.RUNNING)
self.assertEqual(review.claimed_by_role_runtime_session_id, "")
self.assertEqual(review.claimed_by_seat_id, "")
self.assertEqual(review.metadata.get("claimed_by_role_session_id"), "")
self.assertEqual(review.metadata.get("claimed_task_id"), "")
self.assertIsNone(
await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
)
self.assertIsNone(
await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 2)
)
)
self.assertEqual(
review.metadata.get("review_source_report_work_item_id"),
report_id,
)
async def test_late_review_cannot_override_parent_awaiting_human(self) -> None:
_report_id, review_id, review_task = await self._setup_running_review(
verdict={
"label": "approve",
"summary": "approve from a now-stale manager turn",
"blocking_issues": [],
"followups": [],
}
)
await self.store.update_delegation_work_item(
"wi-child",
phase=Phase.AWAITING_HUMAN,
metadata_updates={"human_checkpoint_sentinel": "must-survive"},
)
await self.executor._finalize_review_work_item(review_task)
parent = await self.store.get_delegation_work_item("wi-child")
review = await self.store.get_delegation_work_item(review_id)
self.assertEqual(parent.phase, Phase.AWAITING_HUMAN)
self.assertEqual(
parent.metadata.get("human_checkpoint_sentinel"), "must-survive"
)
self.assertNotIn("review_resolution_applied_work_item_id", parent.metadata)
self.assertNotIn("structured_review_verdict", parent.metadata)
self.assertNotIn("reviewed_at", parent.metadata)
self.assertIn(review.phase, DONE_PHASES)
self.assertEqual(
review.metadata.get("review_work_item_outcome"),
"target_no_longer_awaiting_manager_review",
)
self.assertNotEqual(
review.metadata.get("review_resolution_state"), "applied"
)
self.assertNotIn("review_resolution", review.metadata)
self.assertEqual(review.claimed_by_role_runtime_session_id, "")
self.assertEqual(review.claimed_by_seat_id, "")
async def test_late_review_is_stale_when_newer_applied_report_exists(self) -> None:
report_v1_id, review_v1_id, review_v1_task = (
await self._setup_running_review(
verdict={
"label": "approve",
"summary": "approval based on report v1",
"blocking_issues": [],
"followups": [],
}
)
)
report_v2_id = report_work_item_id_for_attempt("wi-child", 2)
report_v2 = DelegationWorkItem(
work_item_id=report_v2_id,
run_id="run-1",
cell_id="team::cto",
team_id="team::cto",
role_id="engineer",
seat_id="seat::team::cto::engineer",
manager_role_id="cto",
manager_seat_id="seat::team::cto::cto",
parent_work_item_id="wi-child",
title="Report attempt 2",
summary="Newer durable handoff.",
kind="report",
projection_id=report_v2_id,
phase=Phase.APPROVED,
batch_index=2,
metadata={
"runtime_model": "multi_team_org",
"work_kind": "report",
"report_execution_work_item": True,
"report_target_work_item_id": "wi-child",
"report_attempt": 2,
"report_card_outcome": "applied",
"completion_report": "authoritative report v2",
},
)
await self.store.save_delegation_work_item(report_v2)
await self.store.update_delegation_work_item(
"wi-child",
metadata_updates={
"completion_report": "authoritative report v2",
"newer_report_sentinel": "must-survive",
},
)
await self.executor._finalize_review_work_item(review_v1_task)
parent = await self.store.get_delegation_work_item("wi-child")
review_v1 = await self.store.get_delegation_work_item(review_v1_id)
self.assertEqual(parent.phase, Phase.AWAITING_MANAGER_REVIEW)
self.assertEqual(parent.metadata.get("completion_report"), "authoritative report v2")
self.assertEqual(parent.metadata.get("newer_report_sentinel"), "must-survive")
self.assertNotIn("review_resolution_applied_work_item_id", parent.metadata)
self.assertNotIn("structured_review_verdict", parent.metadata)
self.assertNotIn("reviewed_at", parent.metadata)
self.assertIn(review_v1.phase, DONE_PHASES)
self.assertEqual(
review_v1.metadata.get("review_source_report_work_item_id"),
report_v1_id,
)
self.assertEqual(review_v1.metadata.get("review_resolution_state"), "stale")
self.assertEqual(
review_v1.metadata.get("review_resolution_stale_reason"),
"source_report_superseded",
)
self.assertEqual(
review_v1.metadata.get("review_work_item_outcome"),
"superseded_by_newer_report",
)
self.assertEqual(
(review_v1.metadata.get("review_resolution") or {}).get(
"source_report_work_item_id"
),
report_v1_id,
)
async def test_reconcile_replays_terminal_review_after_parent_write_failure(
self,
) -> None:
report_id, review_id, review_task = await self._setup_running_review(
verdict={
"label": "reject",
"summary": "needs rework",
"blocking_issues": ["fix the defect"],
"followups": [],
}
)
original_apply = self.store.apply_delegation_review_resolution
injected = False
async def fail_first_parent_projection(work_item_id: str, **kwargs):
nonlocal injected
if not injected and work_item_id == "wi-child":
injected = True
raise RuntimeError("injected child verdict projection failure")
return await original_apply(work_item_id, **kwargs)
self.store.apply_delegation_review_resolution = AsyncMock(
side_effect=fail_first_parent_projection
)
try:
await self.executor._finalize_review_work_item(review_task)
finally:
self.store.apply_delegation_review_resolution = original_apply
self.assertTrue(injected)
parent_before = await self.store.get_delegation_work_item("wi-child")
review_before = await self.store.get_delegation_work_item(review_id)
self.assertEqual(parent_before.phase, Phase.AWAITING_MANAGER_REVIEW)
self.assertIn(review_before.phase, DONE_PHASES)
self.assertEqual(
review_before.metadata.get("review_source_report_work_item_id"),
report_id,
)
await self._run_reconcile()
await self._run_reconcile()
parent_after = await self.store.get_delegation_work_item("wi-child")
self.assertEqual(parent_after.phase, Phase.READY_FOR_REWORK)
self.assertEqual(
(parent_after.metadata.get("structured_review_verdict") or {}).get(
"label"
),
"reject",
)
self.assertEqual(parent_after.metadata.get("review_rework_count"), 1)
self.assertIn(
"fix the defect",
str(parent_after.metadata.get("rework_feedback", "")),
)
self.assertEqual(
parent_after.metadata.get("review_resolution_applied_work_item_id"),
review_id,
)
report_cards = [
item for item in await self._auxiliary_cards() if item.kind == "report"
]
review_cards = [
item for item in await self._auxiliary_cards() if item.kind == "review"
]
self.assertEqual([item.work_item_id for item in report_cards], [report_id])
self.assertEqual([item.work_item_id for item in review_cards], [review_id])
# A later worker attempt re-enters review with the old journal still
# present. The atomic applied stamp must make that verdict immutable
# history, not a resolution to replay onto the new output.
await self.store.update_delegation_work_item(
"wi-child",
phase=Phase.RUNNING,
)
await self.store.update_delegation_work_item(
"wi-child",
phase=Phase.AWAITING_MANAGER_REVIEW,
)
await self._run_reconcile()
next_cycle_parent = await self.store.get_delegation_work_item("wi-child")
self.assertEqual(next_cycle_parent.phase, Phase.AWAITING_MANAGER_REVIEW)
next_report = await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
self.assertIsNotNone(next_report)
self.assertEqual(next_report.phase, Phase.READY)
self.assertIsNone(
await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 2)
)
)
async def test_report_card_commit_survives_parent_counter_failure(self) -> None:
await self._save_awaiting_parent()
worker_task = _build_worker_task()
original_update = self.store.update_delegation_work_item
async def fail_report_counter(work_item_id: str, **kwargs):
metadata_updates = dict(kwargs.get("metadata_updates") or {})
if work_item_id == "wi-child" and "report_attempt_count" in metadata_updates:
raise RuntimeError("injected crash after report-card commit")
return await original_update(work_item_id, **kwargs)
self.store.update_delegation_work_item = AsyncMock(side_effect=fail_report_counter)
first = await self.executor._ensure_report_work_item_for_work_item(
"wi-child", worker_task=worker_task
)
self.assertIsNotNone(first)
self.assertEqual(first.work_item_id, report_work_item_id_for_attempt("wi-child", 1))
parent = await self.store.get_delegation_work_item("wi-child")
self.assertNotIn("report_attempt_count", parent.metadata or {})
# Make a re-save of v1 observably wrong: retry must discover and
# return the persisted RUNNING card instead of trying READY again.
await original_update(first.work_item_id, phase=Phase.RUNNING)
second = await self.executor._ensure_report_work_item_for_work_item(
"wi-child", worker_task=worker_task
)
self.assertIsNotNone(second)
self.assertEqual(second.work_item_id, first.work_item_id)
self.assertEqual(second.phase, Phase.RUNNING)
self.assertIsNone(
await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
)
async def test_review_card_commit_survives_parent_counter_failure(self) -> None:
await self._save_awaiting_parent()
worker_task = _build_worker_task()
original_update = self.store.update_delegation_work_item
async def fail_review_counter(work_item_id: str, **kwargs):
metadata_updates = dict(kwargs.get("metadata_updates") or {})
if work_item_id == "wi-child" and "review_attempt_count" in metadata_updates:
raise RuntimeError("injected crash after review-card commit")
return await original_update(work_item_id, **kwargs)
self.store.update_delegation_work_item = AsyncMock(side_effect=fail_review_counter)
first = await self.executor._ensure_review_work_item_for_work_item(
"wi-child",
worker_task=worker_task,
completion_report="handoff",
metadata_updates={
"review_owner_role_id": "cto",
"review_owner_seat_id": "seat::team::cto::cto",
},
)
self.assertIsNotNone(first)
self.assertEqual(first.work_item_id, review_work_item_id_for_attempt("wi-child", 1))
parent = await self.store.get_delegation_work_item("wi-child")
self.assertNotIn("review_attempt_count", parent.metadata or {})
await original_update(first.work_item_id, phase=Phase.RUNNING)
second = await self.executor._ensure_review_work_item_for_work_item(
"wi-child",
worker_task=worker_task,
completion_report="handoff",
metadata_updates={
"review_owner_role_id": "cto",
"review_owner_seat_id": "seat::team::cto::cto",
},
)
self.assertIsNotNone(second)
self.assertEqual(second.work_item_id, first.work_item_id)
self.assertEqual(second.phase, Phase.RUNNING)
self.assertIsNone(
await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 2)
)
)
async def test_reconcile_recovers_review_from_terminal_report(self) -> None:
# Build the real report turn first so this exercises the exact durable
# payload written at the report-DONE crash boundary.
await self.store.save_delegation_work_item(_build_child_work_item())
worker_task = _build_worker_task()
await self.executor._apply_done_transition(
worker_task,
result=TaskResult(status=TaskStatus.DONE, content="execute fallback"),
)
report_id = report_work_item_id_for_attempt("wi-child", 1)
await self.store.update_delegation_work_item(report_id, phase=Phase.RUNNING)
report_task = ReportTurnDoneSpawnsReviewTests()._report_turn_task(
report_card_id=report_id,
target_work_item_id="wi-child",
)
report_payload = (
'{"summary":"durable handoff","deliverables":[],"risks":[],"next_actions":[]}'
)
original_insert = self.store.insert_delegation_work_item_if_absent
async def fail_review_insert(
item: DelegationWorkItem,
) -> bool:
if item.kind == "review":
raise RuntimeError("injected crash while saving review card")
return await original_insert(item)
self.store.insert_delegation_work_item_if_absent = AsyncMock(
side_effect=fail_review_insert
)
await self.executor._apply_report_done_transition(
report_task,
result=TaskResult(status=TaskStatus.DONE, content=report_payload),
)
terminal_report = await self.store.get_delegation_work_item(report_id)
self.assertEqual(terminal_report.phase, Phase.APPROVED)
self.assertEqual(terminal_report.metadata.get("report_card_outcome"), "applied")
self.assertEqual(terminal_report.metadata.get("completion_report"), report_payload)
self.assertEqual(terminal_report.metadata.get("report_completion_raw"), report_payload)
self.assertTrue(terminal_report.metadata.get("review_evidence"))
self.assertIsNone(
await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 1)
)
)
# Simulate restart: there is no runtime Task available to carry the
# payload, only the parent and terminal report rows.
self.store.insert_delegation_work_item_if_absent = original_insert
self.assertEqual(await self.store.get_tasks(), [])
await self._run_reconcile()
await self._run_reconcile()
review = await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 1)
)
self.assertIsNotNone(review)
self.assertEqual(review.metadata.get("review_completion_report"), report_payload)
self.assertEqual(
review.metadata.get("review_source_report_work_item_id"),
report_id,
)
self.assertEqual(
(review.metadata.get("review_evidence") or {}).get("completion_summary"),
report_payload,
)
reports = [item for item in await self._auxiliary_cards() if item.kind == "report"]
self.assertEqual([item.work_item_id for item in reports], [report_id])
self.assertIsNone(
await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
)
async def test_reconcile_repairs_parent_after_terminal_report_projection_failure(
self,
) -> None:
await self.store.save_delegation_work_item(_build_child_work_item())
worker_task = _build_worker_task()
await self.executor._apply_done_transition(
worker_task,
result=TaskResult(status=TaskStatus.DONE, content="execute fallback"),
)
report_id = report_work_item_id_for_attempt("wi-child", 1)
await self.store.update_delegation_work_item(report_id, phase=Phase.RUNNING)
report_task = ReportTurnDoneSpawnsReviewTests()._report_turn_task(
report_card_id=report_id,
target_work_item_id="wi-child",
)
report_payload = "Report payload committed before the parent projection."
original_update = self.store.update_delegation_work_item
async def fail_parent_projection(work_item_id: str, **kwargs):
metadata_updates = dict(kwargs.get("metadata_updates") or {})
if (
work_item_id == "wi-child"
and metadata_updates.get("completion_report") == report_payload
):
raise RuntimeError("injected crash while projecting report to parent")
return await original_update(work_item_id, **kwargs)
self.store.update_delegation_work_item = AsyncMock(
side_effect=fail_parent_projection
)
await self.executor._apply_report_done_transition(
report_task,
result=TaskResult(status=TaskStatus.DONE, content=report_payload),
)
terminal_report = await self.store.get_delegation_work_item(report_id)
self.assertEqual(terminal_report.phase, Phase.APPROVED)
self.assertEqual(terminal_report.metadata.get("report_card_outcome"), "applied")
self.assertEqual(terminal_report.metadata.get("completion_report"), report_payload)
parent_before_reconcile = await self.store.get_delegation_work_item("wi-child")
self.assertNotEqual(
(parent_before_reconcile.metadata or {}).get("completion_report"),
report_payload,
)
self.assertIsNone(
await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 1)
)
)
self.store.update_delegation_work_item = original_update
await self._run_reconcile()
await self._run_reconcile()
parent_after_reconcile = await self.store.get_delegation_work_item("wi-child")
self.assertEqual(
parent_after_reconcile.metadata.get("completion_report"),
report_payload,
)
review = await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 1)
)
self.assertIsNotNone(review)
self.assertEqual(review.metadata.get("review_completion_report"), report_payload)
self.assertEqual(
review.metadata.get("review_source_report_work_item_id"),
report_id,
)
self.assertIsNone(
await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
)
async def test_durable_v2_overrides_lagging_attempt_counters(self) -> None:
await self._save_awaiting_parent()
await self.store.update_delegation_work_item(
"wi-child",
metadata_updates={
"report_attempt_count": 1,
"review_attempt_count": 1,
},
)
terminal_report = DelegationWorkItem(
work_item_id=report_work_item_id_for_attempt("wi-child", 2),
run_id="run-1",
cell_id="team::cto",
team_id="team::cto",
role_id="engineer",
seat_id="seat::team::cto::engineer",
manager_role_id="cto",
manager_seat_id="seat::team::cto::cto",
parent_work_item_id="wi-child",
title="Terminal report v2",
summary="Immutable report history.",
kind="report",
projection_id=report_work_item_id_for_attempt("wi-child", 2),
phase=Phase.APPROVED,
batch_index=2,
metadata={
"runtime_model": "multi_team_org",
"work_kind": "report",
"report_execution_work_item": True,
"report_target_work_item_id": "wi-child",
"report_attempt": 2,
"report_card_outcome": "applied",
"completion_report": "Immutable report history.",
"history_sentinel": "report-v2-must-not-change",
},
)
terminal_review = DelegationWorkItem(
work_item_id=review_work_item_id_for_attempt("wi-child", 2),
run_id="run-1",
cell_id="team::cto",
team_id="team::cto",
role_id="cto",
seat_id="seat::team::cto::cto",
manager_role_id="ceo",
manager_seat_id="seat::team::cto::ceo",
parent_work_item_id="wi-child",
title="Terminal review v2",
summary="Immutable review history.",
kind="review",
projection_id=review_work_item_id_for_attempt("wi-child", 2),
phase=Phase.APPROVED,
batch_index=2,
metadata={
"runtime_model": "multi_team_org",
"work_kind": "review",
"review_execution_work_item": True,
"review_target_work_item_id": "wi-child",
"review_attempt": 2,
"review_work_item_outcome": "approved",
"history_sentinel": "review-v2-must-not-change",
},
)
await self.store.save_delegation_work_item(terminal_report)
await self.store.save_delegation_work_item(terminal_review)
worker_task = _build_worker_task()
new_report = await self.executor._ensure_report_work_item_for_work_item(
"wi-child",
worker_task=worker_task,
)
self.assertIsNotNone(new_report)
self.assertEqual(
new_report.work_item_id,
report_work_item_id_for_attempt("wi-child", 3),
)
# Review and report auxiliaries must never be active in parallel.
self.assertIsNone(
await self.executor._ensure_review_work_item_for_work_item(
"wi-child",
worker_task=worker_task,
completion_report="new completion",
metadata_updates={
"review_owner_role_id": "cto",
"review_owner_seat_id": "seat::team::cto::cto",
},
source_report_item=terminal_report,
)
)
await self.store.update_delegation_work_item(
new_report.work_item_id,
phase=Phase.CANCELLED,
)
new_review = await self.executor._ensure_review_work_item_for_work_item(
"wi-child",
worker_task=worker_task,
completion_report="new completion",
metadata_updates={
"review_owner_role_id": "cto",
"review_owner_seat_id": "seat::team::cto::cto",
},
source_report_item=terminal_report,
)
self.assertIsNotNone(new_review)
self.assertEqual(
new_review.work_item_id,
review_work_item_id_for_attempt("wi-child", 3),
)
persisted_report_v2 = await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
persisted_review_v2 = await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 2)
)
self.assertEqual(persisted_report_v2.phase, Phase.APPROVED)
self.assertEqual(
persisted_report_v2.metadata.get("history_sentinel"),
"report-v2-must-not-change",
)
self.assertEqual(persisted_review_v2.phase, Phase.APPROVED)
self.assertEqual(
persisted_review_v2.metadata.get("history_sentinel"),
"review-v2-must-not-change",
)
async def test_reconcile_without_runtime_task_creates_report(self) -> None:
await self._save_awaiting_parent()
self.assertEqual(await self.store.get_tasks(), [])
await self._run_reconcile()
report = await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 1)
)
self.assertIsNotNone(report)
self.assertEqual(report.phase, Phase.READY)
self.assertEqual(report.metadata.get("report_target_work_item_id"), "wi-child")
self.assertEqual(report.role_id, "engineer")
self.assertEqual(report.seat_id, "seat::team::cto::engineer")
async def test_repeated_reconcile_keeps_at_most_one_active_auxiliary(self) -> None:
await self._save_awaiting_parent()
for _ in range(5):
await self._run_reconcile()
auxiliaries = await self._auxiliary_cards()
active = [item for item in auxiliaries if item.phase not in DONE_PHASES]
self.assertEqual(len(active), 1)
self.assertEqual(active[0].kind, "report")
self.assertEqual(active[0].work_item_id, report_work_item_id_for_attempt("wi-child", 1))
self.assertIsNone(
await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
)
async def test_reconcile_remains_idempotent_after_database_reopen(self) -> None:
await self._save_awaiting_parent()
await self._run_reconcile()
await self.store.close()
self.store = OPCStore(self.root / "tasks.db")
await self.store.initialize()
self.executor = _build_executor(self.store, _make_org_engine(self.root))
for _ in range(3):
await self._run_reconcile()
auxiliaries = await self._auxiliary_cards()
active = [item for item in auxiliaries if item.phase not in DONE_PHASES]
self.assertEqual(len(active), 1)
self.assertEqual(
active[0].work_item_id,
report_work_item_id_for_attempt("wi-child", 1),
)
self.assertIsNone(
await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
)
class ReportCardRunnableFilterTests(unittest.TestCase):
"""Pin the dispatcher-runnability filters for report cards.