Gitinbox/OpenOPC
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(company): make delegation review lifecycle durable

Browse Source
This commit is contained in:
LZH-YS1998
2026-07-14 10:52:22 +08:00
parent 4bc18dcd27
commit 5e02364eb4
19 changed files with 3150 additions and 1087 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_actor_runtime_company_mode.py
+253 -34
View File
@@ -967,6 +967,7 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
)
self.assertEqual(issues, [])
self.assertTrue(self.task.metadata.get("manager_board_mutation_performed"))
async def test_manager_dispatch_guard_accepts_existing_child_mutation(self) -> None:
await self.store.save_delegation_work_item(
@@ -1012,6 +1013,114 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
)
self.assertEqual(issues, [])
self.assertTrue(self.task.metadata.get("manager_board_mutation_performed"))
async def test_manager_dispatch_guard_accepts_existing_child_deletion(self) -> None:
await self.store.save_delegation_work_item(
DelegationWorkItem(
work_item_id="cto-child-item",
run_id="run-1",
cell_id="team::ceo",
team_instance_id="team-instance::run-1::team::ceo",
team_id="team::ceo",
role_id="cto",
seat_id="seat::team::ceo::cto",
seat_state_id="seat-state::run-1::seat::team::ceo::cto",
role_runtime_session_id="role-runtime::run-1::seat::team::ceo::cto",
parent_work_item_id="ceo-work-item",
title="Obsolete CTO Work",
summary="Remove this obsolete branch.",
kind="execute",
projection_id="cto-child-item",
phase=Phase.RUNNING,
manager_role_id="ceo",
manager_seat_id="seat::team::ceo::ceo",
metadata={
"work_item_runtime": True,
"runtime_model": "multi_team_org",
"manager_mutation_revision": 0,
},
)
)
before = await self.executor._snapshot_manager_dispatch_state(self.task)
await self.store.amend_delegation_work_item(
"cto-child-item",
metadata_set={
"manager_mutation_revision": 1,
"manager_mutation_action": "delete",
"deleted_by_manager_tool": True,
"hidden_from_company_kanban": True,
"upstream_visibility": "hidden",
},
)
issues = await self.executor._enforce_manager_dispatch_guard(
self.task,
TaskResult(status=TaskStatus.DONE, content="Deleted the obsolete child work item."),
before_state=before,
)
self.assertEqual(issues, [])
self.assertTrue(self.task.metadata.get("manager_board_mutation_performed"))
async def test_historical_dependency_does_not_satisfy_current_turn_guard(self) -> None:
await self.store.save_delegation_work_item(
DelegationWorkItem(
work_item_id="historical-child",
run_id="run-1",
parent_work_item_id="ceo-work-item",
role_id="cto",
seat_id="seat::team::ceo::cto",
title="Historical child",
summary="Created in an earlier turn.",
kind="execute",
projection_id="historical-child",
phase=Phase.APPROVED,
metadata={"work_item_runtime": True, "runtime_model": "multi_team_org"},
)
)
self.task.metadata["delegation_wait_for_work_item_ids"] = ["historical-child"]
before = await self.executor._snapshot_manager_dispatch_state(self.task)
issues = await self.executor._enforce_manager_dispatch_guard(
self.task,
TaskResult(status=TaskStatus.DONE, content="Handled this new request directly."),
before_state=before,
)
self.assertEqual(len(issues), 1)
self.assertFalse(self.task.metadata.get("manager_board_mutation_performed", False))
async def test_attention_child_created_during_turn_does_not_satisfy_dispatch_guard(self) -> None:
before = await self.executor._snapshot_manager_dispatch_state(self.task)
await self.store.save_delegation_work_item(
DelegationWorkItem(
work_item_id="ceo-attention-item",
run_id="run-1",
parent_work_item_id="ceo-work-item",
role_id="ceo",
seat_id="seat::team::ceo::ceo",
title="CEO attention",
summary="Runtime wake-up wrapper, not delegated business work.",
kind="monitor",
projection_id="ceo-attention-item",
phase=Phase.READY,
metadata={
"work_item_runtime": True,
"runtime_model": "multi_team_org",
"attention_work_item": True,
},
)
)
issues = await self.executor._enforce_manager_dispatch_guard(
self.task,
TaskResult(status=TaskStatus.DONE, content="Handled this request directly."),
before_state=before,
)
self.assertEqual(len(issues), 1)
self.assertFalse(self.task.metadata.get("manager_board_mutation_performed", False))
async def test_manager_dispatch_guard_exhaustion_accepts_turn_instead_of_failing(self) -> None:
# Dispatch is a soft constraint: when the guard reminders run out,
@@ -1029,6 +1138,11 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
# per-turn reset must clear it, otherwise the guard would
# silently pass without reminders.
"manager_board_mutation_performed": True,
"manager_board_modified_work_item_ids": ["stale-modified-child"],
"manager_board_deleted_work_item_ids": ["stale-deleted-child"],
"manager_no_delegation_justification": "stale reason from an earlier turn",
"no_delegation_justification": "another stale reason",
"manager_dispatch_guard_unresolved": "stale unresolved guard",
},
)
task.status = TaskStatus.PENDING
@@ -1054,13 +1168,22 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
# card waits for CEO review and a live report card drives it.
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
self.assertEqual(work_item.phase, Phase.AWAITING_MANAGER_REVIEW)
self.assertEqual(
str((work_item.metadata or {}).get("turn_output_kind", "")), "self_produced"
)
self.assertEqual(
str((work_item.metadata or {}).get("turn_output_source", "")),
"dispatch_guard_exhausted",
self.assertNotIn("manager_board_mutation_performed", task.metadata)
self.assertNotIn("manager_board_modified_work_item_ids", task.metadata)
self.assertNotIn("manager_board_deleted_work_item_ids", task.metadata)
self.assertNotIn("manager_no_delegation_justification", task.metadata)
self.assertNotIn("no_delegation_justification", task.metadata)
self.assertNotIn("stale unresolved guard", task.metadata["manager_dispatch_guard_unresolved"])
dispatch_evidence = dict(
dict((work_item.metadata or {}).get("review_evidence", {}) or {}).get(
"manager_dispatch", {}
)
or {}
)
self.assertEqual(dispatch_evidence.get("outcome"), "self_produced")
self.assertEqual(dispatch_evidence.get("source"), "dispatch_guard_exhausted")
self.assertNotIn("turn_output_kind", work_item.metadata or {})
self.assertNotIn("turn_output_source", work_item.metadata or {})
report_cards = await self._aux_cards_targeting("cto-dispatch-item", "report_target_work_item_id")
self.assertEqual(len(report_cards), 1)
self.assertNotIn(report_cards[0].phase, DONE_PHASES)
@@ -1131,15 +1254,29 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(phase, Phase.AWAITING_MANAGER_REVIEW)
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
self.assertEqual(work_item.phase, Phase.AWAITING_MANAGER_REVIEW)
self.assertEqual(
str((work_item.metadata or {}).get("turn_output_source", "")), "justified"
dispatch_evidence = dict(
dict((work_item.metadata or {}).get("review_evidence", {}) or {}).get(
"manager_dispatch", {}
)
or {}
)
self.assertEqual(dispatch_evidence.get("outcome"), "self_produced")
self.assertEqual(dispatch_evidence.get("source"), "justified")
self.assertEqual(dispatch_evidence.get("note"), "single-seat scoping decision")
self.assertNotIn("turn_output_kind", work_item.metadata or {})
self.assertNotIn("turn_output_source", work_item.metadata or {})
report_cards = await self._aux_cards_targeting("cto-dispatch-item", "report_target_work_item_id")
self.assertEqual(len(report_cards), 1)
report_card = report_cards[0]
self.assertNotIn(report_card.phase, DONE_PHASES)
# Drive the report turn to completion — the review card must appear.
# Production dispatch claims READY → RUNNING before materializing
# the Task; this direct lifecycle test must model that claim.
await self.store.update_delegation_work_item(
report_card.work_item_id,
phase=Phase.RUNNING,
)
report_task = Task(
id="cto-report-task",
title=report_card.title,
@@ -1165,16 +1302,26 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(len(review_cards), 1)
self.assertNotIn(review_cards[0].phase, DONE_PHASES)
self.assertEqual(str(review_cards[0].role_id or ""), "ceo")
review_dispatch_evidence = dict(
dict((review_cards[0].metadata or {}).get("review_evidence", {}) or {}).get(
"manager_dispatch", {}
)
or {}
)
self.assertEqual(review_dispatch_evidence.get("source"), "justified")
async def test_dispatch_turn_that_delegated_keeps_auto_approve(self) -> None:
# Normal delegation flow: a live child card exists in the store, so
# the dispatch exemption applies (children carry the reviewable
# output) — classification comes from store ground truth, not from
# transient task markers.
task = await self._make_cto_dispatch_task()
async def test_historical_business_child_does_not_exempt_current_self_output(self) -> None:
# A child created by a previous attempt is durable board state, not
# proof that this completion delegated its output. The current direct
# work product still needs the manager review chain.
task = await self._make_cto_dispatch_task(
metadata_extra={
"manager_no_delegation_justification": "This new request is a direct architecture decision."
}
)
await self.store.save_delegation_work_item(
DelegationWorkItem(
work_item_id="cto-delegated-child",
work_item_id="cto-historical-child",
run_id="run-1",
cell_id="team::ceo",
team_instance_id="team-instance::run-1::team::ceo",
@@ -1184,26 +1331,93 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
seat_state_id="seat-state::run-1::seat::team::ceo::cto",
role_runtime_session_id="role-runtime::run-1::seat::team::ceo::cto",
parent_work_item_id="cto-dispatch-item",
title="Delegated child",
summary="Build the feature.",
title="Historical delegated child",
summary="Completed in a previous attempt.",
kind="execute",
projection_id="cto-delegated-child",
phase=Phase.READY,
projection_id="cto-historical-child",
phase=Phase.APPROVED,
manager_role_id="cto",
manager_seat_id="seat::team::ceo::cto",
metadata={"work_item_runtime": True, "runtime_model": "multi_team_org"},
)
)
phase = await self.executor._apply_done_transition(
task,
result=TaskResult(
status=TaskStatus.DONE,
content="Made the new architecture decision directly.",
),
)
self.assertEqual(phase, Phase.AWAITING_MANAGER_REVIEW)
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
report_cards = await self._aux_cards_targeting("cto-dispatch-item", "report_target_work_item_id")
self.assertEqual(len(report_cards), 1)
dispatch_evidence = dict(
dict((work_item.metadata or {}).get("review_evidence", {}) or {}).get(
"manager_dispatch", {}
)
or {}
)
self.assertEqual(dispatch_evidence.get("outcome"), "self_produced")
async def test_attention_aux_does_not_exempt_current_self_output(self) -> None:
task = await self._make_cto_dispatch_task(
metadata_extra={
"manager_no_delegation_justification": "This request requires a direct architecture decision."
}
)
await self.store.save_delegation_work_item(
DelegationWorkItem(
work_item_id="cto-attention-item",
run_id="run-1",
parent_work_item_id="cto-dispatch-item",
role_id="cto",
seat_id="seat::team::ceo::cto",
title="CTO attention",
summary="Runtime wake-up wrapper, not delegated business work.",
kind="monitor",
projection_id="cto-attention-item",
phase=Phase.APPROVED,
metadata={
"work_item_runtime": True,
"runtime_model": "multi_team_org",
"attention_work_item": True,
},
)
)
phase = await self.executor._apply_done_transition(
task,
result=TaskResult(
status=TaskStatus.DONE,
content="Made the architecture decision directly.",
),
)
self.assertEqual(phase, Phase.AWAITING_MANAGER_REVIEW)
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
self.assertEqual(work_item.phase, Phase.AWAITING_MANAGER_REVIEW)
self.assertEqual(
len(await self._aux_cards_targeting("cto-dispatch-item", "report_target_work_item_id")),
1,
)
async def test_current_turn_business_board_mutation_keeps_dispatch_auto_approve(self) -> None:
task = await self._make_cto_dispatch_task(
metadata_extra={"manager_board_mutation_performed": True}
)
phase = await self.executor._apply_done_transition(
task, result=TaskResult(status=TaskStatus.DONE, content="Delegated to the team."),
)
self.assertEqual(phase, Phase.APPROVED)
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
self.assertEqual(
str((work_item.metadata or {}).get("turn_output_kind", "")), "delegated"
)
self.assertEqual(work_item.phase, Phase.APPROVED)
self.assertNotIn("turn_output_kind", work_item.metadata or {})
self.assertNotIn("turn_output_source", work_item.metadata or {})
report_cards = await self._aux_cards_targeting("cto-dispatch-item", "report_target_work_item_id")
self.assertEqual(report_cards, [])
@@ -1219,15 +1433,15 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(phase, Phase.AWAITING_MANAGER_REVIEW)
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
self.assertEqual(
str((work_item.metadata or {}).get("turn_output_kind", "")), "self_produced"
)
self.assertEqual(work_item.phase, Phase.AWAITING_MANAGER_REVIEW)
self.assertNotIn("turn_output_kind", work_item.metadata or {})
self.assertNotIn("turn_output_source", work_item.metadata or {})
async def test_top_seat_self_produced_falls_back_to_auto_approve(self) -> None:
# The CEO has no manager to review: the existing no-reviewer
# fallback auto-approves instead of stranding the card.
self.task.status = TaskStatus.DONE
self.task.metadata["work_kind"] = "dispatch"
self.task.metadata["work_kind"] = "intake"
self.task.metadata["manager_dispatch_guard_unresolved"] = "no children"
await self.store.update_delegation_work_item("ceo-work-item", phase=Phase.RUNNING)
@@ -1236,20 +1450,25 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
)
self.assertEqual(phase, Phase.APPROVED)
work_item = await self.store.get_delegation_work_item("ceo-work-item")
self.assertEqual(work_item.phase, Phase.APPROVED)
self.assertEqual(
await self._aux_cards_targeting("ceo-work-item", "report_target_work_item_id"),
[],
)
self.assertEqual(
await self._aux_cards_targeting("ceo-work-item", "review_target_work_item_id"),
[],
)
async def test_reconcile_rebuilds_missing_report_card(self) -> None:
# Legacy shape: card already parked in AWAITING_MANAGER_REVIEW with
# the self_produced marker but no live report/review card (the
# historical spawn refused dispatch parents). The dispatcher-tick
# reconcile must rebuild the report card idempotently.
# Current-state crash shape: phase was durably written but the process
# stopped before its report card was saved. Phase alone is the
# authoritative recovery fact; no output-kind marker is required.
await self._make_cto_dispatch_task()
await self.store.update_delegation_work_item(
"cto-dispatch-item",
phase=Phase.AWAITING_MANAGER_REVIEW,
metadata_updates={
"turn_output_kind": "self_produced",
"turn_output_source": "dispatch_guard_exhausted",
},
)
run_items = await self.store.list_delegation_work_items("run-1")
tests/test_company_runtime_suspend_resume.py
+10 -1
View File
@@ -1067,12 +1067,21 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
self.assertTrue(await engine._company_followup_target_progressed("task-ceo"))
task.metadata.pop("manager_no_delegation_justification", None)
task.metadata["manager_dispatch_guard_unresolved"] = (
"Soft dispatch constraint exhausted; accept the top-seat output."
)
await store.save_task(task)
self.assertTrue(await engine._company_followup_target_progressed("task-ceo"))
task.metadata.pop("manager_dispatch_guard_unresolved", None)
await store.save_task(task)
await store.update_delegation_work_item(
"wi-ceo",
metadata_updates={"manager_board_mutation_performed": True},
)
self.assertTrue(await engine._company_followup_target_progressed("task-ceo"))
# Dispatch outcome markers are attempt-scoped Task state. A stale
# WorkItem copy must not make a later follow-up look progressed.
self.assertFalse(await engine._company_followup_target_progressed("task-ceo"))
async def test_final_decider_followup_keeps_dispatch_turn_mode_with_existing_children(self) -> None:
runtime = CompanyRuntime(org_engine=None, communication=None, store=None)
tests/test_kanban_push_runtime.py
+5 -1
View File
@@ -380,7 +380,11 @@ class ReviewWorkItemLifecycleTests(unittest.IsolatedAsyncioTestCase):
}
await store.save_delegation_work_item(child_work_item)
await executor._close_review_work_item_for_work_item("wi-child", outcome=Phase.FAILED.value)
await executor._persist_terminal_review_card(
review_work_item_id,
phase=Phase.CANCELLED,
outcome=Phase.FAILED.value,
)
refreshed_review = await store.get_delegation_work_item(review_work_item_id)
assert refreshed_review is not None
self.assertEqual(refreshed_review.phase, Phase.CANCELLED)
tests/test_metadata_ownership.py
+34
View File
@@ -83,6 +83,40 @@ class MetadataOwnershipMatrixTests(unittest.TestCase):
self.assertNotIn("progress_log", copied)
self.assertIn("current_turn_mode", EXECUTION_COPY_KEYS)
def test_review_source_report_link_is_work_item_only_not_execution_copy(self) -> None:
key = "review_source_report_work_item_id"
item = _work_item(
{
key: "report::wi-1::v2",
"review_target_work_item_id": "wi-1",
}
)
self.assertEqual(metadata_owner_for_key(key), MetadataOwner.WORK_ITEM)
self.assertTrue(is_work_item_owned_key(key))
self.assertNotIn(key, EXECUTION_COPY_KEYS)
copied = copy_work_item_execution_metadata(item)
self.assertNotIn(key, copied)
self.assertEqual(copied["review_target_work_item_id"], "wi-1")
task = Task(id="task-1", metadata={key: "report::wi-1::v2"})
removed = strip_disallowed_work_item_metadata_from_runtime_task(task)
self.assertIn(key, removed)
self.assertNotIn(key, task.metadata)
for journal_key in (
"review_resolution",
"review_resolution_state",
"review_resolution_applied_work_item_id",
):
self.assertEqual(
metadata_owner_for_key(journal_key),
MetadataOwner.WORK_ITEM,
)
self.assertNotIn(journal_key, EXECUTION_COPY_KEYS)
def test_validate_metadata_ownership_reports_task_only_work_item_field(self) -> None:
item = _work_item({})
task = Task(
tests/test_phase_state_machine_invariants.py
+45 -13
View File
@@ -30,7 +30,10 @@ from opc.layer2_organization.phase import (
TERMINAL_PHASES,
TODO_PHASES,
coerce_phase,
is_resumable_after_claim_release,
is_runnable,
is_runtime_auxiliary_work_item,
is_stale_claim_releasable,
is_terminal,
kanban_column,
validate_transition,
@@ -253,20 +256,49 @@ def test_review_work_item_id_per_attempt_is_unique() -> None:
# ── H: in-flight phase recoverability after stale claim release ───────────
def test_inflight_phase_recoverable_after_claim_release() -> None:
"""Every in-flight phase (RUNNING / WAITING_FOR_*) must be re-claimable
once the previous claim is released (e.g. after process restart).
def test_inflight_claims_are_releasable_but_only_execution_is_resumable() -> None:
"""Claim cleanup and worker re-execution are separate invariants.
Without this invariant, any restart leaves in-flight cards as zombies
that the dispatcher refuses to pick up. The recovery path is provided
by `is_resumable_after_claim_release` defined in `phase.py`.
All in-flight claims belong to the dead process and must be released on
restart. Active execution may then resume; passive review/human parents
must wait for their auxiliary or external actor instead of running twice.
"""
from opc.layer2_organization.phase import is_resumable_after_claim_release
inflight = IN_PROGRESS_PHASES | IN_REVIEW_PHASES
not_recoverable = [p.value for p in inflight if not is_resumable_after_claim_release(p)]
assert not not_recoverable, (
f"in-flight phases that cannot recover after a stale claim is "
f"released: {not_recoverable}. After process restart these cards "
f"would become permanent zombies."
not_releasable = [p.value for p in inflight if not is_stale_claim_releasable(p)]
assert not not_releasable, (
f"in-flight phases whose dead claims cannot be released: {not_releasable}"
)
not_resumable = [
p.value for p in IN_PROGRESS_PHASES
if not is_resumable_after_claim_release(p)
]
assert not not_resumable, (
f"active execution phases that cannot resume: {not_resumable}"
)
incorrectly_resumable = [
p.value for p in IN_REVIEW_PHASES
if is_resumable_after_claim_release(p)
]
assert not incorrectly_resumable, (
f"passive review phases incorrectly resume worker execution: "
f"{incorrectly_resumable}"
)
@pytest.mark.parametrize(
"value",
[
{"attention_work_item": True},
{"report_execution_work_item": True},
{"review_execution_work_item": True},
{"work_kind": "report", "report_target_work_item_id": "parent"},
{"work_kind": "review", "review_target_work_item_id": "parent"},
{"metadata": {"attention_work_item": True}},
],
)
def test_runtime_auxiliary_identity_is_canonical(value: dict[str, object]) -> None:
assert is_runtime_auxiliary_work_item(value)
def test_business_work_item_is_not_runtime_auxiliary() -> None:
assert not is_runtime_auxiliary_work_item({"work_kind": "execute"})
tests/test_review_verdict_and_dep_refresh.py
+30 -15
View File
@@ -214,23 +214,24 @@ class RefreshDependentsForRunTests(unittest.IsolatedAsyncioTestCase):
executor = self._executor()
executor._active_tasks = [manager_task]
follow_up_result = TaskResult(
status=TaskStatus.DONE,
content="Create a PPT deck.",
artifacts={
"follow_up_actions": [
{
"action": "delegate_followup",
"target_role_id": "report_producer",
"title": "Generate PPT deck",
"summary": "Create a PPT with image2 visuals.",
"depends_on_work_item_ids": ["dep-a", "dep-b"],
}
]
},
)
created = await executor._materialize_follow_up_work_items(
manager_task,
TaskResult(
status=TaskStatus.DONE,
content="Create a PPT deck.",
artifacts={
"follow_up_actions": [
{
"action": "delegate_followup",
"target_role_id": "report_producer",
"title": "Generate PPT deck",
"summary": "Create a PPT with image2 visuals.",
"depends_on_work_item_ids": ["dep-a", "dep-b"],
}
]
},
),
follow_up_result,
)
self.assertEqual(len(created), 1)
@@ -239,6 +240,20 @@ class RefreshDependentsForRunTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(follow.phase, Phase.READY)
self.assertEqual(parent_after.phase, Phase.WAITING_FOR_CHILDREN)
# Re-emitting the same dedupe key reuses board state; it is not a
# current-turn creation signal for the dispatch guard.
reused = await executor._materialize_follow_up_work_items(
manager_task,
follow_up_result,
)
self.assertEqual(reused, [])
follow_ups = [
item
for item in await self.store.list_delegation_work_items("run-materialize")
if (item.metadata or {}).get("follow_up_dedupe_key")
]
self.assertEqual(len(follow_ups), 1)
async def test_parent_wakes_when_last_child_approved(self) -> None:
"""The canonical app12 fix: parent in WAITING_FOR_CHILDREN with a
stale claim unblocks to RUNNING and releases the claim when all
tests/test_session_serial_queue.py
+174 -1
View File
@@ -228,6 +228,18 @@ class IsDispatchableQueueFilterTests(unittest.TestCase):
)
self.assertFalse(is_dispatchable(item))
def test_passive_review_parent_without_claim_is_not_dispatchable(self) -> None:
for phase in (Phase.AWAITING_MANAGER_REVIEW, Phase.AWAITING_HUMAN):
with self.subTest(phase=phase):
item = DelegationWorkItem(
work_item_id=f"wi-{phase.value}",
run_id="r", cell_id="c", role_id="cto",
seat_id="seat", manager_role_id="ceo",
manager_seat_id="seat::ceo", title="t",
phase=phase,
)
self.assertFalse(is_dispatchable(item))
# ── enqueue_session_work_on_runnable_hook ────────────────────────────────
@@ -467,7 +479,10 @@ class SerialQueueReconcilerTests(_StoreFixture):
async def test_reconcile_preserves_valid_marker_for_busy_session(self) -> None:
self.store.role_serial_queue_enabled = True
sid = await self._seed_session(focused="wi-active", pending=["wi-next"])
await self._save_item(wid="wi-active", sid=sid, phase=Phase.RUNNING)
active = await self._save_item(wid="wi-active", sid=sid, phase=Phase.RUNNING)
active.claimed_by_role_runtime_session_id = sid
active.claimed_by_seat_id = "seat"
await self.store.save_delegation_work_item(active)
await self._save_item(wid="wi-next", sid=sid, marker=True)
result = await reconcile_role_serial_queues(self.store, "r")
@@ -479,6 +494,64 @@ class SerialQueueReconcilerTests(_StoreFixture):
self.assertEqual(item.metadata.get("queued_behind_session"), sid)
self.assertFalse(is_dispatchable(item))
async def test_reconcile_promotes_aux_behind_unclaimed_review_focus(self) -> None:
self.store.role_serial_queue_enabled = True
sid = await self._seed_session(
focused="wi-parent",
pending=["wi-report"],
status="running",
)
await self._save_item(
wid="wi-parent",
sid=sid,
phase=Phase.AWAITING_MANAGER_REVIEW,
)
report = await self._save_item(wid="wi-report", sid=sid, marker=True)
report.metadata.update({
"report_execution_work_item": True,
"report_target_work_item_id": "wi-parent",
})
await self.store.save_delegation_work_item(report)
result = await reconcile_role_serial_queues(self.store, "r")
self.assertIn(sid, result["cleared_focus_session_ids"])
self.assertIn("wi-report", result["promoted_work_item_ids"])
session = await self.store.get_delegation_role_session(sid)
self.assertEqual(session.focused_work_item_id, "")
self.assertEqual(session.pending_work_item_ids, [])
report_after = await self.store.get_delegation_work_item("wi-report")
self.assertNotIn("queued_behind_session", report_after.metadata)
self.assertTrue(is_dispatchable(report_after))
async def test_reconcile_preserves_live_claimed_review_focus(self) -> None:
self.store.role_serial_queue_enabled = True
sid = await self._seed_session(
focused="wi-parent",
pending=["wi-report"],
status="running",
)
parent = await self._save_item(
wid="wi-parent",
sid=sid,
phase=Phase.AWAITING_MANAGER_REVIEW,
)
parent.claimed_by_role_runtime_session_id = sid
parent.claimed_by_seat_id = "seat"
await self.store.save_delegation_work_item(parent)
await self._save_item(wid="wi-report", sid=sid, marker=True)
result = await reconcile_role_serial_queues(self.store, "r")
self.assertEqual(result["cleared_focus_session_ids"], [])
self.assertEqual(result["promoted_work_item_ids"], [])
session = await self.store.get_delegation_role_session(sid)
self.assertEqual(session.focused_work_item_id, "wi-parent")
self.assertEqual(session.pending_work_item_ids, ["wi-report"])
report = await self.store.get_delegation_work_item("wi-report")
self.assertEqual(report.metadata.get("queued_behind_session"), sid)
self.assertFalse(is_dispatchable(report))
async def test_reconcile_prunes_dead_entries_and_promotes_one_head(self) -> None:
self.store.role_serial_queue_enabled = True
sid = "role-runtime::r::cto"
@@ -559,6 +632,106 @@ class SerialQueueReconcilerTests(_StoreFixture):
self.assertEqual(repaired.status, TaskStatus.DONE)
class StartupClaimSweepTests(_StoreFixture):
async def test_restart_clears_dead_running_focus_so_auxiliary_can_resume(self) -> None:
sid = "role-runtime::r::cto"
report_id = "report::wi-parent::v1"
await self.store.save_delegation_role_session(
DelegationRoleSession(
role_session_id=sid,
run_id="r",
role_id="cto",
focused_work_item_id=report_id,
status="running",
)
)
await self.store.save_delegation_work_item(
DelegationWorkItem(
work_item_id=report_id,
run_id="r",
cell_id="c",
role_id="cto",
seat_id="seat",
role_runtime_session_id=sid,
manager_role_id="ceo",
manager_seat_id="seat::ceo",
title="report",
kind="report",
phase=Phase.RUNNING,
claimed_by_role_runtime_session_id=sid,
claimed_by_seat_id="seat",
metadata={
"report_execution_work_item": True,
"report_target_work_item_id": "wi-parent",
"claimed_by_role_session_id": sid,
"claimed_task_id": "task-report",
},
)
)
db_path = self.store.db_path
await self.store.close()
self.store = OPCStore(db_path=db_path)
await self.store.initialize()
self.store.role_serial_queue_enabled = True
report = await self.store.get_delegation_work_item(report_id)
session_before = await self.store.get_delegation_role_session(sid)
self.assertEqual(report.claimed_by_role_runtime_session_id, "")
self.assertTrue(is_dispatchable(report))
self.assertEqual(session_before.focused_work_item_id, report_id)
self.assertEqual(session_before.status, "running")
result = await reconcile_role_serial_queues(self.store, "r")
session_after = await self.store.get_delegation_role_session(sid)
self.assertIn(sid, result["cleared_focus_session_ids"])
self.assertEqual(session_after.focused_work_item_id, "")
self.assertEqual(session_after.status, "idle")
self.assertTrue(is_dispatchable(await self.store.get_delegation_work_item(report_id)))
async def test_restart_releases_review_claim_without_dispatching_parent(self) -> None:
parent = DelegationWorkItem(
work_item_id="wi-parent",
run_id="r", cell_id="c", role_id="cto", seat_id="seat",
manager_role_id="ceo", manager_seat_id="seat::ceo", title="parent",
phase=Phase.AWAITING_MANAGER_REVIEW,
claimed_by_role_runtime_session_id="dead-role-session",
claimed_by_seat_id="seat",
metadata={
"claimed_by_role_session_id": "dead-role-session",
"claimed_task_id": "task-parent",
},
)
report = DelegationWorkItem(
work_item_id="wi-report",
run_id="r", cell_id="c", role_id="cto", seat_id="seat",
manager_role_id="ceo", manager_seat_id="seat::ceo", title="report",
phase=Phase.READY,
metadata={
"report_execution_work_item": True,
"report_target_work_item_id": "wi-parent",
},
)
await self.store.save_delegation_work_item(parent)
await self.store.save_delegation_work_item(report)
db_path = self.store.db_path
await self.store.close()
self.store = OPCStore(db_path=db_path)
await self.store.initialize()
parent_after = await self.store.get_delegation_work_item("wi-parent")
report_after = await self.store.get_delegation_work_item("wi-report")
self.assertEqual(parent_after.claimed_by_role_runtime_session_id, "")
self.assertEqual(parent_after.claimed_by_seat_id, "")
self.assertEqual(parent_after.metadata.get("claimed_by_role_session_id"), "")
self.assertEqual(parent_after.metadata.get("claimed_task_id"), "")
self.assertTrue(parent_after.metadata.get("claim_swept_at"))
self.assertFalse(is_dispatchable(parent_after))
self.assertTrue(is_dispatchable(report_after))
# ── Config loader picks up the feature flag ──────────────────────────────
tests/test_verdict_parse_retry.py
+125
View File
@@ -29,6 +29,7 @@ from opc.layer2_organization.communication import CommunicationManager
from opc.layer2_organization.company_mode import (
CompanyWorkItemExecutor,
MAX_VERDICT_PARSE_RETRIES,
report_work_item_id_for_attempt,
review_work_item_id_for_attempt,
)
from opc.layer2_organization.org_engine import OrgEngine
@@ -238,6 +239,130 @@ class VerdictParseRetryTests(unittest.IsolatedAsyncioTestCase):
finally:
await store.close()
async def test_retry_card_save_failure_stays_in_review_and_reconciles(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
store = OPCStore(root / "tasks.db")
await store.initialize()
try:
executor = _build_executor(store, _make_org_engine(root))
child, review_id_v1 = _build_review_setup(store)
report_id = report_work_item_id_for_attempt("wi-child", 1)
child.metadata = {
**dict(child.metadata or {}),
"report_attempt_count": 1,
}
report = DelegationWorkItem(
work_item_id=report_id,
run_id="run-1",
cell_id="team::cto",
team_id="team::cto",
role_id="engineer",
seat_id="seat::team::cto::engineer",
manager_role_id="cto",
manager_seat_id="seat::team::cto::cto",
parent_work_item_id="wi-child",
title="Report #1: Build feature",
summary="Durable worker report.",
kind="report",
projection_id=report_id,
phase=Phase.APPROVED,
batch_index=1,
metadata={
"runtime_model": "multi_team_org",
"work_kind": "report",
"report_execution_work_item": True,
"report_attempt": 1,
"report_target_work_item_id": "wi-child",
"report_card_outcome": "applied",
"completion_report": "Durable worker report.",
"review_evidence": {
"completion_summary": "Durable worker report."
},
},
)
review = _make_review_card(review_card_id=review_id_v1)
review.metadata = {
**dict(review.metadata or {}),
"review_source_report_work_item_id": report_id,
}
await store.save_delegation_work_item(child)
await store.save_delegation_work_item(report)
await store.save_delegation_work_item(review)
original_insert = store.insert_delegation_work_item_if_absent
review_id_v2 = review_work_item_id_for_attempt("wi-child", 2)
injected = False
async def fail_first_retry_card_insert(
item: DelegationWorkItem,
) -> bool:
nonlocal injected
if not injected and item.work_item_id == review_id_v2:
injected = True
raise RuntimeError("injected review retry save failure")
return await original_insert(item)
store.insert_delegation_work_item_if_absent = AsyncMock(
side_effect=fail_first_retry_card_insert
)
review_task = _make_review_task(
review_card_id=review_id_v1,
structured_verdict={"unparseable": True},
)
try:
await executor._finalize_review_work_item(review_task)
finally:
store.insert_delegation_work_item_if_absent = original_insert
self.assertTrue(injected)
child_after_failure = await store.get_delegation_work_item(
"wi-child"
)
failed_review = await store.get_delegation_work_item(review_id_v1)
self.assertEqual(
child_after_failure.phase,
Phase.AWAITING_MANAGER_REVIEW,
)
self.assertFalse(
child_after_failure.metadata.get(
"review_verdict_parse_failed_auto_done", False
)
)
self.assertEqual(failed_review.phase, Phase.CANCELLED)
self.assertEqual(
failed_review.metadata.get("review_work_item_outcome"),
"verdict_parse_failed",
)
self.assertIsNone(
await store.get_delegation_work_item(review_id_v2)
)
run_items = await store.list_delegation_work_items("run-1")
await executor._reconcile_missing_review_chain(run_items)
await executor._reconcile_missing_review_chain(
await store.list_delegation_work_items("run-1")
)
retry = await store.get_delegation_work_item(review_id_v2)
self.assertIsNotNone(retry)
self.assertEqual(retry.phase, Phase.READY)
self.assertEqual(
retry.metadata.get("review_source_report_work_item_id"),
report_id,
)
self.assertEqual(
retry.metadata.get("review_retry_reason"),
"verdict_parse_failed",
)
self.assertIsNone(
await store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
)
finally:
await store.close()
async def test_retry_preserves_review_owner_when_runtime_task_lost_owner_metadata(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
tests/test_wake_and_delivery.py
+4 -41
View File
@@ -18,7 +18,6 @@ from __future__ import annotations
import tempfile
import unittest
from datetime import datetime
from pathlib import Path
from unittest.mock import MagicMock
@@ -599,45 +598,6 @@ class DeliveryCardPhaseSyncTests(unittest.IsolatedAsyncioTestCase):
self.assertFalse(any(item.kind == "report" for item in items))
self.assertFalse(any(item.kind == "review" for item in items))
async def test_repair_stuck_aggregate_review_item_unblocks_dependents(self) -> None:
executor = self._executor()
stuck = DelegationWorkItem(
work_item_id="aggregate-stuck",
run_id="r",
cell_id="c",
role_id="cto",
seat_id="seat-cto",
manager_role_id="ceo",
manager_seat_id="seat-ceo",
title="Stuck aggregate",
kind="execute",
phase=Phase.AWAITING_MANAGER_REVIEW,
metadata={"work_kind": "synthesize", "work_item_turn_type": "aggregate"},
)
dependent = DelegationWorkItem(
work_item_id="final-qa",
run_id="r",
cell_id="c",
role_id="coo",
seat_id="seat-coo",
manager_role_id="ceo",
manager_seat_id="seat-ceo",
title="Final QA",
kind="review",
phase=Phase.WAITING_DEPENDENCIES,
metadata={"dependency_work_item_ids": ["aggregate-stuck"]},
)
await self.store.save_delegation_work_item(stuck)
await self.store.save_delegation_work_item(dependent)
work_items = await self.store.list_delegation_work_items("r")
await executor._repair_stuck_aggregate_review_items(work_items)
repaired = await self.store.get_delegation_work_item("aggregate-stuck")
unblocked = await self.store.get_delegation_work_item("final-qa")
self.assertEqual(repaired.phase, Phase.APPROVED)
self.assertEqual(unblocked.phase, Phase.READY)
async def test_stale_report_for_delivery_parent_closes_without_review_card(self) -> None:
executor = self._executor()
parent = DelegationWorkItem(
@@ -697,7 +657,10 @@ class DeliveryCardPhaseSyncTests(unittest.IsolatedAsyncioTestCase):
closed = await self.store.get_delegation_work_item(report.work_item_id)
self.assertEqual(closed.phase, Phase.APPROVED)
self.assertEqual(closed.metadata["report_card_outcome"], "non_reviewable_parent")
self.assertEqual(
closed.metadata["report_card_outcome"],
"parent_not_awaiting_review",
)
items = await self.store.list_delegation_work_items("r")
self.assertFalse(any(str(item.work_item_id).startswith("review::deliver-parent") for item in items))
tests/test_worker_report_handoff.py
+962
View File
@@ -32,6 +32,7 @@ from opc.layer2_organization.company_mode import (
review_work_item_id_for_attempt,
)
from opc.layer2_organization.org_engine import OrgEngine
from opc.layer2_organization.phase import DONE_PHASES
from opc.layer2_organization.work_item_links import set_linked_work_item_id
@@ -201,6 +202,10 @@ class WorkerExecuteDoneSpawnsReportTests(unittest.IsolatedAsyncioTestCase):
worker_task = _build_worker_task()
worker_task.metadata = dict(worker_task.metadata or {})
worker_task.metadata["work_kind"] = "dispatch"
# Delegation is attempt-scoped. The child row alone may be
# historical, so mirror the tool's current-turn mutation
# marker instead of asking DONE routing to infer from rows.
worker_task.metadata["manager_board_mutation_performed"] = True
await store.save_task(worker_task)
await executor._apply_done_transition(
@@ -396,6 +401,963 @@ class ReportTurnDoneSpawnsReviewTests(unittest.IsolatedAsyncioTestCase):
await store.close()
class ReviewChainRecoveryTests(unittest.IsolatedAsyncioTestCase):
"""Crash boundaries use persisted auxiliary cards as their journal.
The attempt counters on the parent are only lookup caches: a crash can
happen after an auxiliary card commits but before its counter does. The
report card also has to contain the completed report before review-card
creation is attempted, so reconciliation can resume without a live Task.
"""
async def asyncSetUp(self) -> None:
self._tmpdir = tempfile.TemporaryDirectory()
self.root = Path(self._tmpdir.name)
self.store = OPCStore(self.root / "tasks.db")
await self.store.initialize()
self.executor = _build_executor(self.store, _make_org_engine(self.root))
async def asyncTearDown(self) -> None:
await self.store.close()
self._tmpdir.cleanup()
async def _save_awaiting_parent(self) -> DelegationWorkItem:
parent = _build_child_work_item()
parent.phase = Phase.AWAITING_MANAGER_REVIEW
parent.metadata = {
**dict(parent.metadata or {}),
"review_owner_role_id": "cto",
"review_owner_seat_id": "seat::team::cto::cto",
"completion_report": "execute-turn fallback",
}
await self.store.save_delegation_work_item(parent)
return parent
async def _run_reconcile(self) -> list[DelegationWorkItem]:
items = await self.store.list_delegation_work_items("run-1")
return await self.executor._reconcile_missing_review_chain(items)
async def _auxiliary_cards(self) -> list[DelegationWorkItem]:
return [
item
for item in await self.store.list_delegation_work_items("run-1")
if str((item.metadata or {}).get("report_target_work_item_id", "") or "").strip()
== "wi-child"
or str((item.metadata or {}).get("review_target_work_item_id", "") or "").strip()
== "wi-child"
]
async def _setup_running_report(
self,
) -> tuple[str, Task]:
"""Create the real execute->report handoff and claim its report card."""
await self.store.save_delegation_work_item(_build_child_work_item())
worker_task = _build_worker_task()
await self.executor._apply_done_transition(
worker_task,
result=TaskResult(status=TaskStatus.DONE, content="execute fallback"),
)
report_id = report_work_item_id_for_attempt("wi-child", 1)
role_session_id = "role-runtime::run-1::engineer"
await self.store.update_delegation_work_item(
report_id,
phase=Phase.RUNNING,
claimed_by_role_runtime_session_id=role_session_id,
claimed_by_seat_id="seat::team::cto::engineer",
metadata_updates={
"claimed_by_role_session_id": role_session_id,
"claimed_task_id": "task-report-1",
},
)
report_task = ReportTurnDoneSpawnsReviewTests()._report_turn_task(
report_card_id=report_id,
target_work_item_id="wi-child",
)
return report_id, report_task
async def _setup_running_review(
self,
*,
verdict: dict[str, object],
) -> tuple[str, str, Task]:
"""Create and claim a review card using the production report path."""
report_id, report_task = await self._setup_running_report()
await self.executor._apply_report_done_transition(
report_task,
result=TaskResult(status=TaskStatus.DONE, content="durable report v1"),
)
review_id = review_work_item_id_for_attempt("wi-child", 1)
role_session_id = "role-runtime::run-1::cto"
await self.store.update_delegation_work_item(
review_id,
phase=Phase.RUNNING,
claimed_by_role_runtime_session_id=role_session_id,
claimed_by_seat_id="seat::team::cto::cto",
metadata_updates={
"claimed_by_role_session_id": role_session_id,
"claimed_task_id": "task-review-1",
},
)
review_item = await self.store.get_delegation_work_item(review_id)
self.assertIsNotNone(review_item)
review_task = Task(
id="task-review-1",
title="Review #1: Build feature",
project_id="proj1",
session_id="session-cto",
parent_session_id="session-root",
assigned_to="cto",
status=TaskStatus.DONE,
metadata={
**dict(review_item.metadata or {}),
"execution_mode": "company_mode",
"runtime_model": "multi_team_org",
"delegation_run_id": "run-1",
"work_item_runtime": True,
"review_execution_work_item": True,
"structured_review_verdict": verdict,
},
)
set_linked_work_item_id(review_task, review_id)
return report_id, review_id, review_task
async def test_insert_if_absent_preserves_claimed_deterministic_aux_card(self) -> None:
cases = (
(
"report",
report_work_item_id_for_attempt("wi-child", 1),
"engineer",
"seat::team::cto::engineer",
{"report_target_work_item_id": "wi-child", "report_attempt": 1},
),
(
"review",
review_work_item_id_for_attempt("wi-child", 1),
"cto",
"seat::team::cto::cto",
{"review_target_work_item_id": "wi-child", "review_attempt": 1},
),
)
for kind, work_item_id, role_id, seat_id, target_metadata in cases:
with self.subTest(kind=kind):
ready = DelegationWorkItem(
work_item_id=work_item_id,
run_id="run-1",
cell_id="team::cto",
team_id="team::cto",
role_id=role_id,
seat_id=seat_id,
parent_work_item_id="wi-child",
title=f"{kind.title()} attempt 1",
kind=kind,
projection_id=work_item_id,
phase=Phase.READY,
batch_index=1,
metadata={
"work_kind": kind,
**target_metadata,
"persisted_sentinel": f"original-{kind}",
},
)
self.assertTrue(
await self.store.insert_delegation_work_item_if_absent(ready)
)
role_session_id = f"role-runtime::run-1::{role_id}"
claimed_task_id = f"task-{kind}-1"
await self.store.update_delegation_work_item(
work_item_id,
phase=Phase.RUNNING,
claimed_by_role_runtime_session_id=role_session_id,
claimed_by_seat_id=seat_id,
metadata_updates={
"claimed_by_role_session_id": role_session_id,
"claimed_task_id": claimed_task_id,
},
)
competing_ready = DelegationWorkItem(
work_item_id=work_item_id,
run_id="run-1",
cell_id="team::cto",
team_id="team::cto",
role_id=role_id,
seat_id=seat_id,
parent_work_item_id="wi-child",
title=f"Competing {kind} attempt 1",
kind=kind,
projection_id=work_item_id,
phase=Phase.READY,
batch_index=1,
metadata={
"work_kind": kind,
**target_metadata,
"persisted_sentinel": f"competing-{kind}",
},
)
inserted = await self.store.insert_delegation_work_item_if_absent(
competing_ready
)
persisted = await self.store.get_delegation_work_item(work_item_id)
self.assertFalse(inserted)
self.assertIsNotNone(persisted)
self.assertEqual(persisted.phase, Phase.RUNNING)
self.assertEqual(
persisted.claimed_by_role_runtime_session_id,
role_session_id,
)
self.assertEqual(persisted.claimed_by_seat_id, seat_id)
self.assertEqual(
persisted.metadata.get("claimed_by_role_session_id"),
role_session_id,
)
self.assertEqual(
persisted.metadata.get("claimed_task_id"), claimed_task_id
)
self.assertEqual(
persisted.metadata.get("persisted_sentinel"),
f"original-{kind}",
)
async def test_report_terminal_write_failure_releases_claim_for_retry(self) -> None:
report_id, report_task = await self._setup_running_report()
original_update = self.store.update_delegation_work_item
injected = False
async def fail_first_terminal_report_write(work_item_id: str, **kwargs):
nonlocal injected
metadata_updates = dict(kwargs.get("metadata_updates") or {})
if (
not injected
and work_item_id == report_id
and kwargs.get("phase") == Phase.APPROVED
and metadata_updates.get("report_card_outcome") == "applied"
):
injected = True
raise RuntimeError("injected terminal report journal failure")
return await original_update(work_item_id, **kwargs)
self.store.update_delegation_work_item = AsyncMock(
side_effect=fail_first_terminal_report_write
)
try:
await self.executor._apply_report_done_transition(
report_task,
result=TaskResult(status=TaskStatus.DONE, content="volatile report"),
)
finally:
self.store.update_delegation_work_item = original_update
self.assertTrue(injected)
parent = await self.store.get_delegation_work_item("wi-child")
report = await self.store.get_delegation_work_item(report_id)
self.assertEqual(parent.phase, Phase.AWAITING_MANAGER_REVIEW)
self.assertEqual(report.phase, Phase.RUNNING)
self.assertEqual(report.claimed_by_role_runtime_session_id, "")
self.assertEqual(report.claimed_by_seat_id, "")
self.assertEqual(report.metadata.get("claimed_by_role_session_id"), "")
self.assertEqual(report.metadata.get("claimed_task_id"), "")
self.assertTrue(
CompanyWorkItemExecutor._work_item_is_runnable(
report,
{"wi-child": parent, report_id: report},
)
)
self.assertIsNone(
await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 1)
)
)
async def test_review_terminal_write_failure_keeps_parent_reviewable(self) -> None:
report_id, review_id, review_task = await self._setup_running_review(
verdict={
"label": "reject",
"summary": "needs rework",
"blocking_issues": ["fix the defect"],
"followups": [],
}
)
original_update = self.store.update_delegation_work_item
injected = False
async def fail_first_terminal_review_write(work_item_id: str, **kwargs):
nonlocal injected
if (
not injected
and work_item_id == review_id
and kwargs.get("phase") in DONE_PHASES
):
injected = True
raise RuntimeError("injected terminal review journal failure")
return await original_update(work_item_id, **kwargs)
self.store.update_delegation_work_item = AsyncMock(
side_effect=fail_first_terminal_review_write
)
try:
await self.executor._finalize_review_work_item(review_task)
finally:
self.store.update_delegation_work_item = original_update
self.assertTrue(injected)
parent = await self.store.get_delegation_work_item("wi-child")
review = await self.store.get_delegation_work_item(review_id)
self.assertEqual(parent.phase, Phase.AWAITING_MANAGER_REVIEW)
self.assertEqual(review.phase, Phase.RUNNING)
self.assertEqual(review.claimed_by_role_runtime_session_id, "")
self.assertEqual(review.claimed_by_seat_id, "")
self.assertEqual(review.metadata.get("claimed_by_role_session_id"), "")
self.assertEqual(review.metadata.get("claimed_task_id"), "")
self.assertIsNone(
await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
)
self.assertIsNone(
await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 2)
)
)
self.assertEqual(
review.metadata.get("review_source_report_work_item_id"),
report_id,
)
async def test_late_review_cannot_override_parent_awaiting_human(self) -> None:
_report_id, review_id, review_task = await self._setup_running_review(
verdict={
"label": "approve",
"summary": "approve from a now-stale manager turn",
"blocking_issues": [],
"followups": [],
}
)
await self.store.update_delegation_work_item(
"wi-child",
phase=Phase.AWAITING_HUMAN,
metadata_updates={"human_checkpoint_sentinel": "must-survive"},
)
await self.executor._finalize_review_work_item(review_task)
parent = await self.store.get_delegation_work_item("wi-child")
review = await self.store.get_delegation_work_item(review_id)
self.assertEqual(parent.phase, Phase.AWAITING_HUMAN)
self.assertEqual(
parent.metadata.get("human_checkpoint_sentinel"), "must-survive"
)
self.assertNotIn("review_resolution_applied_work_item_id", parent.metadata)
self.assertNotIn("structured_review_verdict", parent.metadata)
self.assertNotIn("reviewed_at", parent.metadata)
self.assertIn(review.phase, DONE_PHASES)
self.assertEqual(
review.metadata.get("review_work_item_outcome"),
"target_no_longer_awaiting_manager_review",
)
self.assertNotEqual(
review.metadata.get("review_resolution_state"), "applied"
)
self.assertNotIn("review_resolution", review.metadata)
self.assertEqual(review.claimed_by_role_runtime_session_id, "")
self.assertEqual(review.claimed_by_seat_id, "")
async def test_late_review_is_stale_when_newer_applied_report_exists(self) -> None:
report_v1_id, review_v1_id, review_v1_task = (
await self._setup_running_review(
verdict={
"label": "approve",
"summary": "approval based on report v1",
"blocking_issues": [],
"followups": [],
}
)
)
report_v2_id = report_work_item_id_for_attempt("wi-child", 2)
report_v2 = DelegationWorkItem(
work_item_id=report_v2_id,
run_id="run-1",
cell_id="team::cto",
team_id="team::cto",
role_id="engineer",
seat_id="seat::team::cto::engineer",
manager_role_id="cto",
manager_seat_id="seat::team::cto::cto",
parent_work_item_id="wi-child",
title="Report attempt 2",
summary="Newer durable handoff.",
kind="report",
projection_id=report_v2_id,
phase=Phase.APPROVED,
batch_index=2,
metadata={
"runtime_model": "multi_team_org",
"work_kind": "report",
"report_execution_work_item": True,
"report_target_work_item_id": "wi-child",
"report_attempt": 2,
"report_card_outcome": "applied",
"completion_report": "authoritative report v2",
},
)
await self.store.save_delegation_work_item(report_v2)
await self.store.update_delegation_work_item(
"wi-child",
metadata_updates={
"completion_report": "authoritative report v2",
"newer_report_sentinel": "must-survive",
},
)
await self.executor._finalize_review_work_item(review_v1_task)
parent = await self.store.get_delegation_work_item("wi-child")
review_v1 = await self.store.get_delegation_work_item(review_v1_id)
self.assertEqual(parent.phase, Phase.AWAITING_MANAGER_REVIEW)
self.assertEqual(parent.metadata.get("completion_report"), "authoritative report v2")
self.assertEqual(parent.metadata.get("newer_report_sentinel"), "must-survive")
self.assertNotIn("review_resolution_applied_work_item_id", parent.metadata)
self.assertNotIn("structured_review_verdict", parent.metadata)
self.assertNotIn("reviewed_at", parent.metadata)
self.assertIn(review_v1.phase, DONE_PHASES)
self.assertEqual(
review_v1.metadata.get("review_source_report_work_item_id"),
report_v1_id,
)
self.assertEqual(review_v1.metadata.get("review_resolution_state"), "stale")
self.assertEqual(
review_v1.metadata.get("review_resolution_stale_reason"),
"source_report_superseded",
)
self.assertEqual(
review_v1.metadata.get("review_work_item_outcome"),
"superseded_by_newer_report",
)
self.assertEqual(
(review_v1.metadata.get("review_resolution") or {}).get(
"source_report_work_item_id"
),
report_v1_id,
)
async def test_reconcile_replays_terminal_review_after_parent_write_failure(
self,
) -> None:
report_id, review_id, review_task = await self._setup_running_review(
verdict={
"label": "reject",
"summary": "needs rework",
"blocking_issues": ["fix the defect"],
"followups": [],
}
)
original_apply = self.store.apply_delegation_review_resolution
injected = False
async def fail_first_parent_projection(work_item_id: str, **kwargs):
nonlocal injected
if not injected and work_item_id == "wi-child":
injected = True
raise RuntimeError("injected child verdict projection failure")
return await original_apply(work_item_id, **kwargs)
self.store.apply_delegation_review_resolution = AsyncMock(
side_effect=fail_first_parent_projection
)
try:
await self.executor._finalize_review_work_item(review_task)
finally:
self.store.apply_delegation_review_resolution = original_apply
self.assertTrue(injected)
parent_before = await self.store.get_delegation_work_item("wi-child")
review_before = await self.store.get_delegation_work_item(review_id)
self.assertEqual(parent_before.phase, Phase.AWAITING_MANAGER_REVIEW)
self.assertIn(review_before.phase, DONE_PHASES)
self.assertEqual(
review_before.metadata.get("review_source_report_work_item_id"),
report_id,
)
await self._run_reconcile()
await self._run_reconcile()
parent_after = await self.store.get_delegation_work_item("wi-child")
self.assertEqual(parent_after.phase, Phase.READY_FOR_REWORK)
self.assertEqual(
(parent_after.metadata.get("structured_review_verdict") or {}).get(
"label"
),
"reject",
)
self.assertEqual(parent_after.metadata.get("review_rework_count"), 1)
self.assertIn(
"fix the defect",
str(parent_after.metadata.get("rework_feedback", "")),
)
self.assertEqual(
parent_after.metadata.get("review_resolution_applied_work_item_id"),
review_id,
)
report_cards = [
item for item in await self._auxiliary_cards() if item.kind == "report"
]
review_cards = [
item for item in await self._auxiliary_cards() if item.kind == "review"
]
self.assertEqual([item.work_item_id for item in report_cards], [report_id])
self.assertEqual([item.work_item_id for item in review_cards], [review_id])
# A later worker attempt re-enters review with the old journal still
# present. The atomic applied stamp must make that verdict immutable
# history, not a resolution to replay onto the new output.
await self.store.update_delegation_work_item(
"wi-child",
phase=Phase.RUNNING,
)
await self.store.update_delegation_work_item(
"wi-child",
phase=Phase.AWAITING_MANAGER_REVIEW,
)
await self._run_reconcile()
next_cycle_parent = await self.store.get_delegation_work_item("wi-child")
self.assertEqual(next_cycle_parent.phase, Phase.AWAITING_MANAGER_REVIEW)
next_report = await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
self.assertIsNotNone(next_report)
self.assertEqual(next_report.phase, Phase.READY)
self.assertIsNone(
await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 2)
)
)
async def test_report_card_commit_survives_parent_counter_failure(self) -> None:
await self._save_awaiting_parent()
worker_task = _build_worker_task()
original_update = self.store.update_delegation_work_item
async def fail_report_counter(work_item_id: str, **kwargs):
metadata_updates = dict(kwargs.get("metadata_updates") or {})
if work_item_id == "wi-child" and "report_attempt_count" in metadata_updates:
raise RuntimeError("injected crash after report-card commit")
return await original_update(work_item_id, **kwargs)
self.store.update_delegation_work_item = AsyncMock(side_effect=fail_report_counter)
first = await self.executor._ensure_report_work_item_for_work_item(
"wi-child", worker_task=worker_task
)
self.assertIsNotNone(first)
self.assertEqual(first.work_item_id, report_work_item_id_for_attempt("wi-child", 1))
parent = await self.store.get_delegation_work_item("wi-child")
self.assertNotIn("report_attempt_count", parent.metadata or {})
# Make a re-save of v1 observably wrong: retry must discover and
# return the persisted RUNNING card instead of trying READY again.
await original_update(first.work_item_id, phase=Phase.RUNNING)
second = await self.executor._ensure_report_work_item_for_work_item(
"wi-child", worker_task=worker_task
)
self.assertIsNotNone(second)
self.assertEqual(second.work_item_id, first.work_item_id)
self.assertEqual(second.phase, Phase.RUNNING)
self.assertIsNone(
await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
)
async def test_review_card_commit_survives_parent_counter_failure(self) -> None:
await self._save_awaiting_parent()
worker_task = _build_worker_task()
original_update = self.store.update_delegation_work_item
async def fail_review_counter(work_item_id: str, **kwargs):
metadata_updates = dict(kwargs.get("metadata_updates") or {})
if work_item_id == "wi-child" and "review_attempt_count" in metadata_updates:
raise RuntimeError("injected crash after review-card commit")
return await original_update(work_item_id, **kwargs)
self.store.update_delegation_work_item = AsyncMock(side_effect=fail_review_counter)
first = await self.executor._ensure_review_work_item_for_work_item(
"wi-child",
worker_task=worker_task,
completion_report="handoff",
metadata_updates={
"review_owner_role_id": "cto",
"review_owner_seat_id": "seat::team::cto::cto",
},
)
self.assertIsNotNone(first)
self.assertEqual(first.work_item_id, review_work_item_id_for_attempt("wi-child", 1))
parent = await self.store.get_delegation_work_item("wi-child")
self.assertNotIn("review_attempt_count", parent.metadata or {})
await original_update(first.work_item_id, phase=Phase.RUNNING)
second = await self.executor._ensure_review_work_item_for_work_item(
"wi-child",
worker_task=worker_task,
completion_report="handoff",
metadata_updates={
"review_owner_role_id": "cto",
"review_owner_seat_id": "seat::team::cto::cto",
},
)
self.assertIsNotNone(second)
self.assertEqual(second.work_item_id, first.work_item_id)
self.assertEqual(second.phase, Phase.RUNNING)
self.assertIsNone(
await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 2)
)
)
async def test_reconcile_recovers_review_from_terminal_report(self) -> None:
# Build the real report turn first so this exercises the exact durable
# payload written at the report-DONE crash boundary.
await self.store.save_delegation_work_item(_build_child_work_item())
worker_task = _build_worker_task()
await self.executor._apply_done_transition(
worker_task,
result=TaskResult(status=TaskStatus.DONE, content="execute fallback"),
)
report_id = report_work_item_id_for_attempt("wi-child", 1)
await self.store.update_delegation_work_item(report_id, phase=Phase.RUNNING)
report_task = ReportTurnDoneSpawnsReviewTests()._report_turn_task(
report_card_id=report_id,
target_work_item_id="wi-child",
)
report_payload = (
'{"summary":"durable handoff","deliverables":[],"risks":[],"next_actions":[]}'
)
original_insert = self.store.insert_delegation_work_item_if_absent
async def fail_review_insert(
item: DelegationWorkItem,
) -> bool:
if item.kind == "review":
raise RuntimeError("injected crash while saving review card")
return await original_insert(item)
self.store.insert_delegation_work_item_if_absent = AsyncMock(
side_effect=fail_review_insert
)
await self.executor._apply_report_done_transition(
report_task,
result=TaskResult(status=TaskStatus.DONE, content=report_payload),
)
terminal_report = await self.store.get_delegation_work_item(report_id)
self.assertEqual(terminal_report.phase, Phase.APPROVED)
self.assertEqual(terminal_report.metadata.get("report_card_outcome"), "applied")
self.assertEqual(terminal_report.metadata.get("completion_report"), report_payload)
self.assertEqual(terminal_report.metadata.get("report_completion_raw"), report_payload)
self.assertTrue(terminal_report.metadata.get("review_evidence"))
self.assertIsNone(
await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 1)
)
)
# Simulate restart: there is no runtime Task available to carry the
# payload, only the parent and terminal report rows.
self.store.insert_delegation_work_item_if_absent = original_insert
self.assertEqual(await self.store.get_tasks(), [])
await self._run_reconcile()
await self._run_reconcile()
review = await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 1)
)
self.assertIsNotNone(review)
self.assertEqual(review.metadata.get("review_completion_report"), report_payload)
self.assertEqual(
review.metadata.get("review_source_report_work_item_id"),
report_id,
)
self.assertEqual(
(review.metadata.get("review_evidence") or {}).get("completion_summary"),
report_payload,
)
reports = [item for item in await self._auxiliary_cards() if item.kind == "report"]
self.assertEqual([item.work_item_id for item in reports], [report_id])
self.assertIsNone(
await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
)
async def test_reconcile_repairs_parent_after_terminal_report_projection_failure(
self,
) -> None:
await self.store.save_delegation_work_item(_build_child_work_item())
worker_task = _build_worker_task()
await self.executor._apply_done_transition(
worker_task,
result=TaskResult(status=TaskStatus.DONE, content="execute fallback"),
)
report_id = report_work_item_id_for_attempt("wi-child", 1)
await self.store.update_delegation_work_item(report_id, phase=Phase.RUNNING)
report_task = ReportTurnDoneSpawnsReviewTests()._report_turn_task(
report_card_id=report_id,
target_work_item_id="wi-child",
)
report_payload = "Report payload committed before the parent projection."
original_update = self.store.update_delegation_work_item
async def fail_parent_projection(work_item_id: str, **kwargs):
metadata_updates = dict(kwargs.get("metadata_updates") or {})
if (
work_item_id == "wi-child"
and metadata_updates.get("completion_report") == report_payload
):
raise RuntimeError("injected crash while projecting report to parent")
return await original_update(work_item_id, **kwargs)
self.store.update_delegation_work_item = AsyncMock(
side_effect=fail_parent_projection
)
await self.executor._apply_report_done_transition(
report_task,
result=TaskResult(status=TaskStatus.DONE, content=report_payload),
)
terminal_report = await self.store.get_delegation_work_item(report_id)
self.assertEqual(terminal_report.phase, Phase.APPROVED)
self.assertEqual(terminal_report.metadata.get("report_card_outcome"), "applied")
self.assertEqual(terminal_report.metadata.get("completion_report"), report_payload)
parent_before_reconcile = await self.store.get_delegation_work_item("wi-child")
self.assertNotEqual(
(parent_before_reconcile.metadata or {}).get("completion_report"),
report_payload,
)
self.assertIsNone(
await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 1)
)
)
self.store.update_delegation_work_item = original_update
await self._run_reconcile()
await self._run_reconcile()
parent_after_reconcile = await self.store.get_delegation_work_item("wi-child")
self.assertEqual(
parent_after_reconcile.metadata.get("completion_report"),
report_payload,
)
review = await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 1)
)
self.assertIsNotNone(review)
self.assertEqual(review.metadata.get("review_completion_report"), report_payload)
self.assertEqual(
review.metadata.get("review_source_report_work_item_id"),
report_id,
)
self.assertIsNone(
await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
)
async def test_durable_v2_overrides_lagging_attempt_counters(self) -> None:
await self._save_awaiting_parent()
await self.store.update_delegation_work_item(
"wi-child",
metadata_updates={
"report_attempt_count": 1,
"review_attempt_count": 1,
},
)
terminal_report = DelegationWorkItem(
work_item_id=report_work_item_id_for_attempt("wi-child", 2),
run_id="run-1",
cell_id="team::cto",
team_id="team::cto",
role_id="engineer",
seat_id="seat::team::cto::engineer",
manager_role_id="cto",
manager_seat_id="seat::team::cto::cto",
parent_work_item_id="wi-child",
title="Terminal report v2",
summary="Immutable report history.",
kind="report",
projection_id=report_work_item_id_for_attempt("wi-child", 2),
phase=Phase.APPROVED,
batch_index=2,
metadata={
"runtime_model": "multi_team_org",
"work_kind": "report",
"report_execution_work_item": True,
"report_target_work_item_id": "wi-child",
"report_attempt": 2,
"report_card_outcome": "applied",
"completion_report": "Immutable report history.",
"history_sentinel": "report-v2-must-not-change",
},
)
terminal_review = DelegationWorkItem(
work_item_id=review_work_item_id_for_attempt("wi-child", 2),
run_id="run-1",
cell_id="team::cto",
team_id="team::cto",
role_id="cto",
seat_id="seat::team::cto::cto",
manager_role_id="ceo",
manager_seat_id="seat::team::cto::ceo",
parent_work_item_id="wi-child",
title="Terminal review v2",
summary="Immutable review history.",
kind="review",
projection_id=review_work_item_id_for_attempt("wi-child", 2),
phase=Phase.APPROVED,
batch_index=2,
metadata={
"runtime_model": "multi_team_org",
"work_kind": "review",
"review_execution_work_item": True,
"review_target_work_item_id": "wi-child",
"review_attempt": 2,
"review_work_item_outcome": "approved",
"history_sentinel": "review-v2-must-not-change",
},
)
await self.store.save_delegation_work_item(terminal_report)
await self.store.save_delegation_work_item(terminal_review)
worker_task = _build_worker_task()
new_report = await self.executor._ensure_report_work_item_for_work_item(
"wi-child",
worker_task=worker_task,
)
self.assertIsNotNone(new_report)
self.assertEqual(
new_report.work_item_id,
report_work_item_id_for_attempt("wi-child", 3),
)
# Review and report auxiliaries must never be active in parallel.
self.assertIsNone(
await self.executor._ensure_review_work_item_for_work_item(
"wi-child",
worker_task=worker_task,
completion_report="new completion",
metadata_updates={
"review_owner_role_id": "cto",
"review_owner_seat_id": "seat::team::cto::cto",
},
source_report_item=terminal_report,
)
)
await self.store.update_delegation_work_item(
new_report.work_item_id,
phase=Phase.CANCELLED,
)
new_review = await self.executor._ensure_review_work_item_for_work_item(
"wi-child",
worker_task=worker_task,
completion_report="new completion",
metadata_updates={
"review_owner_role_id": "cto",
"review_owner_seat_id": "seat::team::cto::cto",
},
source_report_item=terminal_report,
)
self.assertIsNotNone(new_review)
self.assertEqual(
new_review.work_item_id,
review_work_item_id_for_attempt("wi-child", 3),
)
persisted_report_v2 = await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
persisted_review_v2 = await self.store.get_delegation_work_item(
review_work_item_id_for_attempt("wi-child", 2)
)
self.assertEqual(persisted_report_v2.phase, Phase.APPROVED)
self.assertEqual(
persisted_report_v2.metadata.get("history_sentinel"),
"report-v2-must-not-change",
)
self.assertEqual(persisted_review_v2.phase, Phase.APPROVED)
self.assertEqual(
persisted_review_v2.metadata.get("history_sentinel"),
"review-v2-must-not-change",
)
async def test_reconcile_without_runtime_task_creates_report(self) -> None:
await self._save_awaiting_parent()
self.assertEqual(await self.store.get_tasks(), [])
await self._run_reconcile()
report = await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 1)
)
self.assertIsNotNone(report)
self.assertEqual(report.phase, Phase.READY)
self.assertEqual(report.metadata.get("report_target_work_item_id"), "wi-child")
self.assertEqual(report.role_id, "engineer")
self.assertEqual(report.seat_id, "seat::team::cto::engineer")
async def test_repeated_reconcile_keeps_at_most_one_active_auxiliary(self) -> None:
await self._save_awaiting_parent()
for _ in range(5):
await self._run_reconcile()
auxiliaries = await self._auxiliary_cards()
active = [item for item in auxiliaries if item.phase not in DONE_PHASES]
self.assertEqual(len(active), 1)
self.assertEqual(active[0].kind, "report")
self.assertEqual(active[0].work_item_id, report_work_item_id_for_attempt("wi-child", 1))
self.assertIsNone(
await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
)
async def test_reconcile_remains_idempotent_after_database_reopen(self) -> None:
await self._save_awaiting_parent()
await self._run_reconcile()
await self.store.close()
self.store = OPCStore(self.root / "tasks.db")
await self.store.initialize()
self.executor = _build_executor(self.store, _make_org_engine(self.root))
for _ in range(3):
await self._run_reconcile()
auxiliaries = await self._auxiliary_cards()
active = [item for item in auxiliaries if item.phase not in DONE_PHASES]
self.assertEqual(len(active), 1)
self.assertEqual(
active[0].work_item_id,
report_work_item_id_for_attempt("wi-child", 1),
)
self.assertIsNone(
await self.store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
)
class ReportCardRunnableFilterTests(unittest.TestCase):
"""Pin the dispatcher-runnability filters for report cards.