fix(company): make delegation review lifecycle durable
This commit is contained in:
+151
-17
@@ -76,7 +76,7 @@ from opc.layer2_organization.phase import (
|
||||
InvalidPhaseTransition,
|
||||
TODO_PHASES,
|
||||
coerce_phase,
|
||||
is_resumable_after_claim_release,
|
||||
is_stale_claim_releasable,
|
||||
is_terminal,
|
||||
kanban_column,
|
||||
on_phase_transition,
|
||||
@@ -4600,9 +4600,9 @@ class OPCStore:
|
||||
We only touch in-flight phases (RUNNING / WAITING_FOR_* /
|
||||
PAUSED / NEEDS_ATTENTION / AWAITING_*) so we never disturb
|
||||
terminal cards. The phase itself is left alone; the sweep only
|
||||
clears the claim. The dispatcher's ``is_dispatchable`` check
|
||||
recognises a non-terminal card with no claim as eligible for
|
||||
re-pick on the next tick.
|
||||
clears the claim. Active execution can subsequently be re-picked;
|
||||
passive AWAITING_* parents remain non-dispatchable while their
|
||||
report/review chain (or human) advances them.
|
||||
"""
|
||||
if self._db is None:
|
||||
return 0
|
||||
@@ -4619,7 +4619,7 @@ class OPCStore:
|
||||
phase = coerce_phase(phase_str)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if not is_resumable_after_claim_release(phase):
|
||||
if not is_stale_claim_releasable(phase):
|
||||
continue
|
||||
metadata = _json_loads(metadata_json, {})
|
||||
metadata["claimed_by_role_session_id"] = ""
|
||||
@@ -5102,13 +5102,26 @@ class OPCStore:
|
||||
|
||||
return total
|
||||
|
||||
async def save_delegation_work_item(self, item: DelegationWorkItem) -> None:
|
||||
async def save_delegation_work_item(
|
||||
self,
|
||||
item: DelegationWorkItem,
|
||||
) -> None:
|
||||
await self._write_delegation_work_item(item, if_absent=False)
|
||||
|
||||
async def _write_delegation_work_item(
|
||||
self,
|
||||
item: DelegationWorkItem,
|
||||
*,
|
||||
if_absent: bool,
|
||||
) -> bool:
|
||||
# Single-source-of-truth gate: every write — whether it goes through
|
||||
# update_delegation_work_item or directly mutates `item.phase` and
|
||||
# then calls save — passes through validate_transition. Skipping the
|
||||
# validation requires a separate code path; there is no way to write
|
||||
# an invalid phase by accident.
|
||||
existing = await self.get_delegation_work_item(item.work_item_id)
|
||||
if if_absent and existing is not None:
|
||||
return False
|
||||
previous_phase = existing.phase if existing is not None else None
|
||||
validate_transition(previous_phase, item.phase)
|
||||
item.metadata = dict(item.metadata or {})
|
||||
@@ -5127,16 +5140,10 @@ class OPCStore:
|
||||
# may want to know "what changed".
|
||||
target_phase = item.phase
|
||||
db = self._require_db()
|
||||
await db.execute(
|
||||
"""INSERT INTO delegation_work_items
|
||||
(work_item_id, run_id, cell_id, team_instance_id, team_id, role_id, seat_id, seat_state_id,
|
||||
role_runtime_session_id, parent_work_item_id, source_role_id, source_seat_id, title, summary,
|
||||
kind, projection_id, phase, batch_id, batch_index,
|
||||
deliverable_summary, blocked_reason, handoff_status, continuation_source, manager_role_id,
|
||||
manager_seat_id, claimed_by_role_runtime_session_id, claimed_by_seat_id, metadata,
|
||||
created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(work_item_id) DO UPDATE SET
|
||||
conflict_action = (
|
||||
"DO NOTHING"
|
||||
if if_absent
|
||||
else """DO UPDATE SET
|
||||
run_id=excluded.run_id,
|
||||
cell_id=excluded.cell_id,
|
||||
team_instance_id=excluded.team_instance_id,
|
||||
@@ -5165,7 +5172,18 @@ class OPCStore:
|
||||
claimed_by_seat_id=excluded.claimed_by_seat_id,
|
||||
metadata=excluded.metadata,
|
||||
created_at=excluded.created_at,
|
||||
updated_at=excluded.updated_at""",
|
||||
updated_at=excluded.updated_at"""
|
||||
)
|
||||
cursor = await db.execute(
|
||||
f"""INSERT INTO delegation_work_items
|
||||
(work_item_id, run_id, cell_id, team_instance_id, team_id, role_id, seat_id, seat_state_id,
|
||||
role_runtime_session_id, parent_work_item_id, source_role_id, source_seat_id, title, summary,
|
||||
kind, projection_id, phase, batch_id, batch_index,
|
||||
deliverable_summary, blocked_reason, handoff_status, continuation_source, manager_role_id,
|
||||
manager_seat_id, claimed_by_role_runtime_session_id, claimed_by_seat_id, metadata,
|
||||
created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT(work_item_id) {conflict_action}""",
|
||||
(
|
||||
item.work_item_id,
|
||||
item.run_id,
|
||||
@@ -5200,6 +5218,8 @@ class OPCStore:
|
||||
),
|
||||
)
|
||||
await db.commit()
|
||||
if if_absent and not (getattr(cursor, "rowcount", 0) or 0):
|
||||
return False
|
||||
# D2 hook fire — propagate phase change to dependent layers
|
||||
# (task.status, role_session.status, dispatcher wake, etc.). All
|
||||
# writes to delegation_work_items pass through here, so this is
|
||||
@@ -5208,6 +5228,20 @@ class OPCStore:
|
||||
await on_phase_transition(previous_phase, target_phase, item, store=self)
|
||||
except Exception: # never let hook failures break the write
|
||||
logger.opt(exception=True).debug("on_phase_transition raised at top level")
|
||||
return True
|
||||
|
||||
async def insert_delegation_work_item_if_absent(
|
||||
self,
|
||||
item: DelegationWorkItem,
|
||||
) -> bool:
|
||||
"""Atomically create a WorkItem without overwriting a concurrent claim.
|
||||
|
||||
Auxiliary report/review IDs are deterministic. A read-before-write
|
||||
check is insufficient because another dispatcher can create and claim
|
||||
the same card between those operations; the conflict decision must be
|
||||
made by SQLite in the insert statement itself.
|
||||
"""
|
||||
return await self._write_delegation_work_item(item, if_absent=True)
|
||||
|
||||
async def list_delegation_work_items(
|
||||
self,
|
||||
@@ -5475,6 +5509,106 @@ class OPCStore:
|
||||
await self.save_delegation_work_item(item)
|
||||
return item
|
||||
|
||||
async def apply_delegation_review_resolution(
|
||||
self,
|
||||
work_item_id: str,
|
||||
*,
|
||||
source_report_work_item_id: str,
|
||||
target_phase: Phase | str,
|
||||
blocked_reason: str,
|
||||
metadata_updates: dict[str, Any],
|
||||
) -> DelegationWorkItem | None:
|
||||
"""Atomically apply a manager verdict to its exact report generation.
|
||||
|
||||
The phase predicate and latest-applied-report predicate live in the
|
||||
same SQLite UPDATE as the child phase + applied-stamp write. This
|
||||
prevents a late manager turn from crossing an AWAITING_HUMAN
|
||||
transition or approving an older report after a newer report landed.
|
||||
``updated_at`` provides optimistic metadata concurrency; a few retries
|
||||
preserve unrelated same-phase updates without weakening either guard.
|
||||
"""
|
||||
target = coerce_phase(target_phase)
|
||||
validate_transition(Phase.AWAITING_MANAGER_REVIEW, target)
|
||||
expected_source = str(source_report_work_item_id or "").strip()
|
||||
db = self._require_db()
|
||||
|
||||
for _attempt in range(3):
|
||||
item = await self.get_delegation_work_item(work_item_id)
|
||||
if item is None or item.phase != Phase.AWAITING_MANAGER_REVIEW:
|
||||
return None
|
||||
metadata = dict(item.metadata or {})
|
||||
metadata.update(dict(metadata_updates or {}))
|
||||
if (
|
||||
self._metadata_has_work_item_projection_identity(metadata)
|
||||
or str(item.projection_id or "").strip()
|
||||
or str(item.kind or "").strip()
|
||||
):
|
||||
metadata, _ = migrate_work_item_projection_metadata(
|
||||
metadata,
|
||||
projection_id_fallback=str(
|
||||
item.projection_id or item.work_item_id or ""
|
||||
).strip(),
|
||||
turn_type_fallback=str(item.kind or "").strip(),
|
||||
)
|
||||
previous_updated_at = item.updated_at.isoformat()
|
||||
updated_at = datetime.now()
|
||||
cursor = await db.execute(
|
||||
"""UPDATE delegation_work_items
|
||||
SET phase = ?, blocked_reason = ?, metadata = ?, updated_at = ?
|
||||
WHERE work_item_id = ?
|
||||
AND phase = ?
|
||||
AND updated_at = ?
|
||||
AND COALESCE((
|
||||
SELECT report.work_item_id
|
||||
FROM delegation_work_items AS report
|
||||
WHERE report.parent_work_item_id = ?
|
||||
AND report.kind = 'report'
|
||||
AND json_extract(
|
||||
report.metadata,
|
||||
'$.report_target_work_item_id'
|
||||
) = ?
|
||||
AND json_extract(
|
||||
report.metadata,
|
||||
'$.report_card_outcome'
|
||||
) = 'applied'
|
||||
ORDER BY report.batch_index DESC,
|
||||
report.created_at DESC,
|
||||
report.work_item_id DESC
|
||||
LIMIT 1
|
||||
), '') = ?""",
|
||||
(
|
||||
target.value,
|
||||
str(blocked_reason or ""),
|
||||
_json_dumps(metadata),
|
||||
updated_at.isoformat(),
|
||||
work_item_id,
|
||||
Phase.AWAITING_MANAGER_REVIEW.value,
|
||||
previous_updated_at,
|
||||
work_item_id,
|
||||
work_item_id,
|
||||
expected_source,
|
||||
),
|
||||
)
|
||||
await db.commit()
|
||||
if not (getattr(cursor, "rowcount", 0) or 0):
|
||||
continue
|
||||
persisted = await self.get_delegation_work_item(work_item_id)
|
||||
if persisted is None:
|
||||
return None
|
||||
try:
|
||||
await on_phase_transition(
|
||||
Phase.AWAITING_MANAGER_REVIEW,
|
||||
target,
|
||||
persisted,
|
||||
store=self,
|
||||
)
|
||||
except Exception:
|
||||
logger.opt(exception=True).debug(
|
||||
"on_phase_transition raised after review-resolution CAS"
|
||||
)
|
||||
return persisted
|
||||
return None
|
||||
|
||||
async def reopen_approved_delegation_work_item_for_rework(
|
||||
self,
|
||||
work_item_id: str,
|
||||
|
||||
+3
-15
@@ -117,6 +117,7 @@ from opc.layer2_organization.session_scoping import (
|
||||
is_top_level_company_session,
|
||||
task_session_scope_id,
|
||||
)
|
||||
from opc.layer2_organization.turn_mode import reset_manager_dispatch_turn_metadata
|
||||
from opc.layer2_organization.seat_executor import EngineSeatExecutor
|
||||
from opc.layer2_organization.work_item_runtime import (
|
||||
is_work_item_runtime_metadata,
|
||||
@@ -6945,11 +6946,7 @@ class OPCEngine:
|
||||
task.metadata.pop("delegation_pending_work_item_ids", None)
|
||||
task.metadata.pop("delegated_children_pending", None)
|
||||
task.metadata.pop("delegation_wait_for_work_item_ids", None)
|
||||
task.metadata.pop("manager_board_mutation_performed", None)
|
||||
task.metadata.pop("manager_board_modified_work_item_ids", None)
|
||||
task.metadata.pop("manager_board_deleted_work_item_ids", None)
|
||||
task.metadata.pop("manager_no_delegation_justification", None)
|
||||
task.metadata.pop("no_delegation_justification", None)
|
||||
task.metadata = reset_manager_dispatch_turn_metadata(task.metadata)
|
||||
progress = list(task.metadata.get("progress_log", []) or [])
|
||||
progress.append(f"Company follow-up routed to final decider ({resume_source}): {reply}")
|
||||
task.metadata["progress_log"] = progress[-20:]
|
||||
@@ -10211,16 +10208,7 @@ class OPCEngine:
|
||||
return True
|
||||
if str(task_metadata.get("manager_no_delegation_justification", "") or "").strip():
|
||||
return True
|
||||
work_item_id = linked_work_item_id_for_task(task)
|
||||
if not work_item_id or not hasattr(self.store, "get_delegation_work_item"):
|
||||
return False
|
||||
work_item = await self.store.get_delegation_work_item(work_item_id)
|
||||
if work_item is None:
|
||||
return False
|
||||
metadata = dict(getattr(work_item, "metadata", {}) or {})
|
||||
if bool(metadata.get("manager_board_mutation_performed", False)):
|
||||
return True
|
||||
if str(metadata.get("manager_no_delegation_justification", "") or "").strip():
|
||||
if str(task_metadata.get("manager_dispatch_guard_unresolved", "") or "").strip():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
+1223
-878
File diff suppressed because it is too large
Load Diff
@@ -131,6 +131,10 @@ _WORK_ITEM_FIELDS: tuple[MetadataFieldSpec, ...] = (
|
||||
_spec("review_attempt", MetadataOwner.WORK_ITEM, allowed_locations=("work_item", "task_execution_copy"), legacy_fallback=True, migration_policy="work_item_wins"),
|
||||
_spec("review_attempt_count", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), legacy_fallback=True, migration_policy="work_item_wins"),
|
||||
_spec("review_target_work_item_id", MetadataOwner.WORK_ITEM, allowed_locations=("work_item", "task_execution_copy"), legacy_fallback=True, migration_policy="work_item_wins"),
|
||||
_spec("review_source_report_work_item_id", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
|
||||
_spec("review_resolution", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
|
||||
_spec("review_resolution_state", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
|
||||
_spec("review_resolution_applied_work_item_id", MetadataOwner.WORK_ITEM, allowed_locations=("work_item",), migration_policy="work_item_wins"),
|
||||
_spec("review_target_worker_task_id", MetadataOwner.WORK_ITEM, allowed_locations=("work_item", "task_execution_copy"), legacy_fallback=True, migration_policy="work_item_wins"),
|
||||
_spec("review_target_worker_role_id", MetadataOwner.WORK_ITEM, allowed_locations=("work_item", "task_execution_copy"), legacy_fallback=True, migration_policy="work_item_wins"),
|
||||
_spec("review_target_worker_seat_id", MetadataOwner.WORK_ITEM, allowed_locations=("work_item", "task_execution_copy"), legacy_fallback=True, migration_policy="work_item_wins"),
|
||||
|
||||
@@ -49,7 +49,9 @@ __all__ = [
|
||||
"phase_for_task_status",
|
||||
"coerce_phase",
|
||||
"is_review_execution_work_item_metadata",
|
||||
"is_runtime_auxiliary_work_item",
|
||||
"should_hide_work_item_from_company_kanban",
|
||||
"is_stale_claim_releasable",
|
||||
"is_resumable_after_claim_release",
|
||||
"is_orphaned",
|
||||
"is_dispatchable",
|
||||
@@ -146,6 +148,27 @@ def is_report_execution_work_item_metadata(metadata: Mapping[str, Any] | None) -
|
||||
return work_kind == "report" and bool(str(data.get("report_target_work_item_id", "") or "").strip())
|
||||
|
||||
|
||||
def is_runtime_auxiliary_work_item(value_or_metadata: Any) -> bool:
|
||||
"""True for attention, report, and review runtime helper cards.
|
||||
|
||||
Accepts either a work-item-like object, a serialized work item containing
|
||||
a ``metadata`` mapping, or the metadata mapping itself. Keeping this
|
||||
identity check below the company runtime gives lifecycle and board code a
|
||||
single dependency-free definition of a non-business child.
|
||||
"""
|
||||
if isinstance(value_or_metadata, Mapping):
|
||||
nested = value_or_metadata.get("metadata")
|
||||
metadata = nested if isinstance(nested, Mapping) else value_or_metadata
|
||||
else:
|
||||
metadata = getattr(value_or_metadata, "metadata", None)
|
||||
data = dict(metadata or {}) if isinstance(metadata, Mapping) else {}
|
||||
return (
|
||||
bool(data.get("attention_work_item", False))
|
||||
or is_report_execution_work_item_metadata(data)
|
||||
or is_review_execution_work_item_metadata(data)
|
||||
)
|
||||
|
||||
|
||||
def should_hide_work_item_from_company_kanban(metadata: Mapping[str, Any] | None) -> bool:
|
||||
"""True when the kanban UI should not display this work item."""
|
||||
data = dict(metadata or {})
|
||||
@@ -316,40 +339,45 @@ def is_runnable(phase: Phase) -> bool:
|
||||
return phase in RUNNABLE_PHASES
|
||||
|
||||
|
||||
# Phases whose runtime claim, when released as stale (process restart, crashed
|
||||
# session), allow the dispatcher to re-pick the card. Without this set, a card
|
||||
# that was actively running when the process died becomes a zombie: phase still
|
||||
# says RUNNING / WAITING_FOR_*, but no session is alive to make progress.
|
||||
_RESUMABLE_AFTER_STALE_CLAIM: frozenset[Phase] = frozenset({
|
||||
Phase.RUNNING,
|
||||
Phase.WAITING_FOR_PEER,
|
||||
Phase.WAITING_FOR_CHILDREN,
|
||||
Phase.PAUSED,
|
||||
Phase.NEEDS_ATTENTION,
|
||||
Phase.AWAITING_MANAGER_REVIEW,
|
||||
Phase.AWAITING_HUMAN,
|
||||
})
|
||||
# A process restart invalidates every persisted runtime claim, including claims
|
||||
# on passive review states. Releasing a dead claim and allowing the original
|
||||
# worker to execute again are intentionally separate decisions: review parents
|
||||
# must lose their dead claim but remain passive while their report/review
|
||||
# auxiliaries resume.
|
||||
_STALE_CLAIM_RELEASABLE_PHASES: frozenset[Phase] = (
|
||||
IN_PROGRESS_PHASES | IN_REVIEW_PHASES
|
||||
)
|
||||
_RESUMABLE_AFTER_CLAIM_RELEASE_PHASES: frozenset[Phase] = IN_PROGRESS_PHASES
|
||||
|
||||
|
||||
def is_stale_claim_releasable(phase: Phase) -> bool:
|
||||
"""Whether startup recovery may clear a dead runtime claim.
|
||||
|
||||
This includes passive review/human-wait states so a crashed resident
|
||||
session cannot retain ownership forever. It does *not* imply that the
|
||||
original work item may be dispatched again.
|
||||
"""
|
||||
return phase in _STALE_CLAIM_RELEASABLE_PHASES
|
||||
|
||||
|
||||
def is_resumable_after_claim_release(phase: Phase) -> bool:
|
||||
"""True iff a stale claim on this phase can be released and the card
|
||||
re-picked up by the dispatcher (rather than left as a zombie).
|
||||
"""Whether the original worker may resume after its claim is released.
|
||||
|
||||
Used by the periodic stale-claim sweeper. Every in-flight phase must
|
||||
return True here — the invariant test in
|
||||
test_phase_state_machine_invariants.py enforces this.
|
||||
Active execution phases can be re-picked after a crash. Passive
|
||||
``AWAITING_*`` parents cannot: their report/review chain (or human) owns
|
||||
forward progress.
|
||||
"""
|
||||
return phase in _RESUMABLE_AFTER_STALE_CLAIM
|
||||
return phase in _RESUMABLE_AFTER_CLAIM_RELEASE_PHASES
|
||||
|
||||
|
||||
def is_orphaned(item: Any) -> bool:
|
||||
"""A work item is orphaned when its phase says 'in flight' but no
|
||||
runtime session currently holds a claim on it.
|
||||
|
||||
Typical scenario: the process that owned the claim died (restart,
|
||||
crash). On startup the stale-claim sweeper clears the claim
|
||||
metadata; this function then lets the dispatcher re-pick the card
|
||||
on the next tick, eliminating zombie work items (Bug C).
|
||||
Typical scenario: the process that owned an execution claim died. On
|
||||
startup the stale-claim sweeper clears the claim metadata; this function
|
||||
then lets the dispatcher re-pick active execution, but never a passive
|
||||
review parent.
|
||||
"""
|
||||
if not is_resumable_after_claim_release(item.phase):
|
||||
return False
|
||||
|
||||
@@ -31,6 +31,7 @@ from opc.core.models import Phase, TaskStatus
|
||||
from opc.layer2_organization.phase import (
|
||||
DONE_PHASES,
|
||||
RUNNABLE_PHASES,
|
||||
is_stale_claim_releasable,
|
||||
register_phase_transition_hook,
|
||||
task_status_for_phase,
|
||||
)
|
||||
@@ -365,6 +366,17 @@ def _active_focus_id(session: Any, work_item_by_id: dict[str, Any]) -> str:
|
||||
return ""
|
||||
if getattr(focused_item, "phase", None) in DONE_PHASES:
|
||||
return ""
|
||||
phase = getattr(focused_item, "phase", None)
|
||||
if is_stale_claim_releasable(phase):
|
||||
claim = str(
|
||||
getattr(focused_item, "claimed_by_role_runtime_session_id", "") or ""
|
||||
).strip()
|
||||
if not claim:
|
||||
# On a live transition the focused item retains its claim. After
|
||||
# restart the startup sweep removes that dead-process claim; the
|
||||
# persisted session focus must stop blocking the orphaned item (or
|
||||
# a passive parent's report/review helper) from being dispatched.
|
||||
return ""
|
||||
return focused
|
||||
|
||||
|
||||
|
||||
@@ -30,6 +30,29 @@ from typing import Any, Mapping
|
||||
from opc.core.models import Phase
|
||||
|
||||
|
||||
MANAGER_DISPATCH_TURN_METADATA_KEYS: tuple[str, ...] = (
|
||||
"manager_board_mutation_performed",
|
||||
"manager_board_modified_work_item_ids",
|
||||
"manager_board_deleted_work_item_ids",
|
||||
"manager_no_delegation_justification",
|
||||
"no_delegation_justification",
|
||||
"manager_dispatch_guard_unresolved",
|
||||
)
|
||||
|
||||
|
||||
def reset_manager_dispatch_turn_metadata(metadata: Mapping[str, Any] | None) -> dict[str, Any]:
|
||||
"""Return mutable metadata with prior manager-turn outcomes removed.
|
||||
|
||||
These keys describe what happened in one agent turn. They must not be
|
||||
carried into a retry, rework turn, or user follow-up; durable board state
|
||||
(dependencies and child mutation revisions) is intentionally untouched.
|
||||
"""
|
||||
result = dict(metadata or {})
|
||||
for key in MANAGER_DISPATCH_TURN_METADATA_KEYS:
|
||||
result.pop(key, None)
|
||||
return result
|
||||
|
||||
|
||||
class TurnMode(str, Enum):
|
||||
EXECUTE = "execute"
|
||||
DELEGATE = "delegate"
|
||||
|
||||
@@ -155,25 +155,13 @@ def is_delivery_turn(value_or_metadata: Any) -> bool:
|
||||
|
||||
|
||||
def is_manager_reviewable_turn(value_or_metadata: Any) -> bool:
|
||||
"""Return True when a finished WorkItem should enter manager review flow.
|
||||
"""Return the turn type's default manager-review policy.
|
||||
|
||||
An explicit persisted ``turn_output_kind`` wins over the turn-type
|
||||
default: delegation-kind turns (dispatch/intake/plan) are review-exempt
|
||||
only because their deliverable is normally a child card set that gets
|
||||
reviewed per child. When such a turn completed with the manager's own
|
||||
work product instead, the done-transition stamps
|
||||
``turn_output_kind=self_produced`` on the WorkItem and that output is
|
||||
reviewable like any execute turn — every consumer of this predicate
|
||||
(DONE routing, report spawn, report completion, recovery scans) follows
|
||||
automatically.
|
||||
Dispatch/intake/plan are normally board-producing turns and therefore
|
||||
exempt. The executor handles the one dynamic exception — a current turn
|
||||
that produced no board mutation — before it writes the authoritative
|
||||
WorkItem phase. Downstream review plumbing follows that phase directly.
|
||||
"""
|
||||
metadata: Mapping[str, Any] | None = None
|
||||
if isinstance(value_or_metadata, Mapping):
|
||||
metadata = value_or_metadata
|
||||
elif hasattr(value_or_metadata, "metadata"):
|
||||
metadata = getattr(value_or_metadata, "metadata", None) or {}
|
||||
if metadata and str(metadata.get("turn_output_kind", "") or "").strip().lower() == "self_produced":
|
||||
return True
|
||||
turn_type = _turn_type_for_value(value_or_metadata, fallback="")
|
||||
if not turn_type:
|
||||
return False
|
||||
|
||||
@@ -2230,6 +2230,11 @@ def create_collaboration_tools(
|
||||
task.metadata = dict(task.metadata)
|
||||
task.metadata["delegation_wait_for_work_item_ids"] = parent_dependency_ids
|
||||
task.metadata["manager_board_parent_work_item_id"] = parent_work_item_id
|
||||
# Reusing an existing scope-key match is not a current-turn board
|
||||
# mutation. Only newly persisted business children make this
|
||||
# attempt a delegated-board completion.
|
||||
if resolved_pending_items:
|
||||
task.metadata["manager_board_mutation_performed"] = True
|
||||
if attention_work_item_id:
|
||||
task.metadata["attention_business_parent_work_item_id"] = parent_work_item_id
|
||||
task.metadata["attention_work_item_id"] = attention_work_item_id
|
||||
|
||||
@@ -967,6 +967,7 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(issues, [])
|
||||
self.assertTrue(self.task.metadata.get("manager_board_mutation_performed"))
|
||||
|
||||
async def test_manager_dispatch_guard_accepts_existing_child_mutation(self) -> None:
|
||||
await self.store.save_delegation_work_item(
|
||||
@@ -1012,6 +1013,114 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(issues, [])
|
||||
self.assertTrue(self.task.metadata.get("manager_board_mutation_performed"))
|
||||
|
||||
async def test_manager_dispatch_guard_accepts_existing_child_deletion(self) -> None:
|
||||
await self.store.save_delegation_work_item(
|
||||
DelegationWorkItem(
|
||||
work_item_id="cto-child-item",
|
||||
run_id="run-1",
|
||||
cell_id="team::ceo",
|
||||
team_instance_id="team-instance::run-1::team::ceo",
|
||||
team_id="team::ceo",
|
||||
role_id="cto",
|
||||
seat_id="seat::team::ceo::cto",
|
||||
seat_state_id="seat-state::run-1::seat::team::ceo::cto",
|
||||
role_runtime_session_id="role-runtime::run-1::seat::team::ceo::cto",
|
||||
parent_work_item_id="ceo-work-item",
|
||||
title="Obsolete CTO Work",
|
||||
summary="Remove this obsolete branch.",
|
||||
kind="execute",
|
||||
projection_id="cto-child-item",
|
||||
phase=Phase.RUNNING,
|
||||
manager_role_id="ceo",
|
||||
manager_seat_id="seat::team::ceo::ceo",
|
||||
metadata={
|
||||
"work_item_runtime": True,
|
||||
"runtime_model": "multi_team_org",
|
||||
"manager_mutation_revision": 0,
|
||||
},
|
||||
)
|
||||
)
|
||||
before = await self.executor._snapshot_manager_dispatch_state(self.task)
|
||||
await self.store.amend_delegation_work_item(
|
||||
"cto-child-item",
|
||||
metadata_set={
|
||||
"manager_mutation_revision": 1,
|
||||
"manager_mutation_action": "delete",
|
||||
"deleted_by_manager_tool": True,
|
||||
"hidden_from_company_kanban": True,
|
||||
"upstream_visibility": "hidden",
|
||||
},
|
||||
)
|
||||
|
||||
issues = await self.executor._enforce_manager_dispatch_guard(
|
||||
self.task,
|
||||
TaskResult(status=TaskStatus.DONE, content="Deleted the obsolete child work item."),
|
||||
before_state=before,
|
||||
)
|
||||
|
||||
self.assertEqual(issues, [])
|
||||
self.assertTrue(self.task.metadata.get("manager_board_mutation_performed"))
|
||||
|
||||
async def test_historical_dependency_does_not_satisfy_current_turn_guard(self) -> None:
|
||||
await self.store.save_delegation_work_item(
|
||||
DelegationWorkItem(
|
||||
work_item_id="historical-child",
|
||||
run_id="run-1",
|
||||
parent_work_item_id="ceo-work-item",
|
||||
role_id="cto",
|
||||
seat_id="seat::team::ceo::cto",
|
||||
title="Historical child",
|
||||
summary="Created in an earlier turn.",
|
||||
kind="execute",
|
||||
projection_id="historical-child",
|
||||
phase=Phase.APPROVED,
|
||||
metadata={"work_item_runtime": True, "runtime_model": "multi_team_org"},
|
||||
)
|
||||
)
|
||||
self.task.metadata["delegation_wait_for_work_item_ids"] = ["historical-child"]
|
||||
before = await self.executor._snapshot_manager_dispatch_state(self.task)
|
||||
|
||||
issues = await self.executor._enforce_manager_dispatch_guard(
|
||||
self.task,
|
||||
TaskResult(status=TaskStatus.DONE, content="Handled this new request directly."),
|
||||
before_state=before,
|
||||
)
|
||||
|
||||
self.assertEqual(len(issues), 1)
|
||||
self.assertFalse(self.task.metadata.get("manager_board_mutation_performed", False))
|
||||
|
||||
async def test_attention_child_created_during_turn_does_not_satisfy_dispatch_guard(self) -> None:
|
||||
before = await self.executor._snapshot_manager_dispatch_state(self.task)
|
||||
await self.store.save_delegation_work_item(
|
||||
DelegationWorkItem(
|
||||
work_item_id="ceo-attention-item",
|
||||
run_id="run-1",
|
||||
parent_work_item_id="ceo-work-item",
|
||||
role_id="ceo",
|
||||
seat_id="seat::team::ceo::ceo",
|
||||
title="CEO attention",
|
||||
summary="Runtime wake-up wrapper, not delegated business work.",
|
||||
kind="monitor",
|
||||
projection_id="ceo-attention-item",
|
||||
phase=Phase.READY,
|
||||
metadata={
|
||||
"work_item_runtime": True,
|
||||
"runtime_model": "multi_team_org",
|
||||
"attention_work_item": True,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
issues = await self.executor._enforce_manager_dispatch_guard(
|
||||
self.task,
|
||||
TaskResult(status=TaskStatus.DONE, content="Handled this request directly."),
|
||||
before_state=before,
|
||||
)
|
||||
|
||||
self.assertEqual(len(issues), 1)
|
||||
self.assertFalse(self.task.metadata.get("manager_board_mutation_performed", False))
|
||||
|
||||
async def test_manager_dispatch_guard_exhaustion_accepts_turn_instead_of_failing(self) -> None:
|
||||
# Dispatch is a soft constraint: when the guard reminders run out,
|
||||
@@ -1029,6 +1138,11 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
|
||||
# per-turn reset must clear it, otherwise the guard would
|
||||
# silently pass without reminders.
|
||||
"manager_board_mutation_performed": True,
|
||||
"manager_board_modified_work_item_ids": ["stale-modified-child"],
|
||||
"manager_board_deleted_work_item_ids": ["stale-deleted-child"],
|
||||
"manager_no_delegation_justification": "stale reason from an earlier turn",
|
||||
"no_delegation_justification": "another stale reason",
|
||||
"manager_dispatch_guard_unresolved": "stale unresolved guard",
|
||||
},
|
||||
)
|
||||
task.status = TaskStatus.PENDING
|
||||
@@ -1054,13 +1168,22 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
|
||||
# card waits for CEO review and a live report card drives it.
|
||||
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
|
||||
self.assertEqual(work_item.phase, Phase.AWAITING_MANAGER_REVIEW)
|
||||
self.assertEqual(
|
||||
str((work_item.metadata or {}).get("turn_output_kind", "")), "self_produced"
|
||||
self.assertNotIn("manager_board_mutation_performed", task.metadata)
|
||||
self.assertNotIn("manager_board_modified_work_item_ids", task.metadata)
|
||||
self.assertNotIn("manager_board_deleted_work_item_ids", task.metadata)
|
||||
self.assertNotIn("manager_no_delegation_justification", task.metadata)
|
||||
self.assertNotIn("no_delegation_justification", task.metadata)
|
||||
self.assertNotIn("stale unresolved guard", task.metadata["manager_dispatch_guard_unresolved"])
|
||||
dispatch_evidence = dict(
|
||||
dict((work_item.metadata or {}).get("review_evidence", {}) or {}).get(
|
||||
"manager_dispatch", {}
|
||||
)
|
||||
self.assertEqual(
|
||||
str((work_item.metadata or {}).get("turn_output_source", "")),
|
||||
"dispatch_guard_exhausted",
|
||||
or {}
|
||||
)
|
||||
self.assertEqual(dispatch_evidence.get("outcome"), "self_produced")
|
||||
self.assertEqual(dispatch_evidence.get("source"), "dispatch_guard_exhausted")
|
||||
self.assertNotIn("turn_output_kind", work_item.metadata or {})
|
||||
self.assertNotIn("turn_output_source", work_item.metadata or {})
|
||||
report_cards = await self._aux_cards_targeting("cto-dispatch-item", "report_target_work_item_id")
|
||||
self.assertEqual(len(report_cards), 1)
|
||||
self.assertNotIn(report_cards[0].phase, DONE_PHASES)
|
||||
@@ -1131,15 +1254,29 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(phase, Phase.AWAITING_MANAGER_REVIEW)
|
||||
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
|
||||
self.assertEqual(work_item.phase, Phase.AWAITING_MANAGER_REVIEW)
|
||||
self.assertEqual(
|
||||
str((work_item.metadata or {}).get("turn_output_source", "")), "justified"
|
||||
dispatch_evidence = dict(
|
||||
dict((work_item.metadata or {}).get("review_evidence", {}) or {}).get(
|
||||
"manager_dispatch", {}
|
||||
)
|
||||
or {}
|
||||
)
|
||||
self.assertEqual(dispatch_evidence.get("outcome"), "self_produced")
|
||||
self.assertEqual(dispatch_evidence.get("source"), "justified")
|
||||
self.assertEqual(dispatch_evidence.get("note"), "single-seat scoping decision")
|
||||
self.assertNotIn("turn_output_kind", work_item.metadata or {})
|
||||
self.assertNotIn("turn_output_source", work_item.metadata or {})
|
||||
report_cards = await self._aux_cards_targeting("cto-dispatch-item", "report_target_work_item_id")
|
||||
self.assertEqual(len(report_cards), 1)
|
||||
report_card = report_cards[0]
|
||||
self.assertNotIn(report_card.phase, DONE_PHASES)
|
||||
|
||||
# Drive the report turn to completion — the review card must appear.
|
||||
# Production dispatch claims READY → RUNNING before materializing
|
||||
# the Task; this direct lifecycle test must model that claim.
|
||||
await self.store.update_delegation_work_item(
|
||||
report_card.work_item_id,
|
||||
phase=Phase.RUNNING,
|
||||
)
|
||||
report_task = Task(
|
||||
id="cto-report-task",
|
||||
title=report_card.title,
|
||||
@@ -1165,16 +1302,26 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(len(review_cards), 1)
|
||||
self.assertNotIn(review_cards[0].phase, DONE_PHASES)
|
||||
self.assertEqual(str(review_cards[0].role_id or ""), "ceo")
|
||||
review_dispatch_evidence = dict(
|
||||
dict((review_cards[0].metadata or {}).get("review_evidence", {}) or {}).get(
|
||||
"manager_dispatch", {}
|
||||
)
|
||||
or {}
|
||||
)
|
||||
self.assertEqual(review_dispatch_evidence.get("source"), "justified")
|
||||
|
||||
async def test_dispatch_turn_that_delegated_keeps_auto_approve(self) -> None:
|
||||
# Normal delegation flow: a live child card exists in the store, so
|
||||
# the dispatch exemption applies (children carry the reviewable
|
||||
# output) — classification comes from store ground truth, not from
|
||||
# transient task markers.
|
||||
task = await self._make_cto_dispatch_task()
|
||||
async def test_historical_business_child_does_not_exempt_current_self_output(self) -> None:
|
||||
# A child created by a previous attempt is durable board state, not
|
||||
# proof that this completion delegated its output. The current direct
|
||||
# work product still needs the manager review chain.
|
||||
task = await self._make_cto_dispatch_task(
|
||||
metadata_extra={
|
||||
"manager_no_delegation_justification": "This new request is a direct architecture decision."
|
||||
}
|
||||
)
|
||||
await self.store.save_delegation_work_item(
|
||||
DelegationWorkItem(
|
||||
work_item_id="cto-delegated-child",
|
||||
work_item_id="cto-historical-child",
|
||||
run_id="run-1",
|
||||
cell_id="team::ceo",
|
||||
team_instance_id="team-instance::run-1::team::ceo",
|
||||
@@ -1184,26 +1331,93 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
|
||||
seat_state_id="seat-state::run-1::seat::team::ceo::cto",
|
||||
role_runtime_session_id="role-runtime::run-1::seat::team::ceo::cto",
|
||||
parent_work_item_id="cto-dispatch-item",
|
||||
title="Delegated child",
|
||||
summary="Build the feature.",
|
||||
title="Historical delegated child",
|
||||
summary="Completed in a previous attempt.",
|
||||
kind="execute",
|
||||
projection_id="cto-delegated-child",
|
||||
phase=Phase.READY,
|
||||
projection_id="cto-historical-child",
|
||||
phase=Phase.APPROVED,
|
||||
manager_role_id="cto",
|
||||
manager_seat_id="seat::team::ceo::cto",
|
||||
metadata={"work_item_runtime": True, "runtime_model": "multi_team_org"},
|
||||
)
|
||||
)
|
||||
|
||||
phase = await self.executor._apply_done_transition(
|
||||
task,
|
||||
result=TaskResult(
|
||||
status=TaskStatus.DONE,
|
||||
content="Made the new architecture decision directly.",
|
||||
),
|
||||
)
|
||||
|
||||
self.assertEqual(phase, Phase.AWAITING_MANAGER_REVIEW)
|
||||
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
|
||||
report_cards = await self._aux_cards_targeting("cto-dispatch-item", "report_target_work_item_id")
|
||||
self.assertEqual(len(report_cards), 1)
|
||||
dispatch_evidence = dict(
|
||||
dict((work_item.metadata or {}).get("review_evidence", {}) or {}).get(
|
||||
"manager_dispatch", {}
|
||||
)
|
||||
or {}
|
||||
)
|
||||
self.assertEqual(dispatch_evidence.get("outcome"), "self_produced")
|
||||
|
||||
async def test_attention_aux_does_not_exempt_current_self_output(self) -> None:
|
||||
task = await self._make_cto_dispatch_task(
|
||||
metadata_extra={
|
||||
"manager_no_delegation_justification": "This request requires a direct architecture decision."
|
||||
}
|
||||
)
|
||||
await self.store.save_delegation_work_item(
|
||||
DelegationWorkItem(
|
||||
work_item_id="cto-attention-item",
|
||||
run_id="run-1",
|
||||
parent_work_item_id="cto-dispatch-item",
|
||||
role_id="cto",
|
||||
seat_id="seat::team::ceo::cto",
|
||||
title="CTO attention",
|
||||
summary="Runtime wake-up wrapper, not delegated business work.",
|
||||
kind="monitor",
|
||||
projection_id="cto-attention-item",
|
||||
phase=Phase.APPROVED,
|
||||
metadata={
|
||||
"work_item_runtime": True,
|
||||
"runtime_model": "multi_team_org",
|
||||
"attention_work_item": True,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
phase = await self.executor._apply_done_transition(
|
||||
task,
|
||||
result=TaskResult(
|
||||
status=TaskStatus.DONE,
|
||||
content="Made the architecture decision directly.",
|
||||
),
|
||||
)
|
||||
|
||||
self.assertEqual(phase, Phase.AWAITING_MANAGER_REVIEW)
|
||||
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
|
||||
self.assertEqual(work_item.phase, Phase.AWAITING_MANAGER_REVIEW)
|
||||
self.assertEqual(
|
||||
len(await self._aux_cards_targeting("cto-dispatch-item", "report_target_work_item_id")),
|
||||
1,
|
||||
)
|
||||
|
||||
async def test_current_turn_business_board_mutation_keeps_dispatch_auto_approve(self) -> None:
|
||||
task = await self._make_cto_dispatch_task(
|
||||
metadata_extra={"manager_board_mutation_performed": True}
|
||||
)
|
||||
|
||||
phase = await self.executor._apply_done_transition(
|
||||
task, result=TaskResult(status=TaskStatus.DONE, content="Delegated to the team."),
|
||||
)
|
||||
|
||||
self.assertEqual(phase, Phase.APPROVED)
|
||||
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
|
||||
self.assertEqual(
|
||||
str((work_item.metadata or {}).get("turn_output_kind", "")), "delegated"
|
||||
)
|
||||
self.assertEqual(work_item.phase, Phase.APPROVED)
|
||||
self.assertNotIn("turn_output_kind", work_item.metadata or {})
|
||||
self.assertNotIn("turn_output_source", work_item.metadata or {})
|
||||
report_cards = await self._aux_cards_targeting("cto-dispatch-item", "report_target_work_item_id")
|
||||
self.assertEqual(report_cards, [])
|
||||
|
||||
@@ -1219,15 +1433,15 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
self.assertEqual(phase, Phase.AWAITING_MANAGER_REVIEW)
|
||||
work_item = await self.store.get_delegation_work_item("cto-dispatch-item")
|
||||
self.assertEqual(
|
||||
str((work_item.metadata or {}).get("turn_output_kind", "")), "self_produced"
|
||||
)
|
||||
self.assertEqual(work_item.phase, Phase.AWAITING_MANAGER_REVIEW)
|
||||
self.assertNotIn("turn_output_kind", work_item.metadata or {})
|
||||
self.assertNotIn("turn_output_source", work_item.metadata or {})
|
||||
|
||||
async def test_top_seat_self_produced_falls_back_to_auto_approve(self) -> None:
|
||||
# The CEO has no manager to review: the existing no-reviewer
|
||||
# fallback auto-approves instead of stranding the card.
|
||||
self.task.status = TaskStatus.DONE
|
||||
self.task.metadata["work_kind"] = "dispatch"
|
||||
self.task.metadata["work_kind"] = "intake"
|
||||
self.task.metadata["manager_dispatch_guard_unresolved"] = "no children"
|
||||
await self.store.update_delegation_work_item("ceo-work-item", phase=Phase.RUNNING)
|
||||
|
||||
@@ -1236,20 +1450,25 @@ class ActorRuntimeManagerDispatchGuardTests(unittest.IsolatedAsyncioTestCase):
|
||||
)
|
||||
|
||||
self.assertEqual(phase, Phase.APPROVED)
|
||||
work_item = await self.store.get_delegation_work_item("ceo-work-item")
|
||||
self.assertEqual(work_item.phase, Phase.APPROVED)
|
||||
self.assertEqual(
|
||||
await self._aux_cards_targeting("ceo-work-item", "report_target_work_item_id"),
|
||||
[],
|
||||
)
|
||||
self.assertEqual(
|
||||
await self._aux_cards_targeting("ceo-work-item", "review_target_work_item_id"),
|
||||
[],
|
||||
)
|
||||
|
||||
async def test_reconcile_rebuilds_missing_report_card(self) -> None:
|
||||
# Legacy shape: card already parked in AWAITING_MANAGER_REVIEW with
|
||||
# the self_produced marker but no live report/review card (the
|
||||
# historical spawn refused dispatch parents). The dispatcher-tick
|
||||
# reconcile must rebuild the report card idempotently.
|
||||
# Current-state crash shape: phase was durably written but the process
|
||||
# stopped before its report card was saved. Phase alone is the
|
||||
# authoritative recovery fact; no output-kind marker is required.
|
||||
await self._make_cto_dispatch_task()
|
||||
await self.store.update_delegation_work_item(
|
||||
"cto-dispatch-item",
|
||||
phase=Phase.AWAITING_MANAGER_REVIEW,
|
||||
metadata_updates={
|
||||
"turn_output_kind": "self_produced",
|
||||
"turn_output_source": "dispatch_guard_exhausted",
|
||||
},
|
||||
)
|
||||
run_items = await self.store.list_delegation_work_items("run-1")
|
||||
|
||||
|
||||
@@ -1067,12 +1067,21 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertTrue(await engine._company_followup_target_progressed("task-ceo"))
|
||||
|
||||
task.metadata.pop("manager_no_delegation_justification", None)
|
||||
task.metadata["manager_dispatch_guard_unresolved"] = (
|
||||
"Soft dispatch constraint exhausted; accept the top-seat output."
|
||||
)
|
||||
await store.save_task(task)
|
||||
self.assertTrue(await engine._company_followup_target_progressed("task-ceo"))
|
||||
|
||||
task.metadata.pop("manager_dispatch_guard_unresolved", None)
|
||||
await store.save_task(task)
|
||||
await store.update_delegation_work_item(
|
||||
"wi-ceo",
|
||||
metadata_updates={"manager_board_mutation_performed": True},
|
||||
)
|
||||
self.assertTrue(await engine._company_followup_target_progressed("task-ceo"))
|
||||
# Dispatch outcome markers are attempt-scoped Task state. A stale
|
||||
# WorkItem copy must not make a later follow-up look progressed.
|
||||
self.assertFalse(await engine._company_followup_target_progressed("task-ceo"))
|
||||
|
||||
async def test_final_decider_followup_keeps_dispatch_turn_mode_with_existing_children(self) -> None:
|
||||
runtime = CompanyRuntime(org_engine=None, communication=None, store=None)
|
||||
|
||||
@@ -380,7 +380,11 @@ class ReviewWorkItemLifecycleTests(unittest.IsolatedAsyncioTestCase):
|
||||
}
|
||||
await store.save_delegation_work_item(child_work_item)
|
||||
|
||||
await executor._close_review_work_item_for_work_item("wi-child", outcome=Phase.FAILED.value)
|
||||
await executor._persist_terminal_review_card(
|
||||
review_work_item_id,
|
||||
phase=Phase.CANCELLED,
|
||||
outcome=Phase.FAILED.value,
|
||||
)
|
||||
refreshed_review = await store.get_delegation_work_item(review_work_item_id)
|
||||
assert refreshed_review is not None
|
||||
self.assertEqual(refreshed_review.phase, Phase.CANCELLED)
|
||||
|
||||
@@ -83,6 +83,40 @@ class MetadataOwnershipMatrixTests(unittest.TestCase):
|
||||
self.assertNotIn("progress_log", copied)
|
||||
self.assertIn("current_turn_mode", EXECUTION_COPY_KEYS)
|
||||
|
||||
def test_review_source_report_link_is_work_item_only_not_execution_copy(self) -> None:
|
||||
key = "review_source_report_work_item_id"
|
||||
item = _work_item(
|
||||
{
|
||||
key: "report::wi-1::v2",
|
||||
"review_target_work_item_id": "wi-1",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual(metadata_owner_for_key(key), MetadataOwner.WORK_ITEM)
|
||||
self.assertTrue(is_work_item_owned_key(key))
|
||||
self.assertNotIn(key, EXECUTION_COPY_KEYS)
|
||||
|
||||
copied = copy_work_item_execution_metadata(item)
|
||||
|
||||
self.assertNotIn(key, copied)
|
||||
self.assertEqual(copied["review_target_work_item_id"], "wi-1")
|
||||
|
||||
task = Task(id="task-1", metadata={key: "report::wi-1::v2"})
|
||||
removed = strip_disallowed_work_item_metadata_from_runtime_task(task)
|
||||
self.assertIn(key, removed)
|
||||
self.assertNotIn(key, task.metadata)
|
||||
|
||||
for journal_key in (
|
||||
"review_resolution",
|
||||
"review_resolution_state",
|
||||
"review_resolution_applied_work_item_id",
|
||||
):
|
||||
self.assertEqual(
|
||||
metadata_owner_for_key(journal_key),
|
||||
MetadataOwner.WORK_ITEM,
|
||||
)
|
||||
self.assertNotIn(journal_key, EXECUTION_COPY_KEYS)
|
||||
|
||||
def test_validate_metadata_ownership_reports_task_only_work_item_field(self) -> None:
|
||||
item = _work_item({})
|
||||
task = Task(
|
||||
|
||||
@@ -30,7 +30,10 @@ from opc.layer2_organization.phase import (
|
||||
TERMINAL_PHASES,
|
||||
TODO_PHASES,
|
||||
coerce_phase,
|
||||
is_resumable_after_claim_release,
|
||||
is_runnable,
|
||||
is_runtime_auxiliary_work_item,
|
||||
is_stale_claim_releasable,
|
||||
is_terminal,
|
||||
kanban_column,
|
||||
validate_transition,
|
||||
@@ -253,20 +256,49 @@ def test_review_work_item_id_per_attempt_is_unique() -> None:
|
||||
# ── H: in-flight phase recoverability after stale claim release ───────────
|
||||
|
||||
|
||||
def test_inflight_phase_recoverable_after_claim_release() -> None:
|
||||
"""Every in-flight phase (RUNNING / WAITING_FOR_*) must be re-claimable
|
||||
once the previous claim is released (e.g. after process restart).
|
||||
def test_inflight_claims_are_releasable_but_only_execution_is_resumable() -> None:
|
||||
"""Claim cleanup and worker re-execution are separate invariants.
|
||||
|
||||
Without this invariant, any restart leaves in-flight cards as zombies
|
||||
that the dispatcher refuses to pick up. The recovery path is provided
|
||||
by `is_resumable_after_claim_release` defined in `phase.py`.
|
||||
All in-flight claims belong to the dead process and must be released on
|
||||
restart. Active execution may then resume; passive review/human parents
|
||||
must wait for their auxiliary or external actor instead of running twice.
|
||||
"""
|
||||
from opc.layer2_organization.phase import is_resumable_after_claim_release
|
||||
|
||||
inflight = IN_PROGRESS_PHASES | IN_REVIEW_PHASES
|
||||
not_recoverable = [p.value for p in inflight if not is_resumable_after_claim_release(p)]
|
||||
assert not not_recoverable, (
|
||||
f"in-flight phases that cannot recover after a stale claim is "
|
||||
f"released: {not_recoverable}. After process restart these cards "
|
||||
f"would become permanent zombies."
|
||||
not_releasable = [p.value for p in inflight if not is_stale_claim_releasable(p)]
|
||||
assert not not_releasable, (
|
||||
f"in-flight phases whose dead claims cannot be released: {not_releasable}"
|
||||
)
|
||||
not_resumable = [
|
||||
p.value for p in IN_PROGRESS_PHASES
|
||||
if not is_resumable_after_claim_release(p)
|
||||
]
|
||||
assert not not_resumable, (
|
||||
f"active execution phases that cannot resume: {not_resumable}"
|
||||
)
|
||||
incorrectly_resumable = [
|
||||
p.value for p in IN_REVIEW_PHASES
|
||||
if is_resumable_after_claim_release(p)
|
||||
]
|
||||
assert not incorrectly_resumable, (
|
||||
f"passive review phases incorrectly resume worker execution: "
|
||||
f"{incorrectly_resumable}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value",
|
||||
[
|
||||
{"attention_work_item": True},
|
||||
{"report_execution_work_item": True},
|
||||
{"review_execution_work_item": True},
|
||||
{"work_kind": "report", "report_target_work_item_id": "parent"},
|
||||
{"work_kind": "review", "review_target_work_item_id": "parent"},
|
||||
{"metadata": {"attention_work_item": True}},
|
||||
],
|
||||
)
|
||||
def test_runtime_auxiliary_identity_is_canonical(value: dict[str, object]) -> None:
|
||||
assert is_runtime_auxiliary_work_item(value)
|
||||
|
||||
|
||||
def test_business_work_item_is_not_runtime_auxiliary() -> None:
|
||||
assert not is_runtime_auxiliary_work_item({"work_kind": "execute"})
|
||||
|
||||
@@ -214,9 +214,7 @@ class RefreshDependentsForRunTests(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
executor = self._executor()
|
||||
executor._active_tasks = [manager_task]
|
||||
created = await executor._materialize_follow_up_work_items(
|
||||
manager_task,
|
||||
TaskResult(
|
||||
follow_up_result = TaskResult(
|
||||
status=TaskStatus.DONE,
|
||||
content="Create a PPT deck.",
|
||||
artifacts={
|
||||
@@ -230,7 +228,10 @@ class RefreshDependentsForRunTests(unittest.IsolatedAsyncioTestCase):
|
||||
}
|
||||
]
|
||||
},
|
||||
),
|
||||
)
|
||||
created = await executor._materialize_follow_up_work_items(
|
||||
manager_task,
|
||||
follow_up_result,
|
||||
)
|
||||
|
||||
self.assertEqual(len(created), 1)
|
||||
@@ -239,6 +240,20 @@ class RefreshDependentsForRunTests(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(follow.phase, Phase.READY)
|
||||
self.assertEqual(parent_after.phase, Phase.WAITING_FOR_CHILDREN)
|
||||
|
||||
# Re-emitting the same dedupe key reuses board state; it is not a
|
||||
# current-turn creation signal for the dispatch guard.
|
||||
reused = await executor._materialize_follow_up_work_items(
|
||||
manager_task,
|
||||
follow_up_result,
|
||||
)
|
||||
self.assertEqual(reused, [])
|
||||
follow_ups = [
|
||||
item
|
||||
for item in await self.store.list_delegation_work_items("run-materialize")
|
||||
if (item.metadata or {}).get("follow_up_dedupe_key")
|
||||
]
|
||||
self.assertEqual(len(follow_ups), 1)
|
||||
|
||||
async def test_parent_wakes_when_last_child_approved(self) -> None:
|
||||
"""The canonical app12 fix: parent in WAITING_FOR_CHILDREN with a
|
||||
stale claim unblocks to RUNNING and releases the claim when all
|
||||
|
||||
@@ -228,6 +228,18 @@ class IsDispatchableQueueFilterTests(unittest.TestCase):
|
||||
)
|
||||
self.assertFalse(is_dispatchable(item))
|
||||
|
||||
def test_passive_review_parent_without_claim_is_not_dispatchable(self) -> None:
|
||||
for phase in (Phase.AWAITING_MANAGER_REVIEW, Phase.AWAITING_HUMAN):
|
||||
with self.subTest(phase=phase):
|
||||
item = DelegationWorkItem(
|
||||
work_item_id=f"wi-{phase.value}",
|
||||
run_id="r", cell_id="c", role_id="cto",
|
||||
seat_id="seat", manager_role_id="ceo",
|
||||
manager_seat_id="seat::ceo", title="t",
|
||||
phase=phase,
|
||||
)
|
||||
self.assertFalse(is_dispatchable(item))
|
||||
|
||||
|
||||
# ── enqueue_session_work_on_runnable_hook ────────────────────────────────
|
||||
|
||||
@@ -467,7 +479,10 @@ class SerialQueueReconcilerTests(_StoreFixture):
|
||||
async def test_reconcile_preserves_valid_marker_for_busy_session(self) -> None:
|
||||
self.store.role_serial_queue_enabled = True
|
||||
sid = await self._seed_session(focused="wi-active", pending=["wi-next"])
|
||||
await self._save_item(wid="wi-active", sid=sid, phase=Phase.RUNNING)
|
||||
active = await self._save_item(wid="wi-active", sid=sid, phase=Phase.RUNNING)
|
||||
active.claimed_by_role_runtime_session_id = sid
|
||||
active.claimed_by_seat_id = "seat"
|
||||
await self.store.save_delegation_work_item(active)
|
||||
await self._save_item(wid="wi-next", sid=sid, marker=True)
|
||||
|
||||
result = await reconcile_role_serial_queues(self.store, "r")
|
||||
@@ -479,6 +494,64 @@ class SerialQueueReconcilerTests(_StoreFixture):
|
||||
self.assertEqual(item.metadata.get("queued_behind_session"), sid)
|
||||
self.assertFalse(is_dispatchable(item))
|
||||
|
||||
async def test_reconcile_promotes_aux_behind_unclaimed_review_focus(self) -> None:
|
||||
self.store.role_serial_queue_enabled = True
|
||||
sid = await self._seed_session(
|
||||
focused="wi-parent",
|
||||
pending=["wi-report"],
|
||||
status="running",
|
||||
)
|
||||
await self._save_item(
|
||||
wid="wi-parent",
|
||||
sid=sid,
|
||||
phase=Phase.AWAITING_MANAGER_REVIEW,
|
||||
)
|
||||
report = await self._save_item(wid="wi-report", sid=sid, marker=True)
|
||||
report.metadata.update({
|
||||
"report_execution_work_item": True,
|
||||
"report_target_work_item_id": "wi-parent",
|
||||
})
|
||||
await self.store.save_delegation_work_item(report)
|
||||
|
||||
result = await reconcile_role_serial_queues(self.store, "r")
|
||||
|
||||
self.assertIn(sid, result["cleared_focus_session_ids"])
|
||||
self.assertIn("wi-report", result["promoted_work_item_ids"])
|
||||
session = await self.store.get_delegation_role_session(sid)
|
||||
self.assertEqual(session.focused_work_item_id, "")
|
||||
self.assertEqual(session.pending_work_item_ids, [])
|
||||
report_after = await self.store.get_delegation_work_item("wi-report")
|
||||
self.assertNotIn("queued_behind_session", report_after.metadata)
|
||||
self.assertTrue(is_dispatchable(report_after))
|
||||
|
||||
async def test_reconcile_preserves_live_claimed_review_focus(self) -> None:
|
||||
self.store.role_serial_queue_enabled = True
|
||||
sid = await self._seed_session(
|
||||
focused="wi-parent",
|
||||
pending=["wi-report"],
|
||||
status="running",
|
||||
)
|
||||
parent = await self._save_item(
|
||||
wid="wi-parent",
|
||||
sid=sid,
|
||||
phase=Phase.AWAITING_MANAGER_REVIEW,
|
||||
)
|
||||
parent.claimed_by_role_runtime_session_id = sid
|
||||
parent.claimed_by_seat_id = "seat"
|
||||
await self.store.save_delegation_work_item(parent)
|
||||
await self._save_item(wid="wi-report", sid=sid, marker=True)
|
||||
|
||||
result = await reconcile_role_serial_queues(self.store, "r")
|
||||
|
||||
self.assertEqual(result["cleared_focus_session_ids"], [])
|
||||
self.assertEqual(result["promoted_work_item_ids"], [])
|
||||
session = await self.store.get_delegation_role_session(sid)
|
||||
self.assertEqual(session.focused_work_item_id, "wi-parent")
|
||||
self.assertEqual(session.pending_work_item_ids, ["wi-report"])
|
||||
report = await self.store.get_delegation_work_item("wi-report")
|
||||
self.assertEqual(report.metadata.get("queued_behind_session"), sid)
|
||||
self.assertFalse(is_dispatchable(report))
|
||||
|
||||
async def test_reconcile_prunes_dead_entries_and_promotes_one_head(self) -> None:
|
||||
self.store.role_serial_queue_enabled = True
|
||||
sid = "role-runtime::r::cto"
|
||||
@@ -559,6 +632,106 @@ class SerialQueueReconcilerTests(_StoreFixture):
|
||||
self.assertEqual(repaired.status, TaskStatus.DONE)
|
||||
|
||||
|
||||
class StartupClaimSweepTests(_StoreFixture):
|
||||
async def test_restart_clears_dead_running_focus_so_auxiliary_can_resume(self) -> None:
|
||||
sid = "role-runtime::r::cto"
|
||||
report_id = "report::wi-parent::v1"
|
||||
await self.store.save_delegation_role_session(
|
||||
DelegationRoleSession(
|
||||
role_session_id=sid,
|
||||
run_id="r",
|
||||
role_id="cto",
|
||||
focused_work_item_id=report_id,
|
||||
status="running",
|
||||
)
|
||||
)
|
||||
await self.store.save_delegation_work_item(
|
||||
DelegationWorkItem(
|
||||
work_item_id=report_id,
|
||||
run_id="r",
|
||||
cell_id="c",
|
||||
role_id="cto",
|
||||
seat_id="seat",
|
||||
role_runtime_session_id=sid,
|
||||
manager_role_id="ceo",
|
||||
manager_seat_id="seat::ceo",
|
||||
title="report",
|
||||
kind="report",
|
||||
phase=Phase.RUNNING,
|
||||
claimed_by_role_runtime_session_id=sid,
|
||||
claimed_by_seat_id="seat",
|
||||
metadata={
|
||||
"report_execution_work_item": True,
|
||||
"report_target_work_item_id": "wi-parent",
|
||||
"claimed_by_role_session_id": sid,
|
||||
"claimed_task_id": "task-report",
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
db_path = self.store.db_path
|
||||
await self.store.close()
|
||||
self.store = OPCStore(db_path=db_path)
|
||||
await self.store.initialize()
|
||||
self.store.role_serial_queue_enabled = True
|
||||
|
||||
report = await self.store.get_delegation_work_item(report_id)
|
||||
session_before = await self.store.get_delegation_role_session(sid)
|
||||
self.assertEqual(report.claimed_by_role_runtime_session_id, "")
|
||||
self.assertTrue(is_dispatchable(report))
|
||||
self.assertEqual(session_before.focused_work_item_id, report_id)
|
||||
self.assertEqual(session_before.status, "running")
|
||||
|
||||
result = await reconcile_role_serial_queues(self.store, "r")
|
||||
|
||||
session_after = await self.store.get_delegation_role_session(sid)
|
||||
self.assertIn(sid, result["cleared_focus_session_ids"])
|
||||
self.assertEqual(session_after.focused_work_item_id, "")
|
||||
self.assertEqual(session_after.status, "idle")
|
||||
self.assertTrue(is_dispatchable(await self.store.get_delegation_work_item(report_id)))
|
||||
|
||||
async def test_restart_releases_review_claim_without_dispatching_parent(self) -> None:
|
||||
parent = DelegationWorkItem(
|
||||
work_item_id="wi-parent",
|
||||
run_id="r", cell_id="c", role_id="cto", seat_id="seat",
|
||||
manager_role_id="ceo", manager_seat_id="seat::ceo", title="parent",
|
||||
phase=Phase.AWAITING_MANAGER_REVIEW,
|
||||
claimed_by_role_runtime_session_id="dead-role-session",
|
||||
claimed_by_seat_id="seat",
|
||||
metadata={
|
||||
"claimed_by_role_session_id": "dead-role-session",
|
||||
"claimed_task_id": "task-parent",
|
||||
},
|
||||
)
|
||||
report = DelegationWorkItem(
|
||||
work_item_id="wi-report",
|
||||
run_id="r", cell_id="c", role_id="cto", seat_id="seat",
|
||||
manager_role_id="ceo", manager_seat_id="seat::ceo", title="report",
|
||||
phase=Phase.READY,
|
||||
metadata={
|
||||
"report_execution_work_item": True,
|
||||
"report_target_work_item_id": "wi-parent",
|
||||
},
|
||||
)
|
||||
await self.store.save_delegation_work_item(parent)
|
||||
await self.store.save_delegation_work_item(report)
|
||||
|
||||
db_path = self.store.db_path
|
||||
await self.store.close()
|
||||
self.store = OPCStore(db_path=db_path)
|
||||
await self.store.initialize()
|
||||
|
||||
parent_after = await self.store.get_delegation_work_item("wi-parent")
|
||||
report_after = await self.store.get_delegation_work_item("wi-report")
|
||||
self.assertEqual(parent_after.claimed_by_role_runtime_session_id, "")
|
||||
self.assertEqual(parent_after.claimed_by_seat_id, "")
|
||||
self.assertEqual(parent_after.metadata.get("claimed_by_role_session_id"), "")
|
||||
self.assertEqual(parent_after.metadata.get("claimed_task_id"), "")
|
||||
self.assertTrue(parent_after.metadata.get("claim_swept_at"))
|
||||
self.assertFalse(is_dispatchable(parent_after))
|
||||
self.assertTrue(is_dispatchable(report_after))
|
||||
|
||||
|
||||
# ── Config loader picks up the feature flag ──────────────────────────────
|
||||
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ from opc.layer2_organization.communication import CommunicationManager
|
||||
from opc.layer2_organization.company_mode import (
|
||||
CompanyWorkItemExecutor,
|
||||
MAX_VERDICT_PARSE_RETRIES,
|
||||
report_work_item_id_for_attempt,
|
||||
review_work_item_id_for_attempt,
|
||||
)
|
||||
from opc.layer2_organization.org_engine import OrgEngine
|
||||
@@ -238,6 +239,130 @@ class VerdictParseRetryTests(unittest.IsolatedAsyncioTestCase):
|
||||
finally:
|
||||
await store.close()
|
||||
|
||||
async def test_retry_card_save_failure_stays_in_review_and_reconciles(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
root = Path(tmpdir)
|
||||
store = OPCStore(root / "tasks.db")
|
||||
await store.initialize()
|
||||
try:
|
||||
executor = _build_executor(store, _make_org_engine(root))
|
||||
child, review_id_v1 = _build_review_setup(store)
|
||||
report_id = report_work_item_id_for_attempt("wi-child", 1)
|
||||
child.metadata = {
|
||||
**dict(child.metadata or {}),
|
||||
"report_attempt_count": 1,
|
||||
}
|
||||
report = DelegationWorkItem(
|
||||
work_item_id=report_id,
|
||||
run_id="run-1",
|
||||
cell_id="team::cto",
|
||||
team_id="team::cto",
|
||||
role_id="engineer",
|
||||
seat_id="seat::team::cto::engineer",
|
||||
manager_role_id="cto",
|
||||
manager_seat_id="seat::team::cto::cto",
|
||||
parent_work_item_id="wi-child",
|
||||
title="Report #1: Build feature",
|
||||
summary="Durable worker report.",
|
||||
kind="report",
|
||||
projection_id=report_id,
|
||||
phase=Phase.APPROVED,
|
||||
batch_index=1,
|
||||
metadata={
|
||||
"runtime_model": "multi_team_org",
|
||||
"work_kind": "report",
|
||||
"report_execution_work_item": True,
|
||||
"report_attempt": 1,
|
||||
"report_target_work_item_id": "wi-child",
|
||||
"report_card_outcome": "applied",
|
||||
"completion_report": "Durable worker report.",
|
||||
"review_evidence": {
|
||||
"completion_summary": "Durable worker report."
|
||||
},
|
||||
},
|
||||
)
|
||||
review = _make_review_card(review_card_id=review_id_v1)
|
||||
review.metadata = {
|
||||
**dict(review.metadata or {}),
|
||||
"review_source_report_work_item_id": report_id,
|
||||
}
|
||||
await store.save_delegation_work_item(child)
|
||||
await store.save_delegation_work_item(report)
|
||||
await store.save_delegation_work_item(review)
|
||||
|
||||
original_insert = store.insert_delegation_work_item_if_absent
|
||||
review_id_v2 = review_work_item_id_for_attempt("wi-child", 2)
|
||||
injected = False
|
||||
|
||||
async def fail_first_retry_card_insert(
|
||||
item: DelegationWorkItem,
|
||||
) -> bool:
|
||||
nonlocal injected
|
||||
if not injected and item.work_item_id == review_id_v2:
|
||||
injected = True
|
||||
raise RuntimeError("injected review retry save failure")
|
||||
return await original_insert(item)
|
||||
|
||||
store.insert_delegation_work_item_if_absent = AsyncMock(
|
||||
side_effect=fail_first_retry_card_insert
|
||||
)
|
||||
review_task = _make_review_task(
|
||||
review_card_id=review_id_v1,
|
||||
structured_verdict={"unparseable": True},
|
||||
)
|
||||
try:
|
||||
await executor._finalize_review_work_item(review_task)
|
||||
finally:
|
||||
store.insert_delegation_work_item_if_absent = original_insert
|
||||
|
||||
self.assertTrue(injected)
|
||||
child_after_failure = await store.get_delegation_work_item(
|
||||
"wi-child"
|
||||
)
|
||||
failed_review = await store.get_delegation_work_item(review_id_v1)
|
||||
self.assertEqual(
|
||||
child_after_failure.phase,
|
||||
Phase.AWAITING_MANAGER_REVIEW,
|
||||
)
|
||||
self.assertFalse(
|
||||
child_after_failure.metadata.get(
|
||||
"review_verdict_parse_failed_auto_done", False
|
||||
)
|
||||
)
|
||||
self.assertEqual(failed_review.phase, Phase.CANCELLED)
|
||||
self.assertEqual(
|
||||
failed_review.metadata.get("review_work_item_outcome"),
|
||||
"verdict_parse_failed",
|
||||
)
|
||||
self.assertIsNone(
|
||||
await store.get_delegation_work_item(review_id_v2)
|
||||
)
|
||||
|
||||
run_items = await store.list_delegation_work_items("run-1")
|
||||
await executor._reconcile_missing_review_chain(run_items)
|
||||
await executor._reconcile_missing_review_chain(
|
||||
await store.list_delegation_work_items("run-1")
|
||||
)
|
||||
|
||||
retry = await store.get_delegation_work_item(review_id_v2)
|
||||
self.assertIsNotNone(retry)
|
||||
self.assertEqual(retry.phase, Phase.READY)
|
||||
self.assertEqual(
|
||||
retry.metadata.get("review_source_report_work_item_id"),
|
||||
report_id,
|
||||
)
|
||||
self.assertEqual(
|
||||
retry.metadata.get("review_retry_reason"),
|
||||
"verdict_parse_failed",
|
||||
)
|
||||
self.assertIsNone(
|
||||
await store.get_delegation_work_item(
|
||||
report_work_item_id_for_attempt("wi-child", 2)
|
||||
)
|
||||
)
|
||||
finally:
|
||||
await store.close()
|
||||
|
||||
async def test_retry_preserves_review_owner_when_runtime_task_lost_owner_metadata(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
root = Path(tmpdir)
|
||||
|
||||
@@ -18,7 +18,6 @@ from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
@@ -599,45 +598,6 @@ class DeliveryCardPhaseSyncTests(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertFalse(any(item.kind == "report" for item in items))
|
||||
self.assertFalse(any(item.kind == "review" for item in items))
|
||||
|
||||
async def test_repair_stuck_aggregate_review_item_unblocks_dependents(self) -> None:
|
||||
executor = self._executor()
|
||||
stuck = DelegationWorkItem(
|
||||
work_item_id="aggregate-stuck",
|
||||
run_id="r",
|
||||
cell_id="c",
|
||||
role_id="cto",
|
||||
seat_id="seat-cto",
|
||||
manager_role_id="ceo",
|
||||
manager_seat_id="seat-ceo",
|
||||
title="Stuck aggregate",
|
||||
kind="execute",
|
||||
phase=Phase.AWAITING_MANAGER_REVIEW,
|
||||
metadata={"work_kind": "synthesize", "work_item_turn_type": "aggregate"},
|
||||
)
|
||||
dependent = DelegationWorkItem(
|
||||
work_item_id="final-qa",
|
||||
run_id="r",
|
||||
cell_id="c",
|
||||
role_id="coo",
|
||||
seat_id="seat-coo",
|
||||
manager_role_id="ceo",
|
||||
manager_seat_id="seat-ceo",
|
||||
title="Final QA",
|
||||
kind="review",
|
||||
phase=Phase.WAITING_DEPENDENCIES,
|
||||
metadata={"dependency_work_item_ids": ["aggregate-stuck"]},
|
||||
)
|
||||
await self.store.save_delegation_work_item(stuck)
|
||||
await self.store.save_delegation_work_item(dependent)
|
||||
|
||||
work_items = await self.store.list_delegation_work_items("r")
|
||||
await executor._repair_stuck_aggregate_review_items(work_items)
|
||||
|
||||
repaired = await self.store.get_delegation_work_item("aggregate-stuck")
|
||||
unblocked = await self.store.get_delegation_work_item("final-qa")
|
||||
self.assertEqual(repaired.phase, Phase.APPROVED)
|
||||
self.assertEqual(unblocked.phase, Phase.READY)
|
||||
|
||||
async def test_stale_report_for_delivery_parent_closes_without_review_card(self) -> None:
|
||||
executor = self._executor()
|
||||
parent = DelegationWorkItem(
|
||||
@@ -697,7 +657,10 @@ class DeliveryCardPhaseSyncTests(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
closed = await self.store.get_delegation_work_item(report.work_item_id)
|
||||
self.assertEqual(closed.phase, Phase.APPROVED)
|
||||
self.assertEqual(closed.metadata["report_card_outcome"], "non_reviewable_parent")
|
||||
self.assertEqual(
|
||||
closed.metadata["report_card_outcome"],
|
||||
"parent_not_awaiting_review",
|
||||
)
|
||||
items = await self.store.list_delegation_work_items("r")
|
||||
self.assertFalse(any(str(item.work_item_id).startswith("review::deliver-parent") for item in items))
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ from opc.layer2_organization.company_mode import (
|
||||
review_work_item_id_for_attempt,
|
||||
)
|
||||
from opc.layer2_organization.org_engine import OrgEngine
|
||||
from opc.layer2_organization.phase import DONE_PHASES
|
||||
from opc.layer2_organization.work_item_links import set_linked_work_item_id
|
||||
|
||||
|
||||
@@ -201,6 +202,10 @@ class WorkerExecuteDoneSpawnsReportTests(unittest.IsolatedAsyncioTestCase):
|
||||
worker_task = _build_worker_task()
|
||||
worker_task.metadata = dict(worker_task.metadata or {})
|
||||
worker_task.metadata["work_kind"] = "dispatch"
|
||||
# Delegation is attempt-scoped. The child row alone may be
|
||||
# historical, so mirror the tool's current-turn mutation
|
||||
# marker instead of asking DONE routing to infer from rows.
|
||||
worker_task.metadata["manager_board_mutation_performed"] = True
|
||||
await store.save_task(worker_task)
|
||||
|
||||
await executor._apply_done_transition(
|
||||
@@ -396,6 +401,963 @@ class ReportTurnDoneSpawnsReviewTests(unittest.IsolatedAsyncioTestCase):
|
||||
await store.close()
|
||||
|
||||
|
||||
class ReviewChainRecoveryTests(unittest.IsolatedAsyncioTestCase):
|
||||
"""Crash boundaries use persisted auxiliary cards as their journal.
|
||||
|
||||
The attempt counters on the parent are only lookup caches: a crash can
|
||||
happen after an auxiliary card commits but before its counter does. The
|
||||
report card also has to contain the completed report before review-card
|
||||
creation is attempted, so reconciliation can resume without a live Task.
|
||||
"""
|
||||
|
||||
async def asyncSetUp(self) -> None:
|
||||
self._tmpdir = tempfile.TemporaryDirectory()
|
||||
self.root = Path(self._tmpdir.name)
|
||||
self.store = OPCStore(self.root / "tasks.db")
|
||||
await self.store.initialize()
|
||||
self.executor = _build_executor(self.store, _make_org_engine(self.root))
|
||||
|
||||
async def asyncTearDown(self) -> None:
|
||||
await self.store.close()
|
||||
self._tmpdir.cleanup()
|
||||
|
||||
async def _save_awaiting_parent(self) -> DelegationWorkItem:
|
||||
parent = _build_child_work_item()
|
||||
parent.phase = Phase.AWAITING_MANAGER_REVIEW
|
||||
parent.metadata = {
|
||||
**dict(parent.metadata or {}),
|
||||
"review_owner_role_id": "cto",
|
||||
"review_owner_seat_id": "seat::team::cto::cto",
|
||||
"completion_report": "execute-turn fallback",
|
||||
}
|
||||
await self.store.save_delegation_work_item(parent)
|
||||
return parent
|
||||
|
||||
async def _run_reconcile(self) -> list[DelegationWorkItem]:
|
||||
items = await self.store.list_delegation_work_items("run-1")
|
||||
return await self.executor._reconcile_missing_review_chain(items)
|
||||
|
||||
async def _auxiliary_cards(self) -> list[DelegationWorkItem]:
|
||||
return [
|
||||
item
|
||||
for item in await self.store.list_delegation_work_items("run-1")
|
||||
if str((item.metadata or {}).get("report_target_work_item_id", "") or "").strip()
|
||||
== "wi-child"
|
||||
or str((item.metadata or {}).get("review_target_work_item_id", "") or "").strip()
|
||||
== "wi-child"
|
||||
]
|
||||
|
||||
async def _setup_running_report(
|
||||
self,
|
||||
) -> tuple[str, Task]:
|
||||
"""Create the real execute->report handoff and claim its report card."""
|
||||
await self.store.save_delegation_work_item(_build_child_work_item())
|
||||
worker_task = _build_worker_task()
|
||||
await self.executor._apply_done_transition(
|
||||
worker_task,
|
||||
result=TaskResult(status=TaskStatus.DONE, content="execute fallback"),
|
||||
)
|
||||
report_id = report_work_item_id_for_attempt("wi-child", 1)
|
||||
role_session_id = "role-runtime::run-1::engineer"
|
||||
await self.store.update_delegation_work_item(
|
||||
report_id,
|
||||
phase=Phase.RUNNING,
|
||||
claimed_by_role_runtime_session_id=role_session_id,
|
||||
claimed_by_seat_id="seat::team::cto::engineer",
|
||||
metadata_updates={
|
||||
"claimed_by_role_session_id": role_session_id,
|
||||
"claimed_task_id": "task-report-1",
|
||||
},
|
||||
)
|
||||
report_task = ReportTurnDoneSpawnsReviewTests()._report_turn_task(
|
||||
report_card_id=report_id,
|
||||
target_work_item_id="wi-child",
|
||||
)
|
||||
return report_id, report_task
|
||||
|
||||
async def _setup_running_review(
|
||||
self,
|
||||
*,
|
||||
verdict: dict[str, object],
|
||||
) -> tuple[str, str, Task]:
|
||||
"""Create and claim a review card using the production report path."""
|
||||
report_id, report_task = await self._setup_running_report()
|
||||
await self.executor._apply_report_done_transition(
|
||||
report_task,
|
||||
result=TaskResult(status=TaskStatus.DONE, content="durable report v1"),
|
||||
)
|
||||
review_id = review_work_item_id_for_attempt("wi-child", 1)
|
||||
role_session_id = "role-runtime::run-1::cto"
|
||||
await self.store.update_delegation_work_item(
|
||||
review_id,
|
||||
phase=Phase.RUNNING,
|
||||
claimed_by_role_runtime_session_id=role_session_id,
|
||||
claimed_by_seat_id="seat::team::cto::cto",
|
||||
metadata_updates={
|
||||
"claimed_by_role_session_id": role_session_id,
|
||||
"claimed_task_id": "task-review-1",
|
||||
},
|
||||
)
|
||||
review_item = await self.store.get_delegation_work_item(review_id)
|
||||
self.assertIsNotNone(review_item)
|
||||
review_task = Task(
|
||||
id="task-review-1",
|
||||
title="Review #1: Build feature",
|
||||
project_id="proj1",
|
||||
session_id="session-cto",
|
||||
parent_session_id="session-root",
|
||||
assigned_to="cto",
|
||||
status=TaskStatus.DONE,
|
||||
metadata={
|
||||
**dict(review_item.metadata or {}),
|
||||
"execution_mode": "company_mode",
|
||||
"runtime_model": "multi_team_org",
|
||||
"delegation_run_id": "run-1",
|
||||
"work_item_runtime": True,
|
||||
"review_execution_work_item": True,
|
||||
"structured_review_verdict": verdict,
|
||||
},
|
||||
)
|
||||
set_linked_work_item_id(review_task, review_id)
|
||||
return report_id, review_id, review_task
|
||||
|
||||
async def test_insert_if_absent_preserves_claimed_deterministic_aux_card(self) -> None:
|
||||
cases = (
|
||||
(
|
||||
"report",
|
||||
report_work_item_id_for_attempt("wi-child", 1),
|
||||
"engineer",
|
||||
"seat::team::cto::engineer",
|
||||
{"report_target_work_item_id": "wi-child", "report_attempt": 1},
|
||||
),
|
||||
(
|
||||
"review",
|
||||
review_work_item_id_for_attempt("wi-child", 1),
|
||||
"cto",
|
||||
"seat::team::cto::cto",
|
||||
{"review_target_work_item_id": "wi-child", "review_attempt": 1},
|
||||
),
|
||||
)
|
||||
|
||||
for kind, work_item_id, role_id, seat_id, target_metadata in cases:
|
||||
with self.subTest(kind=kind):
|
||||
ready = DelegationWorkItem(
|
||||
work_item_id=work_item_id,
|
||||
run_id="run-1",
|
||||
cell_id="team::cto",
|
||||
team_id="team::cto",
|
||||
role_id=role_id,
|
||||
seat_id=seat_id,
|
||||
parent_work_item_id="wi-child",
|
||||
title=f"{kind.title()} attempt 1",
|
||||
kind=kind,
|
||||
projection_id=work_item_id,
|
||||
phase=Phase.READY,
|
||||
batch_index=1,
|
||||
metadata={
|
||||
"work_kind": kind,
|
||||
**target_metadata,
|
||||
"persisted_sentinel": f"original-{kind}",
|
||||
},
|
||||
)
|
||||
self.assertTrue(
|
||||
await self.store.insert_delegation_work_item_if_absent(ready)
|
||||
)
|
||||
|
||||
role_session_id = f"role-runtime::run-1::{role_id}"
|
||||
claimed_task_id = f"task-{kind}-1"
|
||||
await self.store.update_delegation_work_item(
|
||||
work_item_id,
|
||||
phase=Phase.RUNNING,
|
||||
claimed_by_role_runtime_session_id=role_session_id,
|
||||
claimed_by_seat_id=seat_id,
|
||||
metadata_updates={
|
||||
"claimed_by_role_session_id": role_session_id,
|
||||
"claimed_task_id": claimed_task_id,
|
||||
},
|
||||
)
|
||||
|
||||
competing_ready = DelegationWorkItem(
|
||||
work_item_id=work_item_id,
|
||||
run_id="run-1",
|
||||
cell_id="team::cto",
|
||||
team_id="team::cto",
|
||||
role_id=role_id,
|
||||
seat_id=seat_id,
|
||||
parent_work_item_id="wi-child",
|
||||
title=f"Competing {kind} attempt 1",
|
||||
kind=kind,
|
||||
projection_id=work_item_id,
|
||||
phase=Phase.READY,
|
||||
batch_index=1,
|
||||
metadata={
|
||||
"work_kind": kind,
|
||||
**target_metadata,
|
||||
"persisted_sentinel": f"competing-{kind}",
|
||||
},
|
||||
)
|
||||
|
||||
inserted = await self.store.insert_delegation_work_item_if_absent(
|
||||
competing_ready
|
||||
)
|
||||
persisted = await self.store.get_delegation_work_item(work_item_id)
|
||||
|
||||
self.assertFalse(inserted)
|
||||
self.assertIsNotNone(persisted)
|
||||
self.assertEqual(persisted.phase, Phase.RUNNING)
|
||||
self.assertEqual(
|
||||
persisted.claimed_by_role_runtime_session_id,
|
||||
role_session_id,
|
||||
)
|
||||
self.assertEqual(persisted.claimed_by_seat_id, seat_id)
|
||||
self.assertEqual(
|
||||
persisted.metadata.get("claimed_by_role_session_id"),
|
||||
role_session_id,
|
||||
)
|
||||
self.assertEqual(
|
||||
persisted.metadata.get("claimed_task_id"), claimed_task_id
|
||||
)
|
||||
self.assertEqual(
|
||||
persisted.metadata.get("persisted_sentinel"),
|
||||
f"original-{kind}",
|
||||
)
|
||||
|
||||
async def test_report_terminal_write_failure_releases_claim_for_retry(self) -> None:
|
||||
report_id, report_task = await self._setup_running_report()
|
||||
original_update = self.store.update_delegation_work_item
|
||||
injected = False
|
||||
|
||||
async def fail_first_terminal_report_write(work_item_id: str, **kwargs):
|
||||
nonlocal injected
|
||||
metadata_updates = dict(kwargs.get("metadata_updates") or {})
|
||||
if (
|
||||
not injected
|
||||
and work_item_id == report_id
|
||||
and kwargs.get("phase") == Phase.APPROVED
|
||||
and metadata_updates.get("report_card_outcome") == "applied"
|
||||
):
|
||||
injected = True
|
||||
raise RuntimeError("injected terminal report journal failure")
|
||||
return await original_update(work_item_id, **kwargs)
|
||||
|
||||
self.store.update_delegation_work_item = AsyncMock(
|
||||
side_effect=fail_first_terminal_report_write
|
||||
)
|
||||
try:
|
||||
await self.executor._apply_report_done_transition(
|
||||
report_task,
|
||||
result=TaskResult(status=TaskStatus.DONE, content="volatile report"),
|
||||
)
|
||||
finally:
|
||||
self.store.update_delegation_work_item = original_update
|
||||
|
||||
self.assertTrue(injected)
|
||||
parent = await self.store.get_delegation_work_item("wi-child")
|
||||
report = await self.store.get_delegation_work_item(report_id)
|
||||
self.assertEqual(parent.phase, Phase.AWAITING_MANAGER_REVIEW)
|
||||
self.assertEqual(report.phase, Phase.RUNNING)
|
||||
self.assertEqual(report.claimed_by_role_runtime_session_id, "")
|
||||
self.assertEqual(report.claimed_by_seat_id, "")
|
||||
self.assertEqual(report.metadata.get("claimed_by_role_session_id"), "")
|
||||
self.assertEqual(report.metadata.get("claimed_task_id"), "")
|
||||
self.assertTrue(
|
||||
CompanyWorkItemExecutor._work_item_is_runnable(
|
||||
report,
|
||||
{"wi-child": parent, report_id: report},
|
||||
)
|
||||
)
|
||||
self.assertIsNone(
|
||||
await self.store.get_delegation_work_item(
|
||||
review_work_item_id_for_attempt("wi-child", 1)
|
||||
)
|
||||
)
|
||||
|
||||
async def test_review_terminal_write_failure_keeps_parent_reviewable(self) -> None:
|
||||
report_id, review_id, review_task = await self._setup_running_review(
|
||||
verdict={
|
||||
"label": "reject",
|
||||
"summary": "needs rework",
|
||||
"blocking_issues": ["fix the defect"],
|
||||
"followups": [],
|
||||
}
|
||||
)
|
||||
original_update = self.store.update_delegation_work_item
|
||||
injected = False
|
||||
|
||||
async def fail_first_terminal_review_write(work_item_id: str, **kwargs):
|
||||
nonlocal injected
|
||||
if (
|
||||
not injected
|
||||
and work_item_id == review_id
|
||||
and kwargs.get("phase") in DONE_PHASES
|
||||
):
|
||||
injected = True
|
||||
raise RuntimeError("injected terminal review journal failure")
|
||||
return await original_update(work_item_id, **kwargs)
|
||||
|
||||
self.store.update_delegation_work_item = AsyncMock(
|
||||
side_effect=fail_first_terminal_review_write
|
||||
)
|
||||
try:
|
||||
await self.executor._finalize_review_work_item(review_task)
|
||||
finally:
|
||||
self.store.update_delegation_work_item = original_update
|
||||
|
||||
self.assertTrue(injected)
|
||||
parent = await self.store.get_delegation_work_item("wi-child")
|
||||
review = await self.store.get_delegation_work_item(review_id)
|
||||
self.assertEqual(parent.phase, Phase.AWAITING_MANAGER_REVIEW)
|
||||
self.assertEqual(review.phase, Phase.RUNNING)
|
||||
self.assertEqual(review.claimed_by_role_runtime_session_id, "")
|
||||
self.assertEqual(review.claimed_by_seat_id, "")
|
||||
self.assertEqual(review.metadata.get("claimed_by_role_session_id"), "")
|
||||
self.assertEqual(review.metadata.get("claimed_task_id"), "")
|
||||
self.assertIsNone(
|
||||
await self.store.get_delegation_work_item(
|
||||
report_work_item_id_for_attempt("wi-child", 2)
|
||||
)
|
||||
)
|
||||
self.assertIsNone(
|
||||
await self.store.get_delegation_work_item(
|
||||
review_work_item_id_for_attempt("wi-child", 2)
|
||||
)
|
||||
)
|
||||
self.assertEqual(
|
||||
review.metadata.get("review_source_report_work_item_id"),
|
||||
report_id,
|
||||
)
|
||||
|
||||
async def test_late_review_cannot_override_parent_awaiting_human(self) -> None:
|
||||
_report_id, review_id, review_task = await self._setup_running_review(
|
||||
verdict={
|
||||
"label": "approve",
|
||||
"summary": "approve from a now-stale manager turn",
|
||||
"blocking_issues": [],
|
||||
"followups": [],
|
||||
}
|
||||
)
|
||||
await self.store.update_delegation_work_item(
|
||||
"wi-child",
|
||||
phase=Phase.AWAITING_HUMAN,
|
||||
metadata_updates={"human_checkpoint_sentinel": "must-survive"},
|
||||
)
|
||||
|
||||
await self.executor._finalize_review_work_item(review_task)
|
||||
|
||||
parent = await self.store.get_delegation_work_item("wi-child")
|
||||
review = await self.store.get_delegation_work_item(review_id)
|
||||
self.assertEqual(parent.phase, Phase.AWAITING_HUMAN)
|
||||
self.assertEqual(
|
||||
parent.metadata.get("human_checkpoint_sentinel"), "must-survive"
|
||||
)
|
||||
self.assertNotIn("review_resolution_applied_work_item_id", parent.metadata)
|
||||
self.assertNotIn("structured_review_verdict", parent.metadata)
|
||||
self.assertNotIn("reviewed_at", parent.metadata)
|
||||
|
||||
self.assertIn(review.phase, DONE_PHASES)
|
||||
self.assertEqual(
|
||||
review.metadata.get("review_work_item_outcome"),
|
||||
"target_no_longer_awaiting_manager_review",
|
||||
)
|
||||
self.assertNotEqual(
|
||||
review.metadata.get("review_resolution_state"), "applied"
|
||||
)
|
||||
self.assertNotIn("review_resolution", review.metadata)
|
||||
self.assertEqual(review.claimed_by_role_runtime_session_id, "")
|
||||
self.assertEqual(review.claimed_by_seat_id, "")
|
||||
|
||||
async def test_late_review_is_stale_when_newer_applied_report_exists(self) -> None:
|
||||
report_v1_id, review_v1_id, review_v1_task = (
|
||||
await self._setup_running_review(
|
||||
verdict={
|
||||
"label": "approve",
|
||||
"summary": "approval based on report v1",
|
||||
"blocking_issues": [],
|
||||
"followups": [],
|
||||
}
|
||||
)
|
||||
)
|
||||
report_v2_id = report_work_item_id_for_attempt("wi-child", 2)
|
||||
report_v2 = DelegationWorkItem(
|
||||
work_item_id=report_v2_id,
|
||||
run_id="run-1",
|
||||
cell_id="team::cto",
|
||||
team_id="team::cto",
|
||||
role_id="engineer",
|
||||
seat_id="seat::team::cto::engineer",
|
||||
manager_role_id="cto",
|
||||
manager_seat_id="seat::team::cto::cto",
|
||||
parent_work_item_id="wi-child",
|
||||
title="Report attempt 2",
|
||||
summary="Newer durable handoff.",
|
||||
kind="report",
|
||||
projection_id=report_v2_id,
|
||||
phase=Phase.APPROVED,
|
||||
batch_index=2,
|
||||
metadata={
|
||||
"runtime_model": "multi_team_org",
|
||||
"work_kind": "report",
|
||||
"report_execution_work_item": True,
|
||||
"report_target_work_item_id": "wi-child",
|
||||
"report_attempt": 2,
|
||||
"report_card_outcome": "applied",
|
||||
"completion_report": "authoritative report v2",
|
||||
},
|
||||
)
|
||||
await self.store.save_delegation_work_item(report_v2)
|
||||
await self.store.update_delegation_work_item(
|
||||
"wi-child",
|
||||
metadata_updates={
|
||||
"completion_report": "authoritative report v2",
|
||||
"newer_report_sentinel": "must-survive",
|
||||
},
|
||||
)
|
||||
|
||||
await self.executor._finalize_review_work_item(review_v1_task)
|
||||
|
||||
parent = await self.store.get_delegation_work_item("wi-child")
|
||||
review_v1 = await self.store.get_delegation_work_item(review_v1_id)
|
||||
self.assertEqual(parent.phase, Phase.AWAITING_MANAGER_REVIEW)
|
||||
self.assertEqual(parent.metadata.get("completion_report"), "authoritative report v2")
|
||||
self.assertEqual(parent.metadata.get("newer_report_sentinel"), "must-survive")
|
||||
self.assertNotIn("review_resolution_applied_work_item_id", parent.metadata)
|
||||
self.assertNotIn("structured_review_verdict", parent.metadata)
|
||||
self.assertNotIn("reviewed_at", parent.metadata)
|
||||
|
||||
self.assertIn(review_v1.phase, DONE_PHASES)
|
||||
self.assertEqual(
|
||||
review_v1.metadata.get("review_source_report_work_item_id"),
|
||||
report_v1_id,
|
||||
)
|
||||
self.assertEqual(review_v1.metadata.get("review_resolution_state"), "stale")
|
||||
self.assertEqual(
|
||||
review_v1.metadata.get("review_resolution_stale_reason"),
|
||||
"source_report_superseded",
|
||||
)
|
||||
self.assertEqual(
|
||||
review_v1.metadata.get("review_work_item_outcome"),
|
||||
"superseded_by_newer_report",
|
||||
)
|
||||
self.assertEqual(
|
||||
(review_v1.metadata.get("review_resolution") or {}).get(
|
||||
"source_report_work_item_id"
|
||||
),
|
||||
report_v1_id,
|
||||
)
|
||||
|
||||
async def test_reconcile_replays_terminal_review_after_parent_write_failure(
|
||||
self,
|
||||
) -> None:
|
||||
report_id, review_id, review_task = await self._setup_running_review(
|
||||
verdict={
|
||||
"label": "reject",
|
||||
"summary": "needs rework",
|
||||
"blocking_issues": ["fix the defect"],
|
||||
"followups": [],
|
||||
}
|
||||
)
|
||||
original_apply = self.store.apply_delegation_review_resolution
|
||||
injected = False
|
||||
|
||||
async def fail_first_parent_projection(work_item_id: str, **kwargs):
|
||||
nonlocal injected
|
||||
if not injected and work_item_id == "wi-child":
|
||||
injected = True
|
||||
raise RuntimeError("injected child verdict projection failure")
|
||||
return await original_apply(work_item_id, **kwargs)
|
||||
|
||||
self.store.apply_delegation_review_resolution = AsyncMock(
|
||||
side_effect=fail_first_parent_projection
|
||||
)
|
||||
try:
|
||||
await self.executor._finalize_review_work_item(review_task)
|
||||
finally:
|
||||
self.store.apply_delegation_review_resolution = original_apply
|
||||
|
||||
self.assertTrue(injected)
|
||||
parent_before = await self.store.get_delegation_work_item("wi-child")
|
||||
review_before = await self.store.get_delegation_work_item(review_id)
|
||||
self.assertEqual(parent_before.phase, Phase.AWAITING_MANAGER_REVIEW)
|
||||
self.assertIn(review_before.phase, DONE_PHASES)
|
||||
self.assertEqual(
|
||||
review_before.metadata.get("review_source_report_work_item_id"),
|
||||
report_id,
|
||||
)
|
||||
|
||||
await self._run_reconcile()
|
||||
await self._run_reconcile()
|
||||
|
||||
parent_after = await self.store.get_delegation_work_item("wi-child")
|
||||
self.assertEqual(parent_after.phase, Phase.READY_FOR_REWORK)
|
||||
self.assertEqual(
|
||||
(parent_after.metadata.get("structured_review_verdict") or {}).get(
|
||||
"label"
|
||||
),
|
||||
"reject",
|
||||
)
|
||||
self.assertEqual(parent_after.metadata.get("review_rework_count"), 1)
|
||||
self.assertIn(
|
||||
"fix the defect",
|
||||
str(parent_after.metadata.get("rework_feedback", "")),
|
||||
)
|
||||
self.assertEqual(
|
||||
parent_after.metadata.get("review_resolution_applied_work_item_id"),
|
||||
review_id,
|
||||
)
|
||||
report_cards = [
|
||||
item for item in await self._auxiliary_cards() if item.kind == "report"
|
||||
]
|
||||
review_cards = [
|
||||
item for item in await self._auxiliary_cards() if item.kind == "review"
|
||||
]
|
||||
self.assertEqual([item.work_item_id for item in report_cards], [report_id])
|
||||
self.assertEqual([item.work_item_id for item in review_cards], [review_id])
|
||||
|
||||
# A later worker attempt re-enters review with the old journal still
|
||||
# present. The atomic applied stamp must make that verdict immutable
|
||||
# history, not a resolution to replay onto the new output.
|
||||
await self.store.update_delegation_work_item(
|
||||
"wi-child",
|
||||
phase=Phase.RUNNING,
|
||||
)
|
||||
await self.store.update_delegation_work_item(
|
||||
"wi-child",
|
||||
phase=Phase.AWAITING_MANAGER_REVIEW,
|
||||
)
|
||||
await self._run_reconcile()
|
||||
|
||||
next_cycle_parent = await self.store.get_delegation_work_item("wi-child")
|
||||
self.assertEqual(next_cycle_parent.phase, Phase.AWAITING_MANAGER_REVIEW)
|
||||
next_report = await self.store.get_delegation_work_item(
|
||||
report_work_item_id_for_attempt("wi-child", 2)
|
||||
)
|
||||
self.assertIsNotNone(next_report)
|
||||
self.assertEqual(next_report.phase, Phase.READY)
|
||||
self.assertIsNone(
|
||||
await self.store.get_delegation_work_item(
|
||||
review_work_item_id_for_attempt("wi-child", 2)
|
||||
)
|
||||
)
|
||||
|
||||
async def test_report_card_commit_survives_parent_counter_failure(self) -> None:
|
||||
await self._save_awaiting_parent()
|
||||
worker_task = _build_worker_task()
|
||||
|
||||
original_update = self.store.update_delegation_work_item
|
||||
|
||||
async def fail_report_counter(work_item_id: str, **kwargs):
|
||||
metadata_updates = dict(kwargs.get("metadata_updates") or {})
|
||||
if work_item_id == "wi-child" and "report_attempt_count" in metadata_updates:
|
||||
raise RuntimeError("injected crash after report-card commit")
|
||||
return await original_update(work_item_id, **kwargs)
|
||||
|
||||
self.store.update_delegation_work_item = AsyncMock(side_effect=fail_report_counter)
|
||||
|
||||
first = await self.executor._ensure_report_work_item_for_work_item(
|
||||
"wi-child", worker_task=worker_task
|
||||
)
|
||||
self.assertIsNotNone(first)
|
||||
self.assertEqual(first.work_item_id, report_work_item_id_for_attempt("wi-child", 1))
|
||||
parent = await self.store.get_delegation_work_item("wi-child")
|
||||
self.assertNotIn("report_attempt_count", parent.metadata or {})
|
||||
|
||||
# Make a re-save of v1 observably wrong: retry must discover and
|
||||
# return the persisted RUNNING card instead of trying READY again.
|
||||
await original_update(first.work_item_id, phase=Phase.RUNNING)
|
||||
second = await self.executor._ensure_report_work_item_for_work_item(
|
||||
"wi-child", worker_task=worker_task
|
||||
)
|
||||
|
||||
self.assertIsNotNone(second)
|
||||
self.assertEqual(second.work_item_id, first.work_item_id)
|
||||
self.assertEqual(second.phase, Phase.RUNNING)
|
||||
self.assertIsNone(
|
||||
await self.store.get_delegation_work_item(
|
||||
report_work_item_id_for_attempt("wi-child", 2)
|
||||
)
|
||||
)
|
||||
|
||||
async def test_review_card_commit_survives_parent_counter_failure(self) -> None:
|
||||
await self._save_awaiting_parent()
|
||||
worker_task = _build_worker_task()
|
||||
|
||||
original_update = self.store.update_delegation_work_item
|
||||
|
||||
async def fail_review_counter(work_item_id: str, **kwargs):
|
||||
metadata_updates = dict(kwargs.get("metadata_updates") or {})
|
||||
if work_item_id == "wi-child" and "review_attempt_count" in metadata_updates:
|
||||
raise RuntimeError("injected crash after review-card commit")
|
||||
return await original_update(work_item_id, **kwargs)
|
||||
|
||||
self.store.update_delegation_work_item = AsyncMock(side_effect=fail_review_counter)
|
||||
|
||||
first = await self.executor._ensure_review_work_item_for_work_item(
|
||||
"wi-child",
|
||||
worker_task=worker_task,
|
||||
completion_report="handoff",
|
||||
metadata_updates={
|
||||
"review_owner_role_id": "cto",
|
||||
"review_owner_seat_id": "seat::team::cto::cto",
|
||||
},
|
||||
)
|
||||
self.assertIsNotNone(first)
|
||||
self.assertEqual(first.work_item_id, review_work_item_id_for_attempt("wi-child", 1))
|
||||
parent = await self.store.get_delegation_work_item("wi-child")
|
||||
self.assertNotIn("review_attempt_count", parent.metadata or {})
|
||||
|
||||
await original_update(first.work_item_id, phase=Phase.RUNNING)
|
||||
second = await self.executor._ensure_review_work_item_for_work_item(
|
||||
"wi-child",
|
||||
worker_task=worker_task,
|
||||
completion_report="handoff",
|
||||
metadata_updates={
|
||||
"review_owner_role_id": "cto",
|
||||
"review_owner_seat_id": "seat::team::cto::cto",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertIsNotNone(second)
|
||||
self.assertEqual(second.work_item_id, first.work_item_id)
|
||||
self.assertEqual(second.phase, Phase.RUNNING)
|
||||
self.assertIsNone(
|
||||
await self.store.get_delegation_work_item(
|
||||
review_work_item_id_for_attempt("wi-child", 2)
|
||||
)
|
||||
)
|
||||
|
||||
async def test_reconcile_recovers_review_from_terminal_report(self) -> None:
|
||||
# Build the real report turn first so this exercises the exact durable
|
||||
# payload written at the report-DONE crash boundary.
|
||||
await self.store.save_delegation_work_item(_build_child_work_item())
|
||||
worker_task = _build_worker_task()
|
||||
await self.executor._apply_done_transition(
|
||||
worker_task,
|
||||
result=TaskResult(status=TaskStatus.DONE, content="execute fallback"),
|
||||
)
|
||||
report_id = report_work_item_id_for_attempt("wi-child", 1)
|
||||
await self.store.update_delegation_work_item(report_id, phase=Phase.RUNNING)
|
||||
report_task = ReportTurnDoneSpawnsReviewTests()._report_turn_task(
|
||||
report_card_id=report_id,
|
||||
target_work_item_id="wi-child",
|
||||
)
|
||||
report_payload = (
|
||||
'{"summary":"durable handoff","deliverables":[],"risks":[],"next_actions":[]}'
|
||||
)
|
||||
|
||||
original_insert = self.store.insert_delegation_work_item_if_absent
|
||||
|
||||
async def fail_review_insert(
|
||||
item: DelegationWorkItem,
|
||||
) -> bool:
|
||||
if item.kind == "review":
|
||||
raise RuntimeError("injected crash while saving review card")
|
||||
return await original_insert(item)
|
||||
|
||||
self.store.insert_delegation_work_item_if_absent = AsyncMock(
|
||||
side_effect=fail_review_insert
|
||||
)
|
||||
await self.executor._apply_report_done_transition(
|
||||
report_task,
|
||||
result=TaskResult(status=TaskStatus.DONE, content=report_payload),
|
||||
)
|
||||
|
||||
terminal_report = await self.store.get_delegation_work_item(report_id)
|
||||
self.assertEqual(terminal_report.phase, Phase.APPROVED)
|
||||
self.assertEqual(terminal_report.metadata.get("report_card_outcome"), "applied")
|
||||
self.assertEqual(terminal_report.metadata.get("completion_report"), report_payload)
|
||||
self.assertEqual(terminal_report.metadata.get("report_completion_raw"), report_payload)
|
||||
self.assertTrue(terminal_report.metadata.get("review_evidence"))
|
||||
self.assertIsNone(
|
||||
await self.store.get_delegation_work_item(
|
||||
review_work_item_id_for_attempt("wi-child", 1)
|
||||
)
|
||||
)
|
||||
|
||||
# Simulate restart: there is no runtime Task available to carry the
|
||||
# payload, only the parent and terminal report rows.
|
||||
self.store.insert_delegation_work_item_if_absent = original_insert
|
||||
self.assertEqual(await self.store.get_tasks(), [])
|
||||
await self._run_reconcile()
|
||||
await self._run_reconcile()
|
||||
|
||||
review = await self.store.get_delegation_work_item(
|
||||
review_work_item_id_for_attempt("wi-child", 1)
|
||||
)
|
||||
self.assertIsNotNone(review)
|
||||
self.assertEqual(review.metadata.get("review_completion_report"), report_payload)
|
||||
self.assertEqual(
|
||||
review.metadata.get("review_source_report_work_item_id"),
|
||||
report_id,
|
||||
)
|
||||
self.assertEqual(
|
||||
(review.metadata.get("review_evidence") or {}).get("completion_summary"),
|
||||
report_payload,
|
||||
)
|
||||
reports = [item for item in await self._auxiliary_cards() if item.kind == "report"]
|
||||
self.assertEqual([item.work_item_id for item in reports], [report_id])
|
||||
self.assertIsNone(
|
||||
await self.store.get_delegation_work_item(
|
||||
report_work_item_id_for_attempt("wi-child", 2)
|
||||
)
|
||||
)
|
||||
|
||||
async def test_reconcile_repairs_parent_after_terminal_report_projection_failure(
|
||||
self,
|
||||
) -> None:
|
||||
await self.store.save_delegation_work_item(_build_child_work_item())
|
||||
worker_task = _build_worker_task()
|
||||
await self.executor._apply_done_transition(
|
||||
worker_task,
|
||||
result=TaskResult(status=TaskStatus.DONE, content="execute fallback"),
|
||||
)
|
||||
report_id = report_work_item_id_for_attempt("wi-child", 1)
|
||||
await self.store.update_delegation_work_item(report_id, phase=Phase.RUNNING)
|
||||
report_task = ReportTurnDoneSpawnsReviewTests()._report_turn_task(
|
||||
report_card_id=report_id,
|
||||
target_work_item_id="wi-child",
|
||||
)
|
||||
report_payload = "Report payload committed before the parent projection."
|
||||
|
||||
original_update = self.store.update_delegation_work_item
|
||||
|
||||
async def fail_parent_projection(work_item_id: str, **kwargs):
|
||||
metadata_updates = dict(kwargs.get("metadata_updates") or {})
|
||||
if (
|
||||
work_item_id == "wi-child"
|
||||
and metadata_updates.get("completion_report") == report_payload
|
||||
):
|
||||
raise RuntimeError("injected crash while projecting report to parent")
|
||||
return await original_update(work_item_id, **kwargs)
|
||||
|
||||
self.store.update_delegation_work_item = AsyncMock(
|
||||
side_effect=fail_parent_projection
|
||||
)
|
||||
await self.executor._apply_report_done_transition(
|
||||
report_task,
|
||||
result=TaskResult(status=TaskStatus.DONE, content=report_payload),
|
||||
)
|
||||
|
||||
terminal_report = await self.store.get_delegation_work_item(report_id)
|
||||
self.assertEqual(terminal_report.phase, Phase.APPROVED)
|
||||
self.assertEqual(terminal_report.metadata.get("report_card_outcome"), "applied")
|
||||
self.assertEqual(terminal_report.metadata.get("completion_report"), report_payload)
|
||||
parent_before_reconcile = await self.store.get_delegation_work_item("wi-child")
|
||||
self.assertNotEqual(
|
||||
(parent_before_reconcile.metadata or {}).get("completion_report"),
|
||||
report_payload,
|
||||
)
|
||||
self.assertIsNone(
|
||||
await self.store.get_delegation_work_item(
|
||||
review_work_item_id_for_attempt("wi-child", 1)
|
||||
)
|
||||
)
|
||||
|
||||
self.store.update_delegation_work_item = original_update
|
||||
await self._run_reconcile()
|
||||
await self._run_reconcile()
|
||||
|
||||
parent_after_reconcile = await self.store.get_delegation_work_item("wi-child")
|
||||
self.assertEqual(
|
||||
parent_after_reconcile.metadata.get("completion_report"),
|
||||
report_payload,
|
||||
)
|
||||
review = await self.store.get_delegation_work_item(
|
||||
review_work_item_id_for_attempt("wi-child", 1)
|
||||
)
|
||||
self.assertIsNotNone(review)
|
||||
self.assertEqual(review.metadata.get("review_completion_report"), report_payload)
|
||||
self.assertEqual(
|
||||
review.metadata.get("review_source_report_work_item_id"),
|
||||
report_id,
|
||||
)
|
||||
self.assertIsNone(
|
||||
await self.store.get_delegation_work_item(
|
||||
report_work_item_id_for_attempt("wi-child", 2)
|
||||
)
|
||||
)
|
||||
|
||||
async def test_durable_v2_overrides_lagging_attempt_counters(self) -> None:
|
||||
await self._save_awaiting_parent()
|
||||
await self.store.update_delegation_work_item(
|
||||
"wi-child",
|
||||
metadata_updates={
|
||||
"report_attempt_count": 1,
|
||||
"review_attempt_count": 1,
|
||||
},
|
||||
)
|
||||
terminal_report = DelegationWorkItem(
|
||||
work_item_id=report_work_item_id_for_attempt("wi-child", 2),
|
||||
run_id="run-1",
|
||||
cell_id="team::cto",
|
||||
team_id="team::cto",
|
||||
role_id="engineer",
|
||||
seat_id="seat::team::cto::engineer",
|
||||
manager_role_id="cto",
|
||||
manager_seat_id="seat::team::cto::cto",
|
||||
parent_work_item_id="wi-child",
|
||||
title="Terminal report v2",
|
||||
summary="Immutable report history.",
|
||||
kind="report",
|
||||
projection_id=report_work_item_id_for_attempt("wi-child", 2),
|
||||
phase=Phase.APPROVED,
|
||||
batch_index=2,
|
||||
metadata={
|
||||
"runtime_model": "multi_team_org",
|
||||
"work_kind": "report",
|
||||
"report_execution_work_item": True,
|
||||
"report_target_work_item_id": "wi-child",
|
||||
"report_attempt": 2,
|
||||
"report_card_outcome": "applied",
|
||||
"completion_report": "Immutable report history.",
|
||||
"history_sentinel": "report-v2-must-not-change",
|
||||
},
|
||||
)
|
||||
terminal_review = DelegationWorkItem(
|
||||
work_item_id=review_work_item_id_for_attempt("wi-child", 2),
|
||||
run_id="run-1",
|
||||
cell_id="team::cto",
|
||||
team_id="team::cto",
|
||||
role_id="cto",
|
||||
seat_id="seat::team::cto::cto",
|
||||
manager_role_id="ceo",
|
||||
manager_seat_id="seat::team::cto::ceo",
|
||||
parent_work_item_id="wi-child",
|
||||
title="Terminal review v2",
|
||||
summary="Immutable review history.",
|
||||
kind="review",
|
||||
projection_id=review_work_item_id_for_attempt("wi-child", 2),
|
||||
phase=Phase.APPROVED,
|
||||
batch_index=2,
|
||||
metadata={
|
||||
"runtime_model": "multi_team_org",
|
||||
"work_kind": "review",
|
||||
"review_execution_work_item": True,
|
||||
"review_target_work_item_id": "wi-child",
|
||||
"review_attempt": 2,
|
||||
"review_work_item_outcome": "approved",
|
||||
"history_sentinel": "review-v2-must-not-change",
|
||||
},
|
||||
)
|
||||
await self.store.save_delegation_work_item(terminal_report)
|
||||
await self.store.save_delegation_work_item(terminal_review)
|
||||
|
||||
worker_task = _build_worker_task()
|
||||
new_report = await self.executor._ensure_report_work_item_for_work_item(
|
||||
"wi-child",
|
||||
worker_task=worker_task,
|
||||
)
|
||||
self.assertIsNotNone(new_report)
|
||||
self.assertEqual(
|
||||
new_report.work_item_id,
|
||||
report_work_item_id_for_attempt("wi-child", 3),
|
||||
)
|
||||
# Review and report auxiliaries must never be active in parallel.
|
||||
self.assertIsNone(
|
||||
await self.executor._ensure_review_work_item_for_work_item(
|
||||
"wi-child",
|
||||
worker_task=worker_task,
|
||||
completion_report="new completion",
|
||||
metadata_updates={
|
||||
"review_owner_role_id": "cto",
|
||||
"review_owner_seat_id": "seat::team::cto::cto",
|
||||
},
|
||||
source_report_item=terminal_report,
|
||||
)
|
||||
)
|
||||
await self.store.update_delegation_work_item(
|
||||
new_report.work_item_id,
|
||||
phase=Phase.CANCELLED,
|
||||
)
|
||||
new_review = await self.executor._ensure_review_work_item_for_work_item(
|
||||
"wi-child",
|
||||
worker_task=worker_task,
|
||||
completion_report="new completion",
|
||||
metadata_updates={
|
||||
"review_owner_role_id": "cto",
|
||||
"review_owner_seat_id": "seat::team::cto::cto",
|
||||
},
|
||||
source_report_item=terminal_report,
|
||||
)
|
||||
|
||||
self.assertIsNotNone(new_review)
|
||||
self.assertEqual(
|
||||
new_review.work_item_id,
|
||||
review_work_item_id_for_attempt("wi-child", 3),
|
||||
)
|
||||
persisted_report_v2 = await self.store.get_delegation_work_item(
|
||||
report_work_item_id_for_attempt("wi-child", 2)
|
||||
)
|
||||
persisted_review_v2 = await self.store.get_delegation_work_item(
|
||||
review_work_item_id_for_attempt("wi-child", 2)
|
||||
)
|
||||
self.assertEqual(persisted_report_v2.phase, Phase.APPROVED)
|
||||
self.assertEqual(
|
||||
persisted_report_v2.metadata.get("history_sentinel"),
|
||||
"report-v2-must-not-change",
|
||||
)
|
||||
self.assertEqual(persisted_review_v2.phase, Phase.APPROVED)
|
||||
self.assertEqual(
|
||||
persisted_review_v2.metadata.get("history_sentinel"),
|
||||
"review-v2-must-not-change",
|
||||
)
|
||||
|
||||
async def test_reconcile_without_runtime_task_creates_report(self) -> None:
|
||||
await self._save_awaiting_parent()
|
||||
self.assertEqual(await self.store.get_tasks(), [])
|
||||
|
||||
await self._run_reconcile()
|
||||
|
||||
report = await self.store.get_delegation_work_item(
|
||||
report_work_item_id_for_attempt("wi-child", 1)
|
||||
)
|
||||
self.assertIsNotNone(report)
|
||||
self.assertEqual(report.phase, Phase.READY)
|
||||
self.assertEqual(report.metadata.get("report_target_work_item_id"), "wi-child")
|
||||
self.assertEqual(report.role_id, "engineer")
|
||||
self.assertEqual(report.seat_id, "seat::team::cto::engineer")
|
||||
|
||||
async def test_repeated_reconcile_keeps_at_most_one_active_auxiliary(self) -> None:
|
||||
await self._save_awaiting_parent()
|
||||
|
||||
for _ in range(5):
|
||||
await self._run_reconcile()
|
||||
|
||||
auxiliaries = await self._auxiliary_cards()
|
||||
active = [item for item in auxiliaries if item.phase not in DONE_PHASES]
|
||||
self.assertEqual(len(active), 1)
|
||||
self.assertEqual(active[0].kind, "report")
|
||||
self.assertEqual(active[0].work_item_id, report_work_item_id_for_attempt("wi-child", 1))
|
||||
self.assertIsNone(
|
||||
await self.store.get_delegation_work_item(
|
||||
report_work_item_id_for_attempt("wi-child", 2)
|
||||
)
|
||||
)
|
||||
|
||||
async def test_reconcile_remains_idempotent_after_database_reopen(self) -> None:
|
||||
await self._save_awaiting_parent()
|
||||
await self._run_reconcile()
|
||||
await self.store.close()
|
||||
|
||||
self.store = OPCStore(self.root / "tasks.db")
|
||||
await self.store.initialize()
|
||||
self.executor = _build_executor(self.store, _make_org_engine(self.root))
|
||||
for _ in range(3):
|
||||
await self._run_reconcile()
|
||||
|
||||
auxiliaries = await self._auxiliary_cards()
|
||||
active = [item for item in auxiliaries if item.phase not in DONE_PHASES]
|
||||
self.assertEqual(len(active), 1)
|
||||
self.assertEqual(
|
||||
active[0].work_item_id,
|
||||
report_work_item_id_for_attempt("wi-child", 1),
|
||||
)
|
||||
self.assertIsNone(
|
||||
await self.store.get_delegation_work_item(
|
||||
report_work_item_id_for_attempt("wi-child", 2)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
class ReportCardRunnableFilterTests(unittest.TestCase):
|
||||
"""Pin the dispatcher-runnability filters for report cards.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user