Gitinbox/OpenOPC
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
5938b4c2152c84a75043d56dd804fcb8984ce6a5
OpenOPC/tests/test_verdict_parse_retry.py
T

652 lines
29 KiB
Python
Raw Blame History

"""Tests for the verdict-parse-retry path.
When the reviewer agent emits output that the runtime cannot extract
into an approve/reject label, the runtime must NOT silently mark the
worker for rework. Instead it spawns review attempts with a parse-failure
hint; after MAX_VERDICT_PARSE_RETRIES failures, the child is closed as
done/approved with audit metadata because this is a reviewer-side output
failure. Non-final review parse failures must not create human-review
checkpoints or worker rework loops.
"""
from __future__ import annotations
import tempfile
import unittest
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock
from opc.core.config import OPCConfig, RoleConfig
from opc.core.events import EventBus
from opc.core.models import (
DelegationWorkItem,
Phase,
Task,
TaskStatus,
)
from opc.database.store import OPCStore
from opc.layer2_organization.communication import CommunicationManager
from opc.layer2_organization.company_mode import (
CompanyWorkItemExecutor,
MAX_VERDICT_PARSE_RETRIES,
report_work_item_id_for_attempt,
review_work_item_id_for_attempt,
)
from opc.layer2_organization.org_engine import OrgEngine
from opc.layer2_organization.work_item_links import set_linked_work_item_id
def _build_executor(store: OPCStore, org_engine: OrgEngine) -> CompanyWorkItemExecutor:
communication = CommunicationManager(store, EventBus(), org_engine=org_engine)
return CompanyWorkItemExecutor(
org_engine=org_engine,
communication=communication,
approval_engine=MagicMock(),
memory=None,
execute_task=AsyncMock(),
save_task=store.save_task,
store=store,
)
def _make_org_engine(root: Path) -> OrgEngine:
config = OPCConfig()
config.org.company_profile = "custom"
config.org.roles = [
RoleConfig(id="ceo", name="CEO", responsibility="Set direction.", reports_to="owner"),
RoleConfig(id="cto", name="CTO", responsibility="Lead engineering.", reports_to="ceo"),
RoleConfig(id="engineer", name="Engineer", responsibility="Build features.", reports_to="cto"),
]
return OrgEngine(config, root)
def _build_review_setup(store: OPCStore) -> tuple[DelegationWorkItem, str]:
"""Persist a child + first review attempt and return them."""
child = DelegationWorkItem(
work_item_id="wi-child",
run_id="run-1",
cell_id="team::cto",
team_id="team::cto",
role_id="engineer",
seat_id="seat::team::cto::engineer",
manager_role_id="cto",
manager_seat_id="seat::team::cto::cto",
title="Build feature",
summary="Ship the feature.",
kind="execute",
projection_id="wi-child",
phase=Phase.AWAITING_MANAGER_REVIEW,
metadata={
"team_id": "team::cto",
"seat_id": "seat::team::cto::engineer",
"manager_role_id": "cto",
"manager_seat_id": "seat::team::cto::cto",
"runtime_model": "multi_team_org",
"review_owner_role_id": "cto",
"review_owner_seat_id": "seat::team::cto::cto",
"review_attempt_count": 1,
},
)
return child, review_work_item_id_for_attempt("wi-child", 1)
def _make_review_card(*, review_card_id: str) -> DelegationWorkItem:
return DelegationWorkItem(
work_item_id=review_card_id,
run_id="run-1",
cell_id="team::cto",
team_id="team::cto",
role_id="cto",
seat_id="seat::team::cto::cto",
parent_work_item_id="wi-child",
manager_role_id="cto",
manager_seat_id="seat::team::cto::cto",
title="Review #1",
summary="Review the deliverable.",
kind="review",
projection_id=review_card_id,
phase=Phase.RUNNING,
metadata={
"runtime_model": "multi_team_org",
"review_execution_work_item": True,
"review_attempt": 1,
"review_owner_role_id": "cto",
"review_owner_seat_id": "seat::team::cto::cto",
"review_target_work_item_id": "wi-child",
"review_target_worker_task_id": "task-engineer",
"review_target_worker_role_id": "engineer",
"review_target_worker_seat_id": "seat::team::cto::engineer",
"review_target_title": "Build feature",
"review_target_description": "Ship the feature.",
"review_completion_report": "worker said it's done",
"current_turn_mode": "review_execute",
"hidden_from_company_kanban": True,
},
)
def _make_review_task(
*,
review_card_id: str,
structured_verdict: object,
) -> Task:
"""Build the Task object the dispatcher would hand to _finalize."""
task = Task(
id="task-review-1",
title="Review #1",
project_id="proj1",
session_id="session-cto",
parent_session_id="session-cto",
assigned_to="cto",
status=TaskStatus.DONE,
metadata={
"execution_mode": "company_mode",
"runtime_model": "multi_team_org",
"work_item_runtime": True,
"delegation_run_id": "run-1",
"delegation_team_id": "team::cto",
"delegation_seat_id": "seat::team::cto::cto",
"work_item_role_id": "cto",
"review_execution_work_item": True,
"review_attempt": 1,
"review_owner_role_id": "cto",
"review_owner_seat_id": "seat::team::cto::cto",
"review_target_work_item_id": "wi-child",
"review_target_worker_task_id": "task-engineer",
"review_target_worker_role_id": "engineer",
"review_target_worker_seat_id": "seat::team::cto::engineer",
"review_completion_report": "worker said it's done",
"structured_review_verdict": structured_verdict,
},
)
set_linked_work_item_id(task, review_card_id)
return task
class VerdictParseRetryTests(unittest.IsolatedAsyncioTestCase):
async def test_unparseable_verdict_spawns_retry_with_hint(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
store = OPCStore(root / "tasks.db")
await store.initialize()
try:
org_engine = _make_org_engine(root)
executor = _build_executor(store, org_engine)
child, review_id_v1 = _build_review_setup(store)
await store.save_delegation_work_item(child)
await store.save_delegation_work_item(_make_review_card(review_card_id=review_id_v1))
# Worker task must exist — _retry_verdict_parse_failed
# loads it via store.get_task to spawn the next review.
worker_task = Task(
id="task-engineer",
title="Build feature",
project_id="proj1",
session_id="session-eng",
parent_session_id="session-eng",
assigned_to="engineer",
status=TaskStatus.DONE,
metadata={
"execution_mode": "company_mode",
"runtime_model": "multi_team_org",
"work_item_runtime": True,
"delegation_run_id": "run-1",
"delegation_team_id": "team::cto",
"delegation_seat_id": "seat::team::cto::engineer",
"work_item_role_id": "engineer",
"manager_role_id": "cto",
"manager_seat_id": "seat::team::cto::cto",
},
)
set_linked_work_item_id(worker_task, "wi-child")
await store.save_task(worker_task)
await store.link_work_item_runtime_task("wi-child", worker_task.id)
# Reviewer's structured verdict is empty / unparseable.
review_task = _make_review_task(
review_card_id=review_id_v1,
structured_verdict={"some_other_key": "noise"},
)
await executor._finalize_review_work_item(review_task)
# Child must NOT have moved (still in review).
child_after = await store.get_delegation_work_item("wi-child")
self.assertEqual(child_after.phase, Phase.AWAITING_MANAGER_REVIEW)
# Retry counter should be 1.
self.assertEqual(
int(child_after.metadata.get("review_verdict_parse_retry_count", 0)),
1,
)
# Old review card is CANCELLED with verdict_parse_failed outcome.
old_review = await store.get_delegation_work_item(review_id_v1)
self.assertEqual(old_review.phase, Phase.CANCELLED)
self.assertEqual(
old_review.metadata.get("review_work_item_outcome"),
"verdict_parse_failed",
)
# Review #2 spawned, with parse-retry hint.
review_id_v2 = review_work_item_id_for_attempt("wi-child", 2)
new_review = await store.get_delegation_work_item(review_id_v2)
self.assertIsNotNone(new_review)
self.assertEqual(
new_review.metadata.get("review_retry_reason"),
"verdict_parse_failed",
)
self.assertIn(
"could not be parsed",
str(new_review.metadata.get("review_retry_hint", "")),
)
finally:
await store.close()
async def test_retry_card_save_failure_stays_in_review_and_reconciles(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
store = OPCStore(root / "tasks.db")
await store.initialize()
try:
executor = _build_executor(store, _make_org_engine(root))
child, review_id_v1 = _build_review_setup(store)
report_id = report_work_item_id_for_attempt("wi-child", 1)
child.metadata = {
**dict(child.metadata or {}),
"report_attempt_count": 1,
}
report = DelegationWorkItem(
work_item_id=report_id,
run_id="run-1",
cell_id="team::cto",
team_id="team::cto",
role_id="engineer",
seat_id="seat::team::cto::engineer",
manager_role_id="cto",
manager_seat_id="seat::team::cto::cto",
parent_work_item_id="wi-child",
title="Report #1: Build feature",
summary="Durable worker report.",
kind="report",
projection_id=report_id,
phase=Phase.APPROVED,
batch_index=1,
metadata={
"runtime_model": "multi_team_org",
"work_kind": "report",
"report_execution_work_item": True,
"report_attempt": 1,
"report_target_work_item_id": "wi-child",
"report_card_outcome": "applied",
"completion_report": "Durable worker report.",
"review_evidence": {
"completion_summary": "Durable worker report."
},
},
)
review = _make_review_card(review_card_id=review_id_v1)
review.metadata = {
**dict(review.metadata or {}),
"review_source_report_work_item_id": report_id,
}
await store.save_delegation_work_item(child)
await store.save_delegation_work_item(report)
await store.save_delegation_work_item(review)
original_insert = store.insert_delegation_work_item_if_absent
review_id_v2 = review_work_item_id_for_attempt("wi-child", 2)
injected = False
async def fail_first_retry_card_insert(
item: DelegationWorkItem,
) -> bool:
nonlocal injected
if not injected and item.work_item_id == review_id_v2:
injected = True
raise RuntimeError("injected review retry save failure")
return await original_insert(item)
store.insert_delegation_work_item_if_absent = AsyncMock(
side_effect=fail_first_retry_card_insert
)
review_task = _make_review_task(
review_card_id=review_id_v1,
structured_verdict={"unparseable": True},
)
try:
await executor._finalize_review_work_item(review_task)
finally:
store.insert_delegation_work_item_if_absent = original_insert
self.assertTrue(injected)
child_after_failure = await store.get_delegation_work_item(
"wi-child"
)
failed_review = await store.get_delegation_work_item(review_id_v1)
self.assertEqual(
child_after_failure.phase,
Phase.AWAITING_MANAGER_REVIEW,
)
self.assertFalse(
child_after_failure.metadata.get(
"review_verdict_parse_failed_auto_done", False
)
)
self.assertEqual(failed_review.phase, Phase.CANCELLED)
self.assertEqual(
failed_review.metadata.get("review_work_item_outcome"),
"verdict_parse_failed",
)
self.assertIsNone(
await store.get_delegation_work_item(review_id_v2)
)
run_items = await store.list_delegation_work_items("run-1")
await executor._reconcile_missing_review_chain(run_items)
await executor._reconcile_missing_review_chain(
await store.list_delegation_work_items("run-1")
)
retry = await store.get_delegation_work_item(review_id_v2)
self.assertIsNotNone(retry)
self.assertEqual(retry.phase, Phase.READY)
self.assertEqual(
retry.metadata.get("review_source_report_work_item_id"),
report_id,
)
self.assertEqual(
retry.metadata.get("review_retry_reason"),
"verdict_parse_failed",
)
self.assertIsNone(
await store.get_delegation_work_item(
report_work_item_id_for_attempt("wi-child", 2)
)
)
finally:
await store.close()
async def test_retry_preserves_review_owner_when_runtime_task_lost_owner_metadata(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
store = OPCStore(root / "tasks.db")
await store.initialize()
try:
org_engine = _make_org_engine(root)
executor = _build_executor(store, org_engine)
child, review_id_v1 = _build_review_setup(store)
await store.save_delegation_work_item(child)
await store.save_delegation_work_item(_make_review_card(review_card_id=review_id_v1))
# Mirrors the production failure: the completed review
# runtime Task no longer carried review_owner_* fields, and
# review_target_worker_task_id pointed at a report turn whose
# manager seat was not the reviewer seat.
report_task = Task(
id="task-report",
title="Report #1: Build feature",
project_id="proj1",
session_id="session-report",
parent_session_id="session-root",
assigned_to="engineer",
status=TaskStatus.DONE,
metadata={
"execution_mode": "company_mode",
"runtime_model": "multi_team_org",
"work_item_runtime": True,
"delegation_run_id": "run-1",
"delegation_team_id": "team::engineer",
"delegation_seat_id": "seat::team::engineer::engineer",
"work_item_role_id": "engineer",
"manager_role_id": "cto",
"manager_seat_id": "seat::team::cto::engineer",
},
)
await store.save_task(report_task)
review_task = _make_review_task(
review_card_id=review_id_v1,
structured_verdict={"some_other_key": "noise"},
)
review_task.metadata.pop("review_owner_role_id", None)
review_task.metadata.pop("review_owner_seat_id", None)
review_task.metadata["review_target_worker_task_id"] = "task-report"
await executor._finalize_review_work_item(review_task)
review_id_v2 = review_work_item_id_for_attempt("wi-child", 2)
new_review = await store.get_delegation_work_item(review_id_v2)
self.assertIsNotNone(new_review)
self.assertEqual(new_review.role_id, "cto")
self.assertEqual(new_review.seat_id, "seat::team::cto::cto")
self.assertEqual(
new_review.metadata.get("review_owner_seat_id"),
"seat::team::cto::cto",
)
self.assertEqual(new_review.title, "Review #2: Build feature")
child_after = await store.get_delegation_work_item("wi-child")
self.assertTrue(
CompanyWorkItemExecutor._work_item_is_runnable(
new_review,
{
"wi-child": child_after,
review_id_v2: new_review,
},
)
)
finally:
await store.close()
async def test_parse_retry_budget_auto_done_without_worker_rework(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
store = OPCStore(root / "tasks.db")
await store.initialize()
try:
org_engine = _make_org_engine(root)
executor = _build_executor(store, org_engine)
child, review_id_v1 = _build_review_setup(store)
# Already at max retries: the next unparseable verdict
# must not be charged to the worker as rework.
child.metadata = dict(child.metadata or {})
child.metadata["review_verdict_parse_retry_count"] = MAX_VERDICT_PARSE_RETRIES
await store.save_delegation_work_item(child)
await store.save_delegation_work_item(_make_review_card(review_card_id=review_id_v1))
review_task = _make_review_task(
review_card_id=review_id_v1,
structured_verdict={"garbage": True},
)
await executor._finalize_review_work_item(review_task)
child_after = await store.get_delegation_work_item("wi-child")
self.assertEqual(child_after.phase, Phase.APPROVED)
self.assertEqual(str(child_after.metadata.get("rework_feedback", "")), "")
self.assertTrue(child_after.metadata.get("review_verdict_parse_failed_auto_done"))
self.assertIn(
"unparseable",
str(child_after.metadata.get("review_parse_failure_feedback", "")),
)
self.assertEqual(
child_after.metadata.get("structured_review_verdict", {}).get("label"),
"approve",
)
self.assertEqual(
int(child_after.metadata.get("review_rework_count", 0)),
0,
)
review_after = await store.get_delegation_work_item(review_id_v1)
self.assertEqual(
review_after.metadata.get("review_work_item_outcome"),
"verdict_parse_failed_auto_done",
)
# No new review card spawned past the cap.
review_id_v2 = review_work_item_id_for_attempt("wi-child", 2)
self.assertIsNone(await store.get_delegation_work_item(review_id_v2))
finally:
await store.close()
async def test_parseable_approve_applies_mechanically(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
store = OPCStore(root / "tasks.db")
await store.initialize()
try:
org_engine = _make_org_engine(root)
executor = _build_executor(store, org_engine)
child, review_id_v1 = _build_review_setup(store)
await store.save_delegation_work_item(child)
await store.save_delegation_work_item(_make_review_card(review_card_id=review_id_v1))
review_task = _make_review_task(
review_card_id=review_id_v1,
structured_verdict={
"label": "approve",
"summary": "Looks good, ship it.",
"blocking_issues": [],
"followups": [],
},
)
await executor._finalize_review_work_item(review_task)
child_after = await store.get_delegation_work_item("wi-child")
self.assertEqual(child_after.phase, Phase.APPROVED)
self.assertEqual(
int(child_after.metadata.get("review_verdict_parse_retry_count", 0)),
0,
"approve must not touch parse-retry counter",
)
finally:
await store.close()
async def test_parseable_reject_with_no_issues_still_applies(self) -> None:
# The runtime no longer second-guesses verdict shape. A reject
# with empty blocking_issues / followups is mechanically applied
# as rework — the reviewer was trusted to produce it. (This is
# the behavioural inverse of the old _verdict_is_actionable
# path which would have salvaged or degraded.)
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
store = OPCStore(root / "tasks.db")
await store.initialize()
try:
org_engine = _make_org_engine(root)
executor = _build_executor(store, org_engine)
child, review_id_v1 = _build_review_setup(store)
await store.save_delegation_work_item(child)
await store.save_delegation_work_item(_make_review_card(review_card_id=review_id_v1))
review_task = _make_review_task(
review_card_id=review_id_v1,
structured_verdict={
"label": "reject",
"summary": "reject",
"blocking_issues": [],
"followups": [],
},
)
await executor._finalize_review_work_item(review_task)
child_after = await store.get_delegation_work_item("wi-child")
self.assertEqual(child_after.phase, Phase.READY_FOR_REWORK)
self.assertEqual(
int(child_after.metadata.get("review_rework_count", 0)),
1,
)
# Audit fields from the old salvage/degrade path must
# NOT be set — that whole machinery is gone.
self.assertNotIn(
"review_verdict_degraded", child_after.metadata,
)
self.assertNotIn(
"review_verdict_salvaged", child_after.metadata,
)
finally:
await store.close()
async def test_max_review_reworks_default_is_five_and_auto_done(self) -> None:
# After the configured rework cap, the runtime stops sending the
# worker through another rework cycle and marks the item done.
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
store = OPCStore(root / "tasks.db")
await store.initialize()
try:
org_engine = _make_org_engine(root)
executor = _build_executor(store, org_engine)
child, review_id_v1 = _build_review_setup(store)
child.metadata = dict(child.metadata or {})
child.metadata["review_rework_count"] = 5 # at default cap
await store.save_delegation_work_item(child)
await store.save_delegation_work_item(_make_review_card(review_card_id=review_id_v1))
review_task = _make_review_task(
review_card_id=review_id_v1,
structured_verdict={
"label": "reject",
"summary": "still not good",
"blocking_issues": ["fix X"],
"followups": [],
},
)
await executor._finalize_review_work_item(review_task)
child_after = await store.get_delegation_work_item("wi-child")
self.assertEqual(child_after.phase, Phase.APPROVED)
self.assertTrue(child_after.metadata.get("review_rework_cap_reached_auto_done"))
self.assertEqual(int(child_after.metadata.get("review_rework_cap", 0)), 5)
self.assertEqual(
int(child_after.metadata.get("review_rework_count_at_auto_done", 0)),
5,
)
self.assertEqual(int(child_after.metadata.get("review_rework_count", 0)), 5)
review_after = await store.get_delegation_work_item(review_id_v1)
self.assertEqual(
review_after.metadata.get("review_work_item_outcome"),
"auto_done_rework_cap",
)
finally:
await store.close()
async def test_max_review_reworks_overridable_via_metadata(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
store = OPCStore(root / "tasks.db")
await store.initialize()
try:
org_engine = _make_org_engine(root)
executor = _build_executor(store, org_engine)
child, review_id_v1 = _build_review_setup(store)
child.metadata = dict(child.metadata or {})
child.metadata["max_review_reworks"] = 3
child.metadata["review_rework_count"] = 3 # at custom cap
await store.save_delegation_work_item(child)
await store.save_delegation_work_item(_make_review_card(review_card_id=review_id_v1))
review_task = _make_review_task(
review_card_id=review_id_v1,
structured_verdict={
"label": "reject",
"summary": "still failing",
"blocking_issues": ["fix Y"],
"followups": [],
},
)
await executor._finalize_review_work_item(review_task)
child_after = await store.get_delegation_work_item("wi-child")
self.assertEqual(
child_after.phase, Phase.APPROVED,
"custom max_review_reworks=3 must stop rework at the third rework",
)
self.assertTrue(child_after.metadata.get("review_rework_cap_reached_auto_done"))
self.assertEqual(int(child_after.metadata.get("review_rework_cap", 0)), 3)
self.assertEqual(
int(child_after.metadata.get("review_rework_count_at_auto_done", 0)),
3,
)
self.assertEqual(int(child_after.metadata.get("review_rework_count", 0)), 3)
finally:
await store.close()
if __name__ == "__main__":
unittest.main()
Reference in New Issue View Git Blame Copy Permalink