Files
OpenOPC/tests/test_stop_resume_native_pin.py
LZH-YS1998 d14f3920e0 fix(company): stop/resume identity truth, failure-path closure, quota park
OBS-11 — stop/resume killed pure-native runs over a phantom external pin.
Role templates' preferred_external_agent leaked into execution identity even
when the user requested native and execution actually ran native; on resume
the availability gate trusted the pin and failed every non-terminal item.
Root fixes across the whole chain:
- Staffing card per-role defaults are now the RESOLVED backend (explicit
  session agent choice > runnable template preference > native), never a
  hardcoded external default; seat enrichment and the dispatch selector's
  locked branch downgrade provably unavailable externals to native and
  record the wish in execution_agent_unavailable.
- The resume availability gate fails closed only when a resumable external
  session actually exists; a bare pin heals to native (snapshot AND task
  durable identity) and the run resumes — mirroring dispatch fallback.
- Suspend-checkpoint replies: force_resume (chat/headless spelling) is
  recognized alongside ui_force_resume, and bare continuation tokens
  (English and Chinese spellings) take the plain-resume path instead of
  being routed to the final decider as content, which reopened the
  already-approved intake card.

OBS-5 — failed runs never closed and dropped new input. The dispatcher's
convergence exit now settles terminally-failed runs (status=failed,
lifecycle=closed_failed, run_failure metadata) and emits a
company_run_failure_review card whose replies never swallow messages:
dismiss acknowledges, content falls through so normal routing starts a
fresh run. _maybe_resume_existing_company_runtime no longer re-executes a
terminally-failed tree: control replies get an honest closed status,
content-bearing input starts a new run.

OBS-6 — provider quota exhaustion terminally failed work items. Rate-limit
rejections are classified (LLMProvider.is_rate_limit_error, covering
status codes, exception types, and English/Chinese provider error text),
the agent runtime raises typed ProviderQuotaExhaustedError instead of
burning conversation-feedback retries, and the company dispatcher parks:
the item returns to READY (attempt interrupted, no terminal failure), the
member session idles, and claiming backs off exponentially (60s doubling
to a 900s cap; a quiet 30min resets the streak) before resuming
automatically.

Verified end-to-end on the real minimax-m3 campaign: same goal, same 300s
stop point, same run shape that previously killed the whole tree within
90s now resumes cleanly and completes with all items approved; staffing
defaults native for all 11 roles.

Tests: test_stop_resume_native_pin (10), test_run_failure_settlement (6),
test_provider_quota_park (9); attempt-ledger, recruiter, and
suspend-resume suites updated to the new contracts (their old assertions
pinned the defective behaviors).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-28 17:03:40 +08:00

238 lines
9.9 KiB
Python

"""Regression: stop/resume must not kill runs over an unavailable agent pin.
OBS-11: a corporate role template's ``preferred_external_agent: codex`` was
stamped into execution identity even when the run was requested and executed
as native. On resume, the availability gate trusted that pin and failed every
non-terminal work item closed. These tests pin the four legs of the fix:
1. identity truth — seat enrichment and the dispatch selector record the
backend that actually runs (native fallback when the external agent is
provably unavailable),
2. resume gate symmetry — a pin without a resumable external session heals
to native instead of failing the item,
3. control/content separation — a bare "continue" (or force_resume metadata,
in either spelling) resumes the runtime instead of being routed to the
final decider as a follow-up.
"""
from __future__ import annotations
import unittest
from pathlib import Path
from tempfile import TemporaryDirectory
from types import SimpleNamespace
from opc.core.models import DelegationWorkItem, Task, TaskStatus
from opc.database.store import OPCStore
from opc.engine import OPCEngine
from opc.layer2_organization.phase import Phase
from opc.layer2_organization.work_item_links import set_linked_work_item_id
class _AdapterRegistryStub:
def __init__(self, available: list[str]):
self._available = list(available)
def list_available(self) -> list[str]:
return list(self._available)
def get(self, name: str):
return object() if name in self._available else None
def get_ordered_available(self):
return [(name, object()) for name in self._available]
class PlainResumeControlReplyTests(unittest.TestCase):
def test_control_tokens_are_plain_resume(self) -> None:
# includes the Chinese continuation spellings the product accepts
for reply in ("continue", " Resume ", "proceed", "ok", "y", "继续", "恢复", ""):
self.assertTrue(OPCEngine._is_plain_resume_control_reply(reply), reply)
def test_content_is_not_plain_resume(self) -> None:
# the Chinese sample starts with a control word but carries content,
# so it must NOT be treated as a bare control reply
for reply in ("make a ppt outline", "继续,但先修复报告第3节", "deny"):
self.assertFalse(OPCEngine._is_plain_resume_control_reply(reply), reply)
def test_force_resume_metadata_both_spellings(self) -> None:
self.assertTrue(OPCEngine._reply_metadata_requests_force_resume({"ui_force_resume": True}))
self.assertTrue(OPCEngine._reply_metadata_requests_force_resume({"force_resume": True}))
self.assertFalse(OPCEngine._reply_metadata_requests_force_resume({"other": True}))
self.assertFalse(OPCEngine._reply_metadata_requests_force_resume(None))
class LockedAgentAvailabilityFallbackTests(unittest.IsolatedAsyncioTestCase):
def _engine(self, available: list[str]) -> OPCEngine:
engine = OPCEngine(project_id="p")
engine.org_engine = SimpleNamespace()
engine.adapter_registry = _AdapterRegistryStub(available)
return engine
async def test_locked_unavailable_external_falls_back_to_native(self) -> None:
engine = self._engine(available=[])
task = Task(id="t1", title="t", project_id="p", session_id="s")
task.metadata["execution_agent_locked"] = True
task.metadata["selected_execution_agent"] = "codex"
selected = await engine._assign_task_execution_agent(task)
self.assertIsNone(selected)
self.assertIsNone(task.assigned_external_agent)
self.assertEqual(task.metadata["selected_execution_agent"], "native")
self.assertEqual(task.metadata["execution_agent_unavailable"], "codex")
self.assertEqual(
task.metadata["agent_selection"]["decision_reason"],
"locked_external_agent_unavailable_native_fallback",
)
async def test_locked_available_external_is_kept(self) -> None:
engine = self._engine(available=["codex"])
task = Task(id="t2", title="t", project_id="p", session_id="s")
task.metadata["execution_agent_locked"] = True
task.metadata["selected_execution_agent"] = "codex"
selected = await engine._assign_task_execution_agent(task)
self.assertEqual(selected, "codex")
self.assertEqual(task.assigned_external_agent, "codex")
class SeatEnrichmentIdentityTruthTests(unittest.TestCase):
def _engine(self, available: list[str], role_preferred: str | None) -> OPCEngine:
engine = OPCEngine(project_id="p")
engine.adapter_registry = _AdapterRegistryStub(available)
role = SimpleNamespace(preferred_external_agent=role_preferred)
engine.org_engine = SimpleNamespace(
get_agent=lambda role_id: role,
get_employee=lambda employee_id: None,
get_default_employee_for_role=lambda role_id: None,
list_employees=lambda role_id=None: [],
ensure_fallback_employee_for_role=lambda role_id, persist=False: None,
)
return engine
def _enrich(self, engine: OPCEngine, preferred_agent: str | None) -> dict:
decision = SimpleNamespace(preferred_agent=preferred_agent)
topology = {"seats": [{"role_id": "cto", "seat_id": "seat-cto"}]}
enriched = engine._enrich_runtime_delegation_topology(
runtime_topology=topology,
decision=decision,
project_id="p",
)
return enriched["seats"][0]
def test_explicit_native_wins_over_role_preference(self) -> None:
engine = self._engine(available=["codex"], role_preferred="codex")
seat = self._enrich(engine, preferred_agent="native")
self.assertEqual(seat["selected_execution_agent"], "native")
self.assertTrue(seat["force_native_execution"])
def test_unavailable_role_preference_resolves_to_native(self) -> None:
engine = self._engine(available=[], role_preferred="codex")
seat = self._enrich(engine, preferred_agent=None)
self.assertEqual(seat["selected_execution_agent"], "native")
self.assertEqual(seat["execution_agent_unavailable"], "codex")
def test_available_role_preference_is_kept(self) -> None:
engine = self._engine(available=["codex"], role_preferred="codex")
seat = self._enrich(engine, preferred_agent=None)
self.assertEqual(seat["selected_execution_agent"], "codex")
self.assertEqual(seat["execution_agent_unavailable"], "")
class ResumeGateHealTests(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self) -> None:
self._tmp = TemporaryDirectory()
self.store = OPCStore(Path(self._tmp.name) / "tasks.db")
await self.store.initialize()
self.engine = OPCEngine(project_id="p")
self.engine.store = self.store
self.engine.adapter_registry = _AdapterRegistryStub([])
async def asyncTearDown(self) -> None:
await self.store.close()
self._tmp.cleanup()
async def _seed(self) -> Task:
item = DelegationWorkItem(
work_item_id="wi-1",
run_id="run-1",
role_id="cto",
kind="execute",
title="survey",
phase=Phase.READY,
)
await self.store.save_delegation_work_item(item)
task = Task(
id="task-1",
title="survey",
project_id="p",
session_id="s",
status=TaskStatus.RUNNING,
metadata={"delegation_run_id": "run-1"},
)
set_linked_work_item_id(task, "wi-1")
await self.store.save_task(task)
return task
def _payload(self, task: Task, external_sessions: dict) -> dict:
return {
"checkpoint_id": "ckpt-1",
"task_snapshots": [
{
"task_id": task.id,
"execution_identity": {
"selected_execution_agent": "codex",
"assigned_external_agent": "codex",
},
"work_item": {"work_item_id": "wi-1"},
}
],
"active_work_items": [{"work_item_id": "wi-1", "phase": "ready"}],
"external_sessions": external_sessions,
"native_runtime_resume": {},
}
async def test_pin_without_external_session_heals_to_native(self) -> None:
task = await self._seed()
payload = self._payload(task, external_sessions={})
refreshed = await self.engine._prepare_company_runtime_tasks_for_resume(
[task], payload
)
prepared = refreshed[0]
self.assertIsNone(prepared.assigned_external_agent)
self.assertEqual(prepared.metadata.get("selected_execution_agent"), "native")
self.assertEqual(
prepared.metadata.get("resume_execution_agent_healed_from"), "codex"
)
pin = dict(prepared.metadata.get("_company_runtime_resume_execution_agent_pin", {}))
self.assertEqual(pin.get("selected_execution_agent"), "native")
self.assertEqual(pin.get("assigned_external_agent", ""), "")
item = await self.store.get_delegation_work_item("wi-1")
self.assertEqual(item.phase, Phase.READY)
async def test_pin_with_live_external_session_still_fails_closed(self) -> None:
task = await self._seed()
payload = self._payload(
task,
external_sessions={
task.id: {
"status": "active",
"agent_type": "codex",
"resume_session_id": "sess-1",
}
},
)
await self.engine._prepare_company_runtime_tasks_for_resume([task], payload)
item = await self.store.get_delegation_work_item("wi-1")
self.assertEqual(item.phase, Phase.FAILED)
self.assertIn("pinned to external agent", str(item.blocked_reason or ""))
if __name__ == "__main__":
unittest.main()