fix: unify company runtime recovery lifecycle
This commit is contained in:
LZH-YS1998
@@ -1,297 +0,0 @@
|
||||
"""CLI-board-specific company runtime recovery manager.
|
||||
|
||||
Independent from office_ui/recovery_manager.py — same Core Engine APIs,
|
||||
different notification path (TUI event bridge instead of WebSocket broadcast).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from opc.layer2_organization.work_item_identity import work_item_projection_id_from_metadata
|
||||
from opc.layer2_organization.work_item_transition import apply_task_status_transition
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .engine_facade import EngineFacade
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecoverableWorkItem:
|
||||
projection_id: str
|
||||
title: str
|
||||
task_id: str
|
||||
status: str
|
||||
interrupted: bool
|
||||
previous_status: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class InterruptedCompanyRuntime:
|
||||
parent_session_id: str
|
||||
parent_task_id: str
|
||||
project_id: str
|
||||
title: str
|
||||
profile: str
|
||||
interrupted_at: str
|
||||
work_items: list[RecoverableWorkItem] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecoveryStatus:
|
||||
interrupted: list[InterruptedCompanyRuntime] = field(default_factory=list)
|
||||
active_recoveries: list[str] = field(default_factory=list)
|
||||
scanned_at: float = 0.0
|
||||
|
||||
|
||||
def _is_interrupted(task: Any) -> bool:
|
||||
from opc.core.models import TaskStatus
|
||||
if task.status != TaskStatus.FAILED:
|
||||
return False
|
||||
meta = getattr(task, "metadata", {}) or {}
|
||||
if meta.get("interrupted_recovery"):
|
||||
return True
|
||||
result = getattr(task, "result", {}) or {}
|
||||
artifacts = result.get("artifacts", {}) or {}
|
||||
return bool(artifacts.get("interrupted"))
|
||||
|
||||
|
||||
class CliRecoveryManager:
|
||||
"""Scan for interrupted company runtimes and provide resume/cancel."""
|
||||
|
||||
_CACHE_TTL = 10.0
|
||||
|
||||
def __init__(self, facade: EngineFacade) -> None:
|
||||
self._facade = facade
|
||||
self._lock = asyncio.Lock()
|
||||
self._active: dict[str, asyncio.Task[Any]] = {}
|
||||
self._cached: RecoveryStatus | None = None
|
||||
self._cache_until: float = 0.0
|
||||
|
||||
@property
|
||||
def _project_id(self) -> str:
|
||||
return self._facade.project_id or "default"
|
||||
|
||||
async def get_status(self) -> RecoveryStatus:
|
||||
now = time.time()
|
||||
if self._cached is not None and now < self._cache_until:
|
||||
self._cached.active_recoveries = list(self._active.keys())
|
||||
return self._cached
|
||||
status = await self.scan()
|
||||
self._cached = status
|
||||
self._cache_until = now + self._CACHE_TTL
|
||||
return status
|
||||
|
||||
async def scan(self) -> RecoveryStatus:
|
||||
engine = await self._facade.ensure_ready()
|
||||
if not engine.store:
|
||||
return RecoveryStatus()
|
||||
|
||||
try:
|
||||
all_tasks = await engine.store.get_tasks(project_id=self._project_id)
|
||||
except Exception as exc:
|
||||
logger.warning("Recovery scan failed: %s", exc)
|
||||
return RecoveryStatus()
|
||||
|
||||
groups: dict[str, list[Any]] = {}
|
||||
tasks_by_session: dict[str, Any] = {}
|
||||
for task in all_tasks:
|
||||
sid = str(getattr(task, "session_id", "") or "").strip()
|
||||
if sid:
|
||||
tasks_by_session[sid] = task
|
||||
parent_sid = str(getattr(task, "parent_session_id", "") or "").strip()
|
||||
projection_id = work_item_projection_id_from_metadata(getattr(task, "metadata", {}) or {})
|
||||
if parent_sid and projection_id:
|
||||
groups.setdefault(parent_sid, []).append(task)
|
||||
|
||||
from opc.core.models import TaskStatus
|
||||
interrupted: list[InterruptedCompanyRuntime] = []
|
||||
|
||||
for parent_sid, tasks in groups.items():
|
||||
if not any(_is_interrupted(t) for t in tasks):
|
||||
continue
|
||||
non_terminal = [t for t in tasks if t.status not in (TaskStatus.DONE, TaskStatus.CANCELLED)]
|
||||
if not non_terminal:
|
||||
continue
|
||||
|
||||
parent_task = tasks_by_session.get(parent_sid)
|
||||
parent_task_id = parent_task.id if parent_task else parent_sid
|
||||
title = parent_task.title if parent_task else "Unknown company runtime"
|
||||
|
||||
work_items: list[RecoverableWorkItem] = []
|
||||
earliest = ""
|
||||
for t in sorted(tasks, key=lambda x: (x.created_at, x.id)):
|
||||
meta = dict(getattr(t, "metadata", {}) or {})
|
||||
rmeta = meta.get("interrupted_recovery", {})
|
||||
is_int = _is_interrupted(t)
|
||||
if is_int and rmeta.get("detected_at", ""):
|
||||
det = rmeta["detected_at"]
|
||||
if not earliest or det < earliest:
|
||||
earliest = det
|
||||
work_items.append(RecoverableWorkItem(
|
||||
projection_id=work_item_projection_id_from_metadata(meta, fallback=t.id),
|
||||
title=t.title,
|
||||
task_id=t.id,
|
||||
status=t.status.value if hasattr(t.status, "value") else str(t.status),
|
||||
interrupted=is_int,
|
||||
previous_status=rmeta.get("previous_status", ""),
|
||||
))
|
||||
|
||||
profile = ""
|
||||
for t in tasks:
|
||||
p = (getattr(t, "metadata", {}) or {}).get("company_profile", "")
|
||||
if p:
|
||||
profile = p
|
||||
break
|
||||
|
||||
interrupted.append(InterruptedCompanyRuntime(
|
||||
parent_session_id=parent_sid,
|
||||
parent_task_id=parent_task_id,
|
||||
project_id=self._project_id,
|
||||
title=title,
|
||||
profile=profile,
|
||||
interrupted_at=earliest or datetime.now().isoformat(),
|
||||
work_items=work_items,
|
||||
))
|
||||
|
||||
return RecoveryStatus(
|
||||
interrupted=interrupted,
|
||||
active_recoveries=list(self._active.keys()),
|
||||
scanned_at=time.time(),
|
||||
)
|
||||
|
||||
async def resume(self, parent_task_id: str) -> dict[str, Any]:
|
||||
async with self._lock:
|
||||
if parent_task_id in self._active:
|
||||
return {"ok": False, "error": "already_in_progress"}
|
||||
|
||||
status = await self.scan()
|
||||
wf = next((w for w in status.interrupted if w.parent_task_id == parent_task_id), None)
|
||||
if not wf:
|
||||
return {"ok": False, "error": "not_found"}
|
||||
|
||||
engine = await self._facade.ensure_ready()
|
||||
snapshot = await engine._load_company_runtime_snapshot(wf.parent_session_id)
|
||||
if not snapshot:
|
||||
return {"ok": False, "error": "snapshot_unavailable"}
|
||||
|
||||
plan, tasks = snapshot
|
||||
|
||||
await self._clean_checkpoints(wf, tasks)
|
||||
|
||||
from opc.core.models import TaskStatus
|
||||
resumed_ids: list[str] = []
|
||||
for task in tasks:
|
||||
if task.status == TaskStatus.DONE:
|
||||
continue
|
||||
if task.status in (TaskStatus.FAILED, TaskStatus.BLOCKED):
|
||||
task.result = None
|
||||
task.execution_lock = False
|
||||
task.execution_locked_at = None
|
||||
meta = dict(task.metadata)
|
||||
meta.pop("interrupted_recovery", None)
|
||||
progress = list(meta.get("progress_log", []))
|
||||
progress.append(f"[Recovery] Resumed at {datetime.now().isoformat()}")
|
||||
meta["progress_log"] = progress[-20:]
|
||||
task.metadata = meta
|
||||
try:
|
||||
await apply_task_status_transition(
|
||||
engine.store,
|
||||
task,
|
||||
target_status_or_phase=TaskStatus.PENDING,
|
||||
reason="cli_recovery_resume",
|
||||
release_claim=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Recovery resume skipped %s: %s", task.id, exc)
|
||||
continue
|
||||
if task.status != TaskStatus.PENDING:
|
||||
logger.warning("Recovery resume preserved non-runnable phase for %s", task.id)
|
||||
continue
|
||||
await engine.store.save_task(task)
|
||||
resumed_ids.append(work_item_projection_id_from_metadata(meta, fallback=task.id))
|
||||
|
||||
if not resumed_ids:
|
||||
return {"ok": False, "error": "no_work_items_to_resume"}
|
||||
|
||||
self._cache_until = 0.0
|
||||
bg = asyncio.create_task(self._execute(parent_task_id, plan, tasks))
|
||||
self._active[parent_task_id] = bg
|
||||
return {"ok": True, "resumed_work_item_projection_ids": resumed_ids}
|
||||
|
||||
async def cancel(self, parent_task_id: str) -> dict[str, Any]:
|
||||
async with self._lock:
|
||||
bg = self._active.pop(parent_task_id, None)
|
||||
if bg and not bg.done():
|
||||
bg.cancel()
|
||||
|
||||
status = await self.scan()
|
||||
wf = next((w for w in status.interrupted if w.parent_task_id == parent_task_id), None)
|
||||
if not wf:
|
||||
return {"ok": False, "error": "not_found"}
|
||||
|
||||
engine = await self._facade.ensure_ready()
|
||||
snapshot = await engine._load_company_runtime_snapshot(wf.parent_session_id)
|
||||
if not snapshot:
|
||||
return {"ok": False, "error": "snapshot_unavailable"}
|
||||
|
||||
_, tasks = snapshot
|
||||
from opc.core.models import TaskStatus
|
||||
cancelled = 0
|
||||
for task in tasks:
|
||||
if task.status not in (TaskStatus.DONE, TaskStatus.CANCELLED):
|
||||
try:
|
||||
await apply_task_status_transition(
|
||||
engine.store,
|
||||
task,
|
||||
target_status_or_phase=TaskStatus.CANCELLED,
|
||||
reason="cli_recovery_cancel",
|
||||
release_claim=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Recovery cancel skipped %s: %s", task.id, exc)
|
||||
continue
|
||||
if task.status != TaskStatus.CANCELLED:
|
||||
logger.warning("Recovery cancel preserved non-cancelled phase for %s", task.id)
|
||||
continue
|
||||
cancelled += 1
|
||||
|
||||
await self._clean_checkpoints(wf, tasks)
|
||||
self._cache_until = 0.0
|
||||
return {"ok": True, "cancelled_count": cancelled}
|
||||
|
||||
async def _execute(self, parent_task_id: str, plan: Any, tasks: list[Any]) -> None:
|
||||
try:
|
||||
engine = await self._facade.ensure_ready()
|
||||
executor = engine.company_executor
|
||||
if not executor:
|
||||
raise RuntimeError("company_executor not available")
|
||||
await executor.execute(plan, tasks)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.warning("Recovery execution failed for %s: %s", parent_task_id, exc)
|
||||
finally:
|
||||
self._active.pop(parent_task_id, None)
|
||||
self._cache_until = 0.0
|
||||
|
||||
async def _clean_checkpoints(self, wf: InterruptedCompanyRuntime, tasks: list[Any]) -> None:
|
||||
engine = await self._facade.ensure_ready()
|
||||
if not engine.store:
|
||||
return
|
||||
session_ids = {str(getattr(t, "session_id", "") or "").strip() for t in tasks}
|
||||
session_ids.add(wf.parent_session_id)
|
||||
session_ids.discard("")
|
||||
try:
|
||||
pending = await engine.store.get_pending_checkpoints(project_id=wf.project_id)
|
||||
for cp in pending:
|
||||
if str(cp.session_id or "").strip() in session_ids:
|
||||
await engine.store.resolve_execution_checkpoint(cp.checkpoint_id, status="cancelled")
|
||||
except Exception as exc:
|
||||
logger.debug("Checkpoint cleanup error: %s", exc)
|
||||
@@ -15,7 +15,6 @@ from opc.plugins.cli_board.state.store import BoardStateStore
|
||||
from opc.plugins.cli_board.tui.screens.help import HelpScreen
|
||||
from opc.plugins.cli_board.tui.screens.palette import CommandPaletteScreen, PaletteCommand
|
||||
from opc.plugins.cli_board.tui.screens.prompt import PromptField, PromptScreen
|
||||
from opc.plugins.cli_board.tui.screens.recovery import RecoveryAction, RecoveryScreen
|
||||
from opc.plugins.cli_board.widgets.activity_pane import ActivityPaneWidget
|
||||
from opc.plugins.cli_board.widgets.context_tabs import ContextTabsWidget
|
||||
from opc.plugins.cli_board.widgets.detail_pane import DetailPaneWidget
|
||||
@@ -36,7 +35,6 @@ if TYPE_CHECKING:
|
||||
from opc.plugins.cli_board.services.engine_facade import EngineFacade
|
||||
from opc.plugins.cli_board.services.event_bridge import CliBoardEventBridge
|
||||
from opc.plugins.cli_board.services.reconcile import ReconcileLoop
|
||||
from opc.plugins.cli_board.services.recovery import CliRecoveryManager
|
||||
|
||||
|
||||
class CliBoardApp(App[None]):
|
||||
@@ -74,7 +72,6 @@ class CliBoardApp(App[None]):
|
||||
Binding("x", "cancel_task", "Cancel"),
|
||||
Binding("t", "retry_selected", "Retry"),
|
||||
Binding("e", "checkpoint_feedback", "Feedback"),
|
||||
Binding("w", "recovery_scan", "Recovery"),
|
||||
Binding("R", "rename_session", "Rename", show=False),
|
||||
Binding("D", "delete_session", "Delete", show=False),
|
||||
Binding("E", "switch_mode", "Mode", show=False),
|
||||
@@ -115,7 +112,6 @@ class CliBoardApp(App[None]):
|
||||
self.repository: BoardRepository | None = None
|
||||
self.actions: BoardActions | None = None
|
||||
self.event_bridge: CliBoardEventBridge | None = None
|
||||
self.recovery_manager: CliRecoveryManager | None = None
|
||||
self.reconcile_loop: ReconcileLoop | None = None
|
||||
self.exec_mode = "task"
|
||||
self.company_profile = "corporate"
|
||||
@@ -143,13 +139,10 @@ class CliBoardApp(App[None]):
|
||||
from opc.plugins.cli_board.services.engine_facade import EngineFacade
|
||||
from opc.plugins.cli_board.services.event_bridge import CliBoardEventBridge
|
||||
|
||||
from opc.plugins.cli_board.services.recovery import CliRecoveryManager
|
||||
|
||||
self.facade = EngineFacade(project_id=self.project_id)
|
||||
self.repository = BoardRepository(self.facade, project_id=self.project_id)
|
||||
self.actions = BoardActions(self.facade, project_id=self.project_id)
|
||||
self.event_bridge = CliBoardEventBridge(self._handle_board_event)
|
||||
self.recovery_manager = CliRecoveryManager(self.facade)
|
||||
|
||||
def compose(self) -> ComposeResult:
|
||||
yield Header(show_clock=True)
|
||||
@@ -650,36 +643,6 @@ class CliBoardApp(App[None]):
|
||||
success_message=f"{label} checkpoint for {task.title}{suffix}.",
|
||||
)
|
||||
|
||||
def action_recovery_scan(self) -> None:
|
||||
if self._readonly_guard():
|
||||
return
|
||||
self._action_recovery_scan()
|
||||
|
||||
@work(group="modal", exclusive=True)
|
||||
async def _action_recovery_scan(self) -> None:
|
||||
if self.recovery_manager is None:
|
||||
self.status_widget.set_message("Recovery unavailable.")
|
||||
return
|
||||
status = await self.recovery_manager.get_status()
|
||||
result = await self.push_screen_wait(RecoveryScreen(status))
|
||||
if result is None:
|
||||
return
|
||||
if result.action == "resume":
|
||||
self.status_widget.set_message(f"Resuming {result.parent_task_id}...")
|
||||
outcome = await self.recovery_manager.resume(result.parent_task_id)
|
||||
if outcome.get("ok"):
|
||||
ids = outcome.get("resumed_work_item_projection_ids", [])
|
||||
self.status_widget.set_message(f"Resumed {len(ids)} work item(s).")
|
||||
else:
|
||||
self.status_widget.set_message(f"Resume failed: {outcome.get('error', '?')}.")
|
||||
elif result.action == "cancel":
|
||||
outcome = await self.recovery_manager.cancel(result.parent_task_id)
|
||||
if outcome.get("ok"):
|
||||
self.status_widget.set_message(f"Cancelled {outcome.get('cancelled_count', 0)} task(s).")
|
||||
else:
|
||||
self.status_widget.set_message(f"Cancel failed: {outcome.get('error', '?')}.")
|
||||
await self._refresh_snapshot(reason="recovery", silent=True)
|
||||
|
||||
def action_rename_session(self) -> None:
|
||||
if self._readonly_guard():
|
||||
return
|
||||
@@ -1354,7 +1317,6 @@ class CliBoardApp(App[None]):
|
||||
PaletteCommand("view_focus", "Switch to Focus", "Zoom into the selected task.", "3"),
|
||||
PaletteCommand("view_pipeline", "Switch to Projection", "Show the read-only work-item projection for the selected company run.", "4"),
|
||||
PaletteCommand("view_org", "Switch to Organisation", "Show read-only org structure.", "5"),
|
||||
PaletteCommand("recovery_scan", "Runtime Recovery", "Scan and resume interrupted company runtimes.", "w"),
|
||||
PaletteCommand("rename_session", "Rename Session", "Change the title of the selected task.", "R"),
|
||||
PaletteCommand("delete_session", "Delete Session", "Cancel and remove the selected task.", "D"),
|
||||
PaletteCommand("purge_cancelled", "Purge Cancelled Tasks", "Permanently delete all cancelled/failed tasks.", ""),
|
||||
@@ -1382,7 +1344,6 @@ class CliBoardApp(App[None]):
|
||||
"project_delete",
|
||||
"session_config",
|
||||
"org_add_role",
|
||||
"recovery_scan",
|
||||
"rename_session",
|
||||
"delete_session",
|
||||
"purge_cancelled",
|
||||
|
||||
@@ -49,7 +49,7 @@ class HelpScreen(ModalScreen[None]):
|
||||
" s: reply in session m: move between columns\n"
|
||||
" a / d: approve / deny checkpoint\n"
|
||||
" e: checkpoint feedback (approve/deny with message)\n"
|
||||
" c: done x: cancel t: retry w: runtime recovery\n"
|
||||
" c: done x: cancel t: retry\n"
|
||||
"\n"
|
||||
"Session Management\n"
|
||||
" R: rename session D: delete session\n"
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
"""Recovery modal screen for the CLI board."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from textual.app import ComposeResult
|
||||
from textual.containers import Horizontal, Vertical, VerticalScroll
|
||||
from textual.screen import ModalScreen
|
||||
from textual.widgets import Button, Label, Static
|
||||
|
||||
from opc.plugins.cli_board.services.recovery import RecoveryStatus
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecoveryAction:
|
||||
action: str # "resume" | "cancel" | "dismiss"
|
||||
parent_task_id: str = ""
|
||||
|
||||
|
||||
class RecoveryScreen(ModalScreen[RecoveryAction | None]):
|
||||
"""Show interrupted company runtimes with resume/cancel options."""
|
||||
|
||||
DEFAULT_CSS = """
|
||||
RecoveryScreen {
|
||||
align: center middle;
|
||||
}
|
||||
|
||||
.recovery-dialog {
|
||||
width: 88;
|
||||
max-width: 90%;
|
||||
height: auto;
|
||||
max-height: 80%;
|
||||
border: solid $primary;
|
||||
background: $surface;
|
||||
padding: 1 2;
|
||||
}
|
||||
|
||||
.recovery-title {
|
||||
text-style: bold;
|
||||
margin-bottom: 1;
|
||||
}
|
||||
|
||||
.recovery-runtime {
|
||||
margin-bottom: 1;
|
||||
padding: 1;
|
||||
border: round $secondary;
|
||||
}
|
||||
|
||||
.recovery-actions {
|
||||
align-horizontal: right;
|
||||
height: auto;
|
||||
margin-top: 1;
|
||||
}
|
||||
|
||||
.recovery-empty {
|
||||
color: $text-muted;
|
||||
margin: 1;
|
||||
}
|
||||
"""
|
||||
|
||||
BINDINGS = [("escape", "dismiss_screen", "Close")]
|
||||
|
||||
def __init__(self, status: RecoveryStatus) -> None:
|
||||
super().__init__()
|
||||
self.status = status
|
||||
|
||||
def compose(self) -> ComposeResult:
|
||||
with Vertical(classes="recovery-dialog"):
|
||||
yield Static("Interrupted Runtimes", classes="recovery-title")
|
||||
|
||||
if not self.status.interrupted:
|
||||
yield Static("No interrupted runtimes found.", classes="recovery-empty")
|
||||
else:
|
||||
with VerticalScroll():
|
||||
for wf in self.status.interrupted:
|
||||
with Vertical(classes="recovery-runtime"):
|
||||
# Runtime header
|
||||
active = wf.parent_task_id in set(self.status.active_recoveries)
|
||||
status_label = " (recovering...)" if active else ""
|
||||
yield Label(f"{wf.title}{status_label}")
|
||||
yield Static(
|
||||
f" Profile: {wf.profile or 'unknown'} "
|
||||
f"Interrupted: {wf.interrupted_at[:19] if wf.interrupted_at else '?'}"
|
||||
)
|
||||
|
||||
# Work-item summary
|
||||
done = sum(1 for s in wf.work_items if s.status == "done")
|
||||
total = len(wf.work_items)
|
||||
failed = sum(1 for s in wf.work_items if s.interrupted)
|
||||
yield Static(
|
||||
f" Work items: {done}/{total} done, {failed} interrupted"
|
||||
)
|
||||
|
||||
if not active:
|
||||
with Horizontal():
|
||||
yield Button(
|
||||
"Resume",
|
||||
id=f"resume-{wf.parent_task_id}",
|
||||
variant="primary",
|
||||
)
|
||||
yield Button(
|
||||
"Cancel",
|
||||
id=f"cancel-{wf.parent_task_id}",
|
||||
variant="error",
|
||||
)
|
||||
|
||||
with Horizontal(classes="recovery-actions"):
|
||||
yield Button("Close", id="close-recovery")
|
||||
|
||||
def on_button_pressed(self, event: Button.Pressed) -> None:
|
||||
btn_id = event.button.id or ""
|
||||
if btn_id == "close-recovery":
|
||||
self.dismiss(None)
|
||||
return
|
||||
if btn_id.startswith("resume-"):
|
||||
task_id = btn_id[len("resume-"):]
|
||||
self.dismiss(RecoveryAction(action="resume", parent_task_id=task_id))
|
||||
return
|
||||
if btn_id.startswith("cancel-"):
|
||||
task_id = btn_id[len("cancel-"):]
|
||||
self.dismiss(RecoveryAction(action="cancel", parent_task_id=task_id))
|
||||
return
|
||||
|
||||
def action_dismiss_screen(self) -> None:
|
||||
self.dismiss(None)
|
||||
Reference in New Issue
Block a user