Gitinbox/OpenOPC
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: unify company runtime recovery lifecycle

Browse Source
This commit is contained in:
LZH-YS1998
2026-07-14 14:35:43 +08:00
parent 5e02364eb4
commit b8202bbe9e
56 changed files with 8753 additions and 3542 deletions
Download Patch File Download Diff File Expand all files Collapse all files
opc/cli/app.py
+256 -232
View File
@@ -2118,33 +2118,6 @@ def runtime_run(task_id: str = typer.Argument(...), project: Optional[str] = typ
asyncio.run(_run_service_command(project, lambda svc: svc.runtime.run_task(project_id=project or "default", task_id=task_id), json_output=json_output))
recovery_app = typer.Typer(help="Recover interrupted company runtimes")
app.add_typer(recovery_app, name="recovery")
@recovery_app.command("scan")
def recovery_scan(project: Optional[str] = typer.Option(None, "--project", "-p"), json_output: bool = typer.Option(False, "--json")):
asyncio.run(_run_service_command(project, lambda svc: svc.runtime.recovery_scan(project_id=project or "default"), json_output=json_output))
@recovery_app.command("resume")
def recovery_resume(parent_task_id: str = typer.Argument(...), project: Optional[str] = typer.Option(None, "--project", "-p"), json_output: bool = typer.Option(False, "--json")):
asyncio.run(_run_service_command(project, lambda svc: svc.runtime.recovery_action(project_id=project or "default", action="resume", parent_task_id=parent_task_id), json_output=json_output))
@recovery_app.command("cancel")
def recovery_cancel(parent_task_id: str = typer.Argument(...), yes: bool = typer.Option(False, "--yes", "-y"), project: Optional[str] = typer.Option(None, "--project", "-p"), json_output: bool = typer.Option(False, "--json")):
if not yes:
console.print("[warning]Destructive command requires --yes.[/warning]")
raise typer.Exit(code=1)
asyncio.run(_run_service_command(project, lambda svc: svc.runtime.recovery_action(project_id=project or "default", action="cancel", parent_task_id=parent_task_id), json_output=json_output))
@recovery_app.command("retry")
def recovery_retry(parent_task_id: str = typer.Argument(...), project: Optional[str] = typer.Option(None, "--project", "-p"), json_output: bool = typer.Option(False, "--json")):
asyncio.run(_run_service_command(project, lambda svc: svc.runtime.recovery_action(project_id=project or "default", action="retry", parent_task_id=parent_task_id), json_output=json_output))
comms_app = typer.Typer(help="Inspect company-mode comms")
app.add_typer(comms_app, name="comms")
@@ -2834,6 +2807,17 @@ class ChatTurnController:
self._closing = True
self.queue.clear()
await self.stop_kanban_watch(silent=True)
prepare = getattr(
self.state.engine,
"prepare_active_company_runtimes_for_shutdown",
None,
)
if callable(prepare):
# Persist company checkpoints and put the engine into
# infrastructure-shutdown mode before cancelling the active turn.
# Engine.shutdown() repeats this idempotently when it closes the
# remaining stores and subsystems.
await prepare()
task = self.active_task
if task is None or task.done():
return
@@ -2994,8 +2978,6 @@ _SLASH_COMMANDS: tuple[_SlashCommandSpec, ...] = (
_SlashCommandSpec("Tasks", "/task rename <task_id> <title>", "Rename a task and its session title."),
_SlashCommandSpec("Tasks", "/task delete <task_id> --yes", "Hard-delete a task and its persisted lifecycle data."),
_SlashCommandSpec("Runtime", "/runtime [--limit N] [--full]", "Show live runtime, active tasks, external sessions, and checkpoints."),
_SlashCommandSpec("Runtime", "/recover [--limit N] [--full]", "Show interrupted runtime and resumable checkpoints.", ("scan", "resume", "cancel", "retry")),
_SlashCommandSpec("Runtime", "/recover resume|cancel|retry <parent_task_id>", "Act on an interrupted company runtime."),
_SlashCommandSpec("Runtime", "/logs <task_id|session_id> [--limit N] [--full]", "Show execution logs, runtime events, tools, and transcript."),
_SlashCommandSpec("Runtime", "/comms <task_id> [--limit N] [--full]", "Show company-mode messages, handoffs, and review notes."),
_SlashCommandSpec("Runtime", "/attachments [--limit N] [--full]", "List current session attachment references."),
@@ -3027,7 +3009,7 @@ _SLASH_COMMANDS: tuple[_SlashCommandSpec, ...] = (
_SlashCommandSpec("Diagnostics", "/checkpoints [--limit N] [--full]", "List pending execution checkpoints."),
)
CommandSpec = _SlashCommandSpec
_SLASH_ALIASES = {"p": "project", "s": "session", "t": "task", "checkpoint": "checkpoints", "recovery": "recover", "work-item": "work-items", "workitems": "work-items"}
_SLASH_ALIASES = {"p": "project", "s": "session", "t": "task", "checkpoint": "checkpoints", "work-item": "work-items", "workitems": "work-items"}
def _initial_company_profile(config: OPCConfig) -> str:
@@ -3862,6 +3844,8 @@ async def _resolve_session_or_task(state: _InteractiveChatState, token: str) ->
async def _resolve_runtime_control_target(state: _InteractiveChatState, target: str = "") -> tuple[str, str]:
from opc.layer2_organization.company_runtime_identity import load_company_runtime_identity_index
store = _require_chat_store(state, label="Session store")
if store is None:
return "", ""
@@ -3869,24 +3853,38 @@ async def _resolve_runtime_control_target(state: _InteractiveChatState, target:
if not raw_target:
console.print("[warning]No current session. Use /session list or /session create first.[/warning]")
return "", ""
project_id = _current_project_id(state.engine)
identity_index = await load_company_runtime_identity_index(store, project_id)
task = await store.get_task(raw_target) if hasattr(store, "get_task") else None
if task is not None:
project_id = str(getattr(task, "project_id", "") or "default")
if project_id != _current_project_id(state.engine):
console.print(f"[warning]Target belongs to project '{project_id}'. Switch project first.[/warning]")
task_project_id = str(getattr(task, "project_id", "") or "default")
if task_project_id != project_id:
console.print(f"[warning]Target belongs to project '{task_project_id}'. Switch project first.[/warning]")
return "", ""
runtime_identity = identity_index.resolve(task_id=raw_target)
if runtime_identity is not None:
return raw_target, runtime_identity.runtime_session_id
return str(getattr(task, "id", "") or ""), str(getattr(task, "session_id", "") or getattr(task, "parent_session_id", "") or "")
runtime_identity = (
identity_index.resolve(runtime_session_id=raw_target)
or identity_index.resolve(task_session_id=raw_target)
)
if runtime_identity is not None:
control_task_id = (
runtime_identity.ui_anchor_task_id
or runtime_identity.config_source_task_id
)
if control_task_id:
return control_task_id, runtime_identity.runtime_session_id
session = await store.get_session(raw_target) if hasattr(store, "get_session") else None
if session is None:
console.print(f"[warning]Task or session not found: {raw_target}[/warning]")
return "", ""
project_id = str(getattr(session, "project_id", "") or "default")
if project_id != _current_project_id(state.engine):
console.print(f"[warning]Target belongs to project '{project_id}'. Switch project first.[/warning]")
session_project_id = str(getattr(session, "project_id", "") or "default")
if session_project_id != project_id:
console.print(f"[warning]Target belongs to project '{session_project_id}'. Switch project first.[/warning]")
return "", ""
if raw_target in state.session_to_task:
return state.session_to_task[raw_target], raw_target
tasks = await store.get_tasks(project_id=project_id) if hasattr(store, "get_tasks") else []
tasks = list(identity_index.tasks)
candidates = [
item for item in tasks
if str(getattr(item, "session_id", "") or "") == raw_target
@@ -3894,8 +3892,15 @@ async def _resolve_runtime_control_target(state: _InteractiveChatState, target:
if not candidates:
console.print(f"[warning]Session is not task-backed: {raw_target}[/warning]")
return "", raw_target
candidates.sort(key=lambda item: bool(str(getattr(item, "parent_session_id", "") or "")))
return str(getattr(candidates[0], "id", "") or ""), raw_target
task_mode_anchor = min(
candidates,
key=lambda item: (
bool(str(getattr(item, "parent_session_id", "") or "")),
str(getattr(item, "created_at", "") or ""),
str(getattr(item, "id", "") or ""),
),
)
return str(getattr(task_mode_anchor, "id", "") or ""), raw_target
def _make_cli_runtime_control_context(state: _InteractiveChatState, controller: ChatTurnController | None = None) -> Any:
@@ -3923,35 +3928,66 @@ def _make_cli_runtime_control_context(state: _InteractiveChatState, controller:
return context
async def _latest_company_suspend_checkpoint(state: _InteractiveChatState) -> Any | None:
session_id = str(state.session_id or "").strip()
if not session_id:
return None
for name in ("get_active_company_runtime_suspend_checkpoint", "get_pending_company_runtime_suspend_checkpoint"):
getter = getattr(state.engine, name, None)
if callable(getter):
try:
checkpoint = await getter(session_id)
except Exception:
checkpoint = None
if checkpoint is not None and str(getattr(checkpoint, "status", "") or "pending") == "pending":
return checkpoint
async def _company_runtime_identity_for_session(
state: _InteractiveChatState,
session_id: str | None = None,
) -> Any | None:
from opc.layer2_organization.company_runtime_identity import (
load_company_runtime_identity_index,
)
runtime_session_id = str(session_id or state.session_id or "").strip()
store = getattr(state.engine, "store", None)
if store is not None and hasattr(store, "get_pending_checkpoints"):
try:
checkpoints = await store.get_pending_checkpoints(
project_id=_current_project_id(state.engine),
session_id=session_id,
checkpoint_types=["company_runtime_suspended", "company_runtime_interrupted"],
)
except TypeError:
checkpoints = await store.get_pending_checkpoints(project_id=_current_project_id(state.engine), session_id=session_id)
except Exception:
checkpoints = []
for checkpoint in list(checkpoints or []):
if str(getattr(checkpoint, "checkpoint_type", "") or "") in {"company_runtime_suspended", "company_runtime_interrupted"}:
return checkpoint
return None
if not runtime_session_id or store is None:
return None
try:
identity_index = await load_company_runtime_identity_index(
store,
_current_project_id(state.engine),
)
# The interactive session may be a role/work-item session. Resolve it
# as a task session after trying the canonical root session so both UI
# roots and child channels converge on the same company runtime.
identity = (
identity_index.resolve(runtime_session_id=runtime_session_id)
or identity_index.resolve(task_session_id=runtime_session_id)
)
except Exception:
return None
return identity
async def _latest_company_suspend_checkpoint(
state: _InteractiveChatState,
session_id: str | None = None,
) -> Any | None:
identity = await _company_runtime_identity_for_session(state, session_id)
return identity.checkpoint if identity is not None and identity.resumable else None
async def _company_runtime_execution_identity(
state: _InteractiveChatState,
runtime_identity: Any,
) -> Any:
"""Read execution configuration from the durable runtime config source."""
from opc.plugins.office_ui.execution_identity import execution_identity_from_task
config_source_task_id = str(
getattr(runtime_identity, "config_source_task_id", "") or ""
).strip()
store = getattr(state.engine, "store", None)
if not config_source_task_id or store is None or not hasattr(store, "get_task"):
raise RuntimeError("Company runtime has no durable configuration source.")
config_task = await store.get_task(config_source_task_id)
if config_task is None:
raise RuntimeError("Company runtime configuration source no longer exists.")
return execution_identity_from_task(
config_task,
default_exec_mode=state.mode,
default_company_profile=state.company_profile,
default_preferred_agent=state.preferred_agent or "native",
default_org_id=state.org_id,
)
async def _handle_stop_slash(state: _InteractiveChatState, args: list[str], controller: ChatTurnController | None = None) -> None:
@@ -3972,22 +4008,35 @@ async def _handle_stop_slash(state: _InteractiveChatState, args: list[str], cont
return
payload = dict(result.payload)
state.runtime_control_state = str(payload.get("runtime_control_state") or payload.get("status") or "stopped")
state.runtime_control_task_id = str(payload.get("resume_parent_task_id") or payload.get("task_id") or task_id)
state.runtime_control_task_id = str(payload.get("task_id") or task_id)
state.runtime_control_session_id = str(payload.get("resume_parent_session_id") or payload.get("session_id") or session_id)
state.runtime_control_checkpoint_id = str(payload.get("checkpoint_id") or "")
console.print("[success]Stopped.[/success] [dim]Send a message to revise, or /continue to resume.[/dim]")
async def _runtime_control_identity_for_task(state: _InteractiveChatState, task_id: str) -> Any:
async def _runtime_control_execution_identity(state: _InteractiveChatState, task_id: str) -> Any:
from opc.layer2_organization.company_runtime_identity import load_company_runtime_identity_index
from opc.plugins.office_ui.execution_identity import execution_identity_from_task
store = getattr(state.engine, "store", None)
task = None
if store is not None and hasattr(store, "get_task") and task_id:
if store is not None and task_id:
try:
task = await store.get_task(task_id)
identity_index = await load_company_runtime_identity_index(
store,
_current_project_id(state.engine),
)
runtime_identity = identity_index.resolve(task_id=task_id)
task = identity_index.task(
runtime_identity.config_source_task_id if runtime_identity is not None else task_id
)
except Exception:
task = None
if task is None and hasattr(store, "get_task"):
try:
task = await store.get_task(task_id)
except Exception:
task = None
return execution_identity_from_task(
task,
default_exec_mode=state.mode,
@@ -4014,13 +4063,24 @@ async def _handle_continue_slash(state: _InteractiveChatState, args: list[str],
task_id, session_id = await _resolve_runtime_control_target(state, target)
if not task_id:
return
identity = await _runtime_control_identity_for_task(state, task_id)
identity = await _runtime_control_execution_identity(state, task_id)
content = " ".join(message_parts).strip() or "Resume the existing runtime."
checkpoint = await _latest_company_suspend_checkpoint(state, session_id)
if identity.exec_mode in {"company", "org", "custom"} and checkpoint is None:
console.print(
"[warning]No suspended or interrupted company runtime is available to continue.[/warning]"
)
return
if controller is not None and controller.is_busy:
checkpoint = await _latest_company_suspend_checkpoint(state)
if checkpoint is None and state.runtime_control_state not in {"suspended", "stopped"}:
console.print("[warning]Busy: wait for the current turn or /stop it before /continue.[/warning]")
return
metadata: dict[str, Any] = {"ui_force_resume": True}
if checkpoint is not None:
metadata["response_to_checkpoint_id"] = str(getattr(checkpoint, "checkpoint_id", "") or "")
metadata["response_to_checkpoint_type"] = str(
getattr(checkpoint, "checkpoint_type", "") or "company_runtime_suspended"
)
item = QueuedChatInput(
text=content,
project_id=_current_project_id(state.engine),
@@ -4030,26 +4090,29 @@ async def _handle_continue_slash(state: _InteractiveChatState, args: list[str],
org_id=identity.org_id,
preferred_agent=identity.preferred_agent,
domains=list(state.domains),
message_metadata={"ui_force_resume": True},
message_metadata=metadata,
)
state.runtime_control_state = "resuming"
state.runtime_control_task_id = task_id
state.runtime_control_session_id = session_id or state.session_id
state.runtime_control_checkpoint_id = ""
state.runtime_control_checkpoint_id = str(getattr(checkpoint, "checkpoint_id", "") or "")
if controller is not None:
await controller.submit_item(item)
else:
previous_session_id = state.session_id
previous_mode = state.mode
previous_profile = state.company_profile
previous_org_id = state.org_id
previous_agent = state.preferred_agent
state.session_id = session_id or state.session_id
state.mode = identity.exec_mode
state.company_profile = identity.company_profile
state.org_id = identity.org_id
state.preferred_agent = identity.preferred_agent
try:
await _process_interactive_chat_message(state, content, message_metadata={"ui_force_resume": True})
await _process_interactive_chat_message(state, content, message_metadata=metadata)
finally:
state.session_id = previous_session_id
state.mode = previous_mode
state.company_profile = previous_profile
state.org_id = previous_org_id
@@ -5630,140 +5693,6 @@ async def _handle_runtime_slash(state: _InteractiveChatState, args: list[str]) -
_render_checkpoint_table(checkpoints[:limit], title="Pending Checkpoints", full=full)
class _ChatRecoveryFacade:
def __init__(self, state: _InteractiveChatState) -> None:
self._state = state
self.project_id = _current_project_id(state.engine)
@property
def store(self): # noqa: ANN201 - preserve recovery facade shape
return getattr(self._state.engine, "store", None)
@property
def opc_home(self) -> Path:
return Path(getattr(self._state.engine, "opc_home", get_opc_home()))
async def ensure_ready(self):
return self._state.engine
def _get_chat_recovery_manager(state: _InteractiveChatState) -> Any:
cached = getattr(state, "_cli_chat_recovery_manager", None)
engine_id = id(state.engine)
if cached and cached[0] == engine_id:
return cached[1]
from opc.plugins.cli_board.services.recovery import CliRecoveryManager
manager = CliRecoveryManager(_ChatRecoveryFacade(state))
setattr(state, "_cli_chat_recovery_manager", (engine_id, manager))
return manager
def _render_recovery_status(status: Any, *, limit: int, full: bool = False) -> None:
workflows = list(getattr(status, "interrupted", []) or [])[:limit]
if workflows:
table = Table(title=f"Interrupted Company Runtimes ({len(workflows)})")
table.add_column("Parent Task")
table.add_column("Parent Session")
table.add_column("Title")
table.add_column("Profile")
table.add_column("Interrupted")
table.add_column("Work Items")
for workflow in workflows:
interrupted_items = [
item for item in list(getattr(workflow, "work_items", []) or [])
if bool(getattr(item, "interrupted", False))
]
table.add_row(
str(getattr(workflow, "parent_task_id", "") or ""),
str(getattr(workflow, "parent_session_id", "") or ""),
_clip_text(getattr(workflow, "title", "") or "", 70, full=full),
str(getattr(workflow, "profile", "") or ""),
str(getattr(workflow, "interrupted_at", "") or ""),
f"{len(interrupted_items)}/{len(list(getattr(workflow, 'work_items', []) or []))}",
)
console.print(table)
else:
console.print("[info]No interrupted company runtimes found.[/info]")
active = list(getattr(status, "active_recoveries", []) or [])
if active:
console.print(f"[info]Active recoveries: {', '.join(active)}[/info]")
async def _find_checkpoint_by_id(store: Any, project_id: str, checkpoint_id: str) -> Any | None:
if hasattr(store, "get_execution_checkpoints"):
checkpoints = await store.get_execution_checkpoints(project_id=project_id)
else:
checkpoints = await store.get_pending_checkpoints(project_id=project_id)
return next((item for item in checkpoints if str(getattr(item, "checkpoint_id", "") or "") == checkpoint_id), None)
async def _handle_recover_slash(state: _InteractiveChatState, args: list[str]) -> None:
store = _require_chat_store(state, label="Recovery store")
if store is None:
return
if args and args[0].lower() == "resume":
if len(args) != 2:
console.print("[warning]Usage: /recover resume <parent_task_id>. Try /recover.[/warning]")
return
manager = _get_chat_recovery_manager(state)
result = await manager.resume(args[1])
if result.get("ok"):
resumed = result.get("resumed_work_item_projection_ids", []) or []
console.print(f"[success]Recovery started for {args[1]}: {', '.join(resumed) or 'runtime queued'}.[/success]")
return
if result.get("error") == "not_found":
checkpoint = await _find_checkpoint_by_id(store, _current_project_id(state.engine), args[1])
if checkpoint:
session_id = str(getattr(checkpoint, "session_id", "") or "")
suffix = f" Try /session resume {session_id}." if session_id else " Try /checkpoints."
console.print(f"[warning]Checkpoint {args[1]} is not resumed directly.{suffix}[/warning]")
return
console.print(f"[warning]Recovery could not start: {result.get('error', 'unknown_error')}[/warning]")
return
if args and args[0].lower() in {"cancel", "retry"}:
action = args[0].lower()
if len(args) < 2:
console.print(f"[warning]Usage: /recover {action} <parent_task_id>{' --yes' if action == 'cancel' else ''}. Try /recover.[/warning]")
return
if action == "cancel":
remaining, ok = _require_yes_arg(args[2:], usage="/recover cancel <parent_task_id> --yes")
if not ok or remaining:
return
elif len(args) > 2:
console.print(f"[warning]Usage: /recover {action} <parent_task_id>[/warning]")
return
payload = await _run_chat_office_service(
state,
lambda svc: svc.runtime.recovery_action(project_id=_current_project_id(state.engine), action=action, parent_task_id=args[1]),
)
if payload:
_emit_payload(payload)
return
try:
if args and args[0].lower() == "scan":
args = args[1:]
args, limit, full = _parse_view_args(args)
except ValueError as exc:
console.print(f"[warning]{exc}. Try /recover --limit 20.[/warning]")
return
if args:
console.print("[warning]Usage: /recover [--limit N] [--full] or /recover resume <parent_task_id>.[/warning]")
return
manager = _get_chat_recovery_manager(state)
status = await manager.get_status()
_render_recovery_status(status, limit=limit, full=full)
if hasattr(store, "get_execution_checkpoints"):
checkpoints = await store.get_execution_checkpoints(
project_id=_current_project_id(state.engine),
statuses=["pending", "resuming"],
)
else:
checkpoints = await store.get_pending_checkpoints(project_id=_current_project_id(state.engine))
_render_checkpoint_table(checkpoints[:limit], title="Recovery Checkpoints", full=full)
async def _resolve_logs_target(
state: _InteractiveChatState,
target: str,
@@ -7487,10 +7416,6 @@ def _busy_slash_policy(command: str, args: list[str]) -> BusyCommandPolicy:
}
if command in readonly_roots:
return BusyCommandPolicy.IMMEDIATE_READONLY
if command == "recover":
if not args or args[0].lower() in {"scan", "status", "list"}:
return BusyCommandPolicy.IMMEDIATE_READONLY
return BusyCommandPolicy.BLOCKED_WHEN_BUSY
if command == "task":
if args and args[0].lower() == "show":
return BusyCommandPolicy.IMMEDIATE_READONLY
@@ -7810,8 +7735,6 @@ async def _handle_chat_slash_command(state: _InteractiveChatState, user_input: s
await _handle_reorg_slash(state, args)
elif command == "runtime":
await _handle_runtime_slash(state, args)
elif command == "recover":
await _handle_recover_slash(state, args)
elif command in {"work-items", "work-item"}:
await _handle_work_items_slash(state, args)
elif command == "logs":
@@ -7835,6 +7758,13 @@ async def _sync_runtime_checkpoint_hint(state: _InteractiveChatState) -> None:
display = state.runtime_display
if not hasattr(display, "set_checkpoint_hint"):
return
runtime_identity = await _company_runtime_identity_for_session(state)
if runtime_identity is not None and runtime_identity.checkpoint is not None:
checkpoint = runtime_identity.checkpoint
checkpoint_type = str(getattr(checkpoint, "checkpoint_type", "") or "pending")
checkpoint_id = str(getattr(checkpoint, "checkpoint_id", "") or "")
display.set_checkpoint_hint(checkpoint_type or checkpoint_id)
return
getter = getattr(state.engine, "get_latest_pending_checkpoint_for_session", None)
if not callable(getter) or not state.session_id:
display.set_checkpoint_hint("")
@@ -7862,24 +7792,110 @@ async def _process_interactive_chat_message(
try:
state.runtime_display.begin_turn()
effective_metadata = message_metadata
if effective_metadata is None:
suspend_checkpoint = await _latest_company_suspend_checkpoint(state)
if suspend_checkpoint is not None:
effective_metadata = {
"response_to_checkpoint_id": str(getattr(suspend_checkpoint, "checkpoint_id", "") or ""),
"response_to_checkpoint_type": str(getattr(suspend_checkpoint, "checkpoint_type", "") or "company_runtime_suspended"),
}
elif state.mode == "company":
execution_session_id = state.session_id
execution_origin_task_id: str | None = None
execution_mode = state.mode
execution_company_profile = state.company_profile
execution_org_id = state.org_id
execution_preferred_agent = state.preferred_agent
handoff_identity = None
runtime_identity = await _company_runtime_identity_for_session(state)
runtime_checkpoint = (
runtime_identity.checkpoint
if runtime_identity is not None
else None
)
explicit_runtime_type = str(
(effective_metadata or {}).get("response_to_checkpoint_type", "") or ""
).strip()
explicit_runtime_id = str(
(effective_metadata or {}).get("response_to_checkpoint_id", "") or ""
).strip()
is_explicit_runtime_handoff = explicit_runtime_type in {
"company_runtime_suspended",
"company_runtime_interrupted",
}
if is_explicit_runtime_handoff:
if runtime_identity is None or runtime_checkpoint is None:
raise RuntimeError(
"Company runtime checkpoint does not match the current session."
)
checkpoint_id = str(
getattr(runtime_checkpoint, "checkpoint_id", "") or ""
).strip()
checkpoint_type = str(
getattr(runtime_checkpoint, "checkpoint_type", "") or ""
).strip()
checkpoint_status = str(
getattr(runtime_checkpoint, "status", "") or ""
).strip().lower()
if (
not explicit_runtime_id
or explicit_runtime_id != checkpoint_id
or explicit_runtime_type != checkpoint_type
):
raise RuntimeError(
"Company runtime checkpoint identity mismatch; refresh before continuing."
)
if checkpoint_status != "pending":
raise RuntimeError(
f"Company runtime checkpoint is {checkpoint_status or 'not pending'}."
)
execution_session_id = runtime_identity.runtime_session_id
execution_origin_task_id = runtime_identity.ui_anchor_task_id or None
handoff_identity = runtime_identity
elif effective_metadata is None and runtime_checkpoint is not None:
checkpoint_status = str(
getattr(runtime_checkpoint, "status", "") or ""
).strip().lower()
if checkpoint_status != "pending":
raise RuntimeError(
f"Company runtime checkpoint is {checkpoint_status or 'not pending'}."
)
if runtime_identity is None:
raise RuntimeError(
"Company runtime checkpoint does not match the current session."
)
effective_metadata = {
"response_to_checkpoint_id": str(getattr(runtime_checkpoint, "checkpoint_id", "") or ""),
"response_to_checkpoint_type": str(getattr(runtime_checkpoint, "checkpoint_type", "") or "company_runtime_suspended"),
}
execution_session_id = runtime_identity.runtime_session_id
execution_origin_task_id = runtime_identity.ui_anchor_task_id or None
handoff_identity = runtime_identity
elif effective_metadata is None:
if runtime_identity is not None and runtime_identity.pending_checkpoint_id:
# An active checkpoint may only be pending or resuming. Never
# let a malformed identity/status fall through as a new turn.
raise RuntimeError(
"Company runtime checkpoint is not available for a new turn."
)
if state.mode == "company":
effective_metadata = {"company_preflight": "manual"}
if handoff_identity is not None:
runtime_execution_identity = await _company_runtime_execution_identity(
state,
handoff_identity,
)
execution_mode = runtime_execution_identity.exec_mode
execution_company_profile = runtime_execution_identity.company_profile
execution_org_id = runtime_execution_identity.org_id
execution_preferred_agent = runtime_execution_identity.preferred_agent
response = await state.engine.process_message(
user_input,
project_id=getattr(state.engine, "project_id", None),
session_id=state.session_id,
mode=state.mode,
org_id=state.org_id or None,
company_profile=state.company_profile if state.mode == "company" else None,
preferred_agent=state.preferred_agent,
session_id=execution_session_id,
mode=execution_mode,
org_id=execution_org_id or None,
company_profile=(
execution_company_profile
if execution_mode == "company"
else None
),
preferred_agent=execution_preferred_agent,
domains=list(state.domains),
origin_task_id=execution_origin_task_id,
message_metadata=effective_metadata,
)
await state.runtime_display.flush()
@@ -7910,7 +7926,15 @@ async def _process_interactive_chat_message(
"company_runtime_interrupted",
}
):
state.runtime_control_state = "running"
remaining_checkpoint = await _latest_company_suspend_checkpoint(state)
if remaining_checkpoint is not None:
state.runtime_control_state = "suspended"
state.runtime_control_checkpoint_id = str(
getattr(remaining_checkpoint, "checkpoint_id", "") or ""
)
else:
state.runtime_control_state = "idle"
state.runtime_control_checkpoint_id = ""
except KeyboardInterrupt:
console.print("\n[warning]Interrupted. Type /quit to exit.[/warning]")
except Exception as e:
opc/core/active_task_runs.py
+269
View File
@@ -0,0 +1,269 @@
"""Controller-local ownership of active task execution attempts.
Persisted task rows describe durable workflow state; they cannot prove that
the controller which owns an execution coroutine is still alive. This
registry intentionally stays in memory and is shared by all engines owned by
one controller.
"""
from __future__ import annotations
import asyncio
from contextlib import contextmanager
from contextvars import ContextVar
import uuid
from collections.abc import Iterator
_CURRENT_HANDOFF: ContextVar[tuple[object, str] | None] = ContextVar(
"opc_active_task_run_handoff",
default=None,
)
_CURRENT_DRIVER_ATTEMPT: ContextVar[tuple[object, str] | None] = ContextVar(
"opc_active_task_run_driver_attempt",
default=None,
)
class ActiveTaskRunAdmissionClosed(RuntimeError):
"""Raised when execution registration starts after shutdown admission closes."""
class ActiveTaskRunRegistry:
"""Track active execution attempts by ``(project_id, task_id)``.
A task can briefly have overlapping attempts while cancellation and a new
dispatch cross. Each registration therefore receives its own token and a
task remains active until its last token is removed.
"""
def __init__(self) -> None:
self._attempts: dict[tuple[str, str], set[str]] = {}
self._scope_locks: dict[tuple[str, str], asyncio.Lock] = {}
self._handoff_refs: dict[str, int] = {}
self._handoffs_drained = asyncio.Event()
self._handoffs_drained.set()
self._admission_closed = False
@staticmethod
def _key(project_id: str | None, task_id: str | None) -> tuple[str, str]:
project = str(project_id or "default").strip() or "default"
task = str(task_id or "").strip()
if not task:
raise ValueError("task_id is required")
return project, task
def register(self, project_id: str | None, task_id: str | None) -> str:
handoff_token = self._current_pending_handoff_token()
driver_attempt_token = self._current_driver_attempt_token()
if (
self._admission_closed
and handoff_token is None
and driver_attempt_token is None
):
raise ActiveTaskRunAdmissionClosed(
"task execution admission is closed for controller shutdown"
)
key = self._key(project_id, task_id)
attempt_token = uuid.uuid4().hex
self._attempts.setdefault(key, set()).add(attempt_token)
# A pre-shutdown WS request is handed off once its first real execution
# coroutine is registered. The reservation itself is deliberately not
# reported by is_active()/active_task_ids(); only this attempt is.
if handoff_token is not None:
self._settle_handoff(handoff_token)
return attempt_token
@contextmanager
def bind_driver_attempt(self, attempt_token: str) -> Iterator[None]:
"""Allow nested attempts while their live scheduler owns the scope.
Closing controller admission rejects new ingress, but a scheduler that
was already running may be between its atomic WorkItem claim and child
coroutine creation. Its nested registrations remain admissible until
that scheduler attempt ends, so shutdown can snapshot the still-live
scope instead of creating an orphan RUNNING claim.
"""
if not self._attempt_token_is_active(attempt_token):
raise ValueError("driver attempt is not active")
context_token = _CURRENT_DRIVER_ATTEMPT.set((self, attempt_token))
try:
yield
finally:
_CURRENT_DRIVER_ATTEMPT.reset(context_token)
def reserve_handoff(self) -> str:
"""Reserve one accepted ingress request until execution is registered.
Reservations bridge the short scheduling gap between the WS router and
``register``. They are controller-local synchronization only and never
become a second liveness source.
"""
if self._admission_closed:
raise ActiveTaskRunAdmissionClosed(
"task execution admission is closed for controller shutdown"
)
token = uuid.uuid4().hex
self._handoff_refs[token] = 1
self._handoffs_drained.clear()
return token
@contextmanager
def bind_handoff(self, handoff_token: str) -> Iterator[None]:
"""Propagate a reservation through tasks spawned by an ingress handler."""
if handoff_token not in self._handoff_refs:
raise ValueError("handoff reservation is not pending")
context_token = _CURRENT_HANDOFF.set((self, handoff_token))
try:
yield
finally:
_CURRENT_HANDOFF.reset(context_token)
def retain_current_handoff(self) -> str | None:
"""Retain the bound reservation for a newly scheduled coroutine."""
handoff_token = self._current_pending_handoff_token()
if handoff_token is None:
return None
self._handoff_refs[handoff_token] += 1
return handoff_token
def release_current_handoff(self) -> bool:
"""Release an accepted request that will not start an execution."""
handoff_token = self._current_pending_handoff_token()
if handoff_token is None:
return False
return self.release_handoff(handoff_token)
def release_handoff(self, handoff_token: str) -> bool:
"""Release one owner, draining a request that exited before execution."""
refs = self._handoff_refs.get(handoff_token)
if refs is None:
return False
if refs > 1:
self._handoff_refs[handoff_token] = refs - 1
return True
self._settle_handoff(handoff_token)
return True
def revoke_handoff(self, handoff_token: str) -> bool:
"""Invalidate every retained owner of a queued ingress handoff.
Controller shutdown uses this after synchronously cancelling a request
which has not registered its first execution attempt. Revocation is
intentionally stronger than ``release_handoff``: callbacks may be
delayed by cancellation cleanup, but the revoked request must neither
keep the shutdown barrier open nor register work after admission has
closed.
"""
if handoff_token not in self._handoff_refs:
return False
self._settle_handoff(handoff_token)
return True
def _current_pending_handoff_token(self) -> str | None:
binding = _CURRENT_HANDOFF.get()
if binding is None or binding[0] is not self:
return None
token = binding[1]
return token if token in self._handoff_refs else None
def _current_driver_attempt_token(self) -> str | None:
binding = _CURRENT_DRIVER_ATTEMPT.get()
if binding is None or binding[0] is not self:
return None
token = binding[1]
return token if self._attempt_token_is_active(token) else None
def _attempt_token_is_active(self, attempt_token: str) -> bool:
return any(
attempt_token in attempts
for attempts in self._attempts.values()
)
def _settle_handoff(self, handoff_token: str) -> None:
self._handoff_refs.pop(handoff_token, None)
if not self._handoff_refs:
self._handoffs_drained.set()
@property
def admission_closed(self) -> bool:
return self._admission_closed
def close_admission(self) -> None:
"""Reject future attempts without dropping attempts already in flight."""
self._admission_closed = True
async def close_admission_and_wait_for_handoffs(self) -> None:
"""Close ingress and wait until every already-accepted request hands off.
A bound pending reservation may still call ``register`` after admission
closes. That registration atomically drains the reservation and turns
the real coroutine into the sole active fact. This wait therefore ends
at handoff, never at completion of the potentially long execution.
"""
self.close_admission()
while self._handoff_refs:
await self._handoffs_drained.wait()
@property
def pending_handoff_count(self) -> int:
return len(self._handoff_refs)
def is_handoff_pending(self, handoff_token: str | None) -> bool:
return bool(handoff_token and handoff_token in self._handoff_refs)
def scope_lock(
self,
project_id: str | None,
runtime_session_id: str | None,
) -> asyncio.Lock:
"""Return the controller-shared lock for one durable runtime scope."""
project = str(project_id or "default").strip() or "default"
session = str(runtime_session_id or "").strip()
if not session:
raise ValueError("runtime_session_id is required")
key = (project, session)
lock = self._scope_locks.get(key)
if lock is None:
lock = asyncio.Lock()
self._scope_locks[key] = lock
return lock
def unregister(
self,
project_id: str | None,
task_id: str | None,
attempt_token: str,
) -> bool:
key = self._key(project_id, task_id)
attempts = self._attempts.get(key)
if not attempts or attempt_token not in attempts:
return False
attempts.remove(attempt_token)
if not attempts:
self._attempts.pop(key, None)
return True
def is_active(self, project_id: str | None, task_id: str | None) -> bool:
return bool(self._attempts.get(self._key(project_id, task_id)))
def active_task_ids(self, project_id: str | None) -> set[str]:
project = str(project_id or "default").strip() or "default"
return {
task_id
for (candidate_project, task_id), attempts in self._attempts.items()
if candidate_project == project and attempts
}
def attempt_count(self, project_id: str | None, task_id: str | None) -> int:
return len(self._attempts.get(self._key(project_id, task_id), ()))
opc/database/store.py
+358 -57
View File
@@ -2862,32 +2862,6 @@ class OPCStore:
await self._db.commit()
return cursor.rowcount > 0
async def renew_task_lock(self, task_id: str) -> bool:
"""Refresh ``execution_locked_at`` for a live running task.
This is the heartbeat used by the WS handler while it is actively
processing a session message. It only updates rows whose ``status`` is
still ``running`` (anything else has already ended and must not keep a
live timestamp), and deliberately ignores ``execution_lock``: that bit
is set only by delegation checkout, while the office_ui dispatch path
serializes through an in-memory asyncio.Lock and never flips it. The
refreshed ``execution_locked_at`` is what ``reset_orphan_running_tasks``
compares against on startup to distinguish live work from abandoned
``running`` rows.
Returns ``True`` when the row was still ``status=running`` (heartbeat
extended), ``False`` otherwise (task ended or gone — caller should
stop heartbeating).
"""
assert self._db
now_iso = datetime.now().isoformat()
async with self._db.execute(
"UPDATE tasks SET execution_locked_at = ? WHERE id = ? AND status = 'running'",
(now_iso, task_id),
) as cursor:
await self._db.commit()
return cursor.rowcount > 0
async def release_task_lock(self, task_id: str) -> None:
assert self._db
await self._db.execute(
@@ -2896,37 +2870,6 @@ class OPCStore:
)
await self._db.commit()
async def reset_orphan_running_tasks(self, *, lease_seconds: int = 300) -> dict[str, int]:
"""Reset ``status=running`` tasks abandoned by a prior process.
Safe single-process assumption: when this runs during server startup,
any task that still claims ``status=running`` must be an orphan — its
worker coroutine died with the previous process. We revert it to
``idle`` so the UI can Continue it, and clear any stale execution lock
regardless of status when the lease has expired.
Returns a summary dict with keys ``statuses_reset`` and
``locks_cleared`` giving the affected row counts.
"""
assert self._db
cutoff_iso = (datetime.now() - timedelta(seconds=int(lease_seconds))).isoformat()
async with self._db.execute(
"UPDATE tasks SET status = 'idle', execution_lock = 0, execution_locked_at = NULL "
"WHERE status = 'running' AND ("
"execution_locked_at IS NULL OR execution_locked_at < ?"
")",
(cutoff_iso,),
) as cursor:
statuses_reset = cursor.rowcount or 0
async with self._db.execute(
"UPDATE tasks SET execution_lock = 0, execution_locked_at = NULL "
"WHERE execution_lock = 1 AND execution_locked_at IS NOT NULL AND execution_locked_at < ?",
(cutoff_iso,),
) as cursor:
locks_cleared = cursor.rowcount or 0
await self._db.commit()
return {"statuses_reset": statuses_reset, "locks_cleared": locks_cleared}
def _row_to_task(self, row: Any, description: Any) -> Task:
cols = [d[0] for d in description]
data = dict(zip(cols, row))
@@ -5509,6 +5452,108 @@ class OPCStore:
await self.save_delegation_work_item(item)
return item
async def claim_delegation_work_item_if_dispatchable(
self,
work_item_id: str,
*,
expected_phase: Phase | str,
role_runtime_session_id: str,
seat_id: str,
task_id: str,
work_item_revision: int = 0,
) -> DelegationWorkItem | None:
"""Atomically claim an unheld, unowned dispatchable WorkItem.
The dispatcher may be operating on a snapshot loaded before a Stop or
shutdown transition. Keeping the phase, claim, queue, and durable
hold predicates in the same UPDATE prevents that stale snapshot from
resurrecting a suspended WorkItem.
"""
phase = coerce_phase(expected_phase)
dispatchable_phases = {
Phase.READY,
Phase.READY_FOR_REWORK,
*IN_PROGRESS_PHASES,
}
if phase not in dispatchable_phases:
return None
if phase != Phase.RUNNING:
validate_transition(phase, Phase.RUNNING)
role_session_id = str(role_runtime_session_id or "").strip()
claimed_task_id = str(task_id or "").strip()
if not work_item_id or not role_session_id or not claimed_task_id:
return None
updated_at = datetime.now()
db = self._require_db()
cursor = await db.execute(
"""UPDATE delegation_work_items
SET phase = ?,
role_runtime_session_id = ?,
claimed_by_role_runtime_session_id = ?,
claimed_by_seat_id = ?,
metadata = json_set(
COALESCE(NULLIF(metadata, ''), '{}'),
'$.claimed_by_role_session_id', ?,
'$.claimed_task_id', ?,
'$.claimed_work_item_revision', ?
),
updated_at = ?
WHERE work_item_id = ?
AND phase = ?
AND COALESCE(claimed_by_role_runtime_session_id, '') = ''
AND COALESCE(claimed_by_seat_id, '') = ''
AND COALESCE(json_extract(metadata, '$.claimed_by_role_session_id'), '') = ''
AND COALESCE(json_extract(metadata, '$.claimed_task_id'), '') = ''
AND COALESCE(json_extract(metadata, '$.dispatch_hold'), '') = ''
AND COALESCE(json_extract(metadata, '$.queued_behind_session'), '') = ''
AND COALESCE(
CAST(json_extract(metadata, '$.manager_mutation_revision') AS INTEGER),
0
) = ?""",
(
Phase.RUNNING.value,
role_session_id,
role_session_id,
str(seat_id or "").strip(),
role_session_id,
claimed_task_id,
int(work_item_revision or 0),
updated_at.isoformat(),
str(work_item_id or "").strip(),
phase.value,
int(work_item_revision or 0),
),
)
await db.commit()
if not (getattr(cursor, "rowcount", 0) or 0):
return None
persisted = await self.get_delegation_work_item(work_item_id)
if persisted is None:
return None
persisted_metadata = dict(persisted.metadata or {})
if (
persisted.phase != Phase.RUNNING
or str(persisted.claimed_by_role_runtime_session_id or "").strip()
!= role_session_id
or str(persisted_metadata.get("claimed_by_role_session_id", "") or "").strip()
!= role_session_id
or str(persisted_metadata.get("claimed_task_id", "") or "").strip()
!= claimed_task_id
or str(persisted_metadata.get("dispatch_hold", "") or "").strip()
or str(persisted_metadata.get("queued_behind_session", "") or "").strip()
):
# Stop/shutdown may have won immediately after the CAS commit and
# cleared this claim while adding a durable hold. The committed
# UPDATE is not permission to spawn once that newer state exists.
return None
# Do not fire the generic phase hooks here. The executor's first
# idempotent RUNNING transition performs projection. Deferring it
# avoids a post-commit hook racing after Stop and projecting the Task
# back to RUNNING from an already-held WorkItem.
return persisted
async def apply_delegation_review_resolution(
self,
work_item_id: str,
@@ -7148,6 +7193,262 @@ class OPCStore:
)
await self._db.commit()
async def get_or_create_active_execution_checkpoint(
self,
checkpoint: ExecutionCheckpoint,
*,
checkpoint_types: list[str] | tuple[str, ...] | set[str],
create_if_missing: bool = True,
) -> tuple[ExecutionCheckpoint | None, bool]:
"""Atomically reuse or create one active checkpoint for a session scope.
``BEGIN IMMEDIATE`` serializes checkpoint creation across independent
controller/store connections. It also repairs historical duplicate
active rows in the same transaction, so concurrent startup/shutdown
reconcilers can never supersede each other's newly-created checkpoint
and leave the scope without a durable recovery point.
"""
assert self._db
clean_types = sorted(
{
str(item).strip()
for item in checkpoint_types
if str(item).strip()
}
)
project_id = str(checkpoint.project_id or "default").strip() or "default"
session_id = str(checkpoint.session_id or "").strip()
checkpoint_type = str(checkpoint.checkpoint_type or "").strip()
if not session_id:
raise ValueError("active execution checkpoint requires session_id")
if checkpoint_type not in clean_types:
raise ValueError(
"checkpoint_type must be included in the active checkpoint scope"
)
placeholders = ", ".join("?" for _ in clean_types)
try:
await self._db.execute("BEGIN IMMEDIATE")
async with self._db.execute(
f"""SELECT * FROM execution_checkpoints
WHERE project_id = ?
AND session_id = ?
AND checkpoint_type IN ({placeholders})
AND status IN ('pending', 'resuming')
ORDER BY updated_at DESC, created_at DESC, checkpoint_id DESC""",
(project_id, session_id, *clean_types),
) as cursor:
rows = await cursor.fetchall()
cols = [description[0] for description in cursor.description]
if rows:
decoded = [dict(zip(cols, row)) for row in rows]
winner_data = decoded[0]
winner_id = str(winner_data["checkpoint_id"])
now = datetime.now().isoformat()
for duplicate in decoded[1:]:
duplicate_id = str(duplicate.get("checkpoint_id", "") or "").strip()
if not duplicate_id:
continue
payload = _json_loads(duplicate.get("payload"), {})
payload["superseded_at"] = now
payload["superseded_by_checkpoint_id"] = winner_id
await self._db.execute(
"""UPDATE execution_checkpoints
SET status = 'superseded', payload = ?, updated_at = ?
WHERE checkpoint_id = ? AND status IN ('pending', 'resuming')""",
(_json_dumps(payload), now, duplicate_id),
)
await self._db.commit()
return (
ExecutionCheckpoint(
checkpoint_id=winner_id,
project_id=str(winner_data["project_id"]),
session_id=winner_data.get("session_id"),
checkpoint_type=str(winner_data["checkpoint_type"]),
status=str(winner_data["status"]),
task_id=winner_data.get("task_id"),
payload=_json_loads(winner_data.get("payload"), {}),
created_at=datetime.fromisoformat(str(winner_data["created_at"])),
updated_at=datetime.fromisoformat(str(winner_data["updated_at"])),
),
False,
)
if not create_if_missing:
await self._db.commit()
return None, False
await self._db.execute(
"""INSERT INTO execution_checkpoints
(checkpoint_id, project_id, session_id, checkpoint_type, status,
task_id, payload, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(
checkpoint.checkpoint_id,
project_id,
session_id,
checkpoint_type,
checkpoint.status,
checkpoint.task_id,
_json_dumps(checkpoint.payload),
checkpoint.created_at.isoformat(),
checkpoint.updated_at.isoformat(),
),
)
await self._db.commit()
return checkpoint, True
except Exception:
await self._db.rollback()
raise
async def normalize_active_execution_checkpoints(
self,
*,
project_id: str,
session_id: str,
checkpoint_types: list[str] | tuple[str, ...] | set[str],
) -> ExecutionCheckpoint | None:
"""Return one active scope owner while superseding duplicate rows.
Unlike creation, startup reconciliation must not manufacture a
checkpoint merely because none exists. This wrapper reuses the same
serialized transaction and duplicate winner rule with insertion
explicitly disabled.
"""
clean_types = sorted(
{
str(item).strip()
for item in checkpoint_types
if str(item).strip()
}
)
if not clean_types or not str(session_id or "").strip():
return None
candidate = ExecutionCheckpoint(
project_id=str(project_id or "default").strip() or "default",
session_id=str(session_id or "").strip(),
checkpoint_type=clean_types[0],
)
checkpoint, _created = await self.get_or_create_active_execution_checkpoint(
candidate,
checkpoint_types=clean_types,
create_if_missing=False,
)
return checkpoint
async def compare_and_set_execution_checkpoint(
self,
checkpoint_id: str,
*,
expected_statuses: list[str] | tuple[str, ...] | set[str],
status: str,
payload: dict[str, Any],
updated_at: datetime | None = None,
) -> bool:
"""Atomically transition a checkpoint only from an expected state.
The conditional UPDATE is the cross-controller guard for checkpoint
consumption. In-memory scope locks serialize one Office controller;
this CAS prevents a standalone CLI/controller from claiming the same
pending runtime concurrently.
"""
assert self._db
expected = [
str(item).strip()
for item in expected_statuses
if str(item).strip()
]
if not checkpoint_id or not expected:
return False
placeholders = ", ".join("?" for _ in expected)
cursor = await self._db.execute(
f"""UPDATE execution_checkpoints
SET status = ?, payload = ?, updated_at = ?
WHERE checkpoint_id = ? AND status IN ({placeholders})""",
(
str(status or "").strip(),
_json_dumps(dict(payload or {})),
(updated_at or datetime.now()).isoformat(),
checkpoint_id,
*expected,
),
)
await self._db.commit()
return cursor.rowcount == 1
async def complete_execution_checkpoint_and_reopen_ui_anchor(
self,
checkpoint_id: str,
*,
project_id: str,
session_id: str,
expected_status: str,
status: str,
payload: dict[str, Any],
ui_anchor_task_id: str = "",
updated_at: datetime | None = None,
) -> bool:
"""Atomically complete a runtime handoff and reopen its UI anchor.
The anchor must never become chat-runnable if a concurrent Stop already
took checkpoint ownership back. Keeping both writes in one SQLite
transaction makes the checkpoint CAS the gate for the UI projection.
"""
assert self._db
checkpoint_id = str(checkpoint_id or "").strip()
project_id = str(project_id or "default").strip() or "default"
session_id = str(session_id or "").strip()
expected_status = str(expected_status or "").strip()
status = str(status or "").strip()
ui_anchor_task_id = str(ui_anchor_task_id or "").strip()
if not checkpoint_id or not session_id or not expected_status or not status:
return False
now = updated_at or datetime.now()
try:
await self._db.execute("BEGIN IMMEDIATE")
cursor = await self._db.execute(
"""UPDATE execution_checkpoints
SET status = ?, payload = ?, updated_at = ?
WHERE checkpoint_id = ?
AND project_id = ?
AND session_id = ?
AND status = ?""",
(
status,
_json_dumps(dict(payload or {})),
now.isoformat(),
checkpoint_id,
project_id,
session_id,
expected_status,
),
)
if cursor.rowcount != 1:
await self._db.rollback()
return False
if ui_anchor_task_id:
await self._db.execute(
"""UPDATE tasks
SET status = ?, execution_lock = 0, execution_locked_at = NULL
WHERE id = ? AND project_id = ? AND status = ?""",
(
TaskStatus.IDLE.value,
ui_anchor_task_id,
project_id,
TaskStatus.CANCELLED.value,
),
)
await self._db.commit()
return True
except Exception:
await self._db.rollback()
raise
async def get_execution_checkpoints(
self,
project_id: str = "default",
opc/engine.py
+1470 -550
View File
File diff suppressed because it is too large Load Diff
opc/layer2_organization/company_mode.py
+195 -142
View File
@@ -17,6 +17,10 @@ from typing import Any, Awaitable, Callable
from loguru import logger
from opc.core.active_task_runs import (
ActiveTaskRunAdmissionClosed,
ActiveTaskRunRegistry,
)
from opc.core.config import DEFAULT_EXTERNAL_AGENT_STARTUP_TIMEOUT_SECONDS, DEFAULT_ORGANIZATION_ID
from opc.core.models import (
AdaptiveRoleProfile,
@@ -456,6 +460,26 @@ class WorkItemOutputBundle:
summary: str = ""
@dataclass(frozen=True)
class CompanyExecutorDriverOwnership:
"""One registry attempt covering a complete company scheduler run."""
registry: ActiveTaskRunRegistry
project_id: str
task_id: str
attempt_token: str
def bind(self):
return self.registry.bind_driver_attempt(self.attempt_token)
def release(self) -> bool:
return self.registry.unregister(
self.project_id,
self.task_id,
self.attempt_token,
)
def serialize_company_runtime_spec(spec: CompanyRuntimeSpec | None) -> dict[str, Any]:
if spec is None:
return {}
@@ -1354,6 +1378,7 @@ class CompanyWorkItemExecutor:
store: Any | None = None,
llm: Any | None = None,
role_prompt_runner: Callable[[Task, str, dict[str, Any], str, bool], Awaitable[str | None]] | None = None,
active_task_run_registry: ActiveTaskRunRegistry | None = None,
) -> None:
self.org_engine = org_engine
self.communication = communication
@@ -1373,6 +1398,7 @@ class CompanyWorkItemExecutor:
self.on_kanban_changed = on_kanban_changed
self.work_item_timeout = work_item_timeout
self.role_prompt_runner = role_prompt_runner
self.active_task_run_registry = active_task_run_registry
self._default_run_state = CompanyExecutorRunState()
self._run_state_var: ContextVar[CompanyExecutorRunState | None] = ContextVar(
f"company-executor-run-state:{id(self)}",
@@ -3368,12 +3394,6 @@ class CompanyWorkItemExecutor:
"cell_id": work_item.cell_id,
"parent_work_item_id": work_item.parent_work_item_id,
}
if work_item.phase == Phase.PAUSED:
task.metadata.setdefault("interrupted_recovery", {
"reason": "work_item_interrupted",
"detected_at": datetime.now().isoformat(),
})
if task.metadata != before_metadata:
changed = True
return changed
@@ -4359,14 +4379,76 @@ class CompanyWorkItemExecutor:
return "dispatch_required"
return "worker_execute"
async def execute(self, plan: CompanyWorkItemRuntimePlan, tasks: list[Task]) -> str:
plan = _coerce_company_work_item_runtime_plan(plan) or CompanyWorkItemRuntimePlan()
plan.metadata = {
**dict(plan.metadata or {}),
"execution_model": "multi_team_org",
"runtime_model": "multi_team_org",
@staticmethod
def _driver_ownership_task(
tasks: list[Task],
*,
preferred_task_ids: set[str] | None = None,
) -> Task | None:
preferred = {
str(task_id or "").strip()
for task_id in set(preferred_task_ids or set())
if str(task_id or "").strip()
}
return await self._execute_multi_team_org(plan, tasks)
candidates = [
task
for task in tasks
if str(getattr(task, "id", "") or "").strip()
and (not preferred or task.id in preferred)
]
for task in candidates:
if linked_work_item_id_for_task(task):
return task
for task in candidates:
if is_work_item_runtime_metadata(dict(task.metadata or {})):
return task
return candidates[0] if candidates else None
def acquire_driver_ownership(
self,
tasks: list[Task],
*,
preferred_task_ids: set[str] | None = None,
) -> CompanyExecutorDriverOwnership | None:
registry = self.active_task_run_registry
task = self._driver_ownership_task(
tasks,
preferred_task_ids=preferred_task_ids,
)
if registry is None or task is None:
return None
project_id = str(task.project_id or "default").strip() or "default"
try:
attempt_token = registry.register(project_id, task.id)
except ActiveTaskRunAdmissionClosed as exc:
raise asyncio.CancelledError(str(exc)) from exc
return CompanyExecutorDriverOwnership(
registry=registry,
project_id=project_id,
task_id=task.id,
attempt_token=attempt_token,
)
async def execute(
self,
plan: CompanyWorkItemRuntimePlan,
tasks: list[Task],
) -> str:
ownership = self.acquire_driver_ownership(tasks)
try:
plan = _coerce_company_work_item_runtime_plan(plan) or CompanyWorkItemRuntimePlan()
plan.metadata = {
**dict(plan.metadata or {}),
"execution_model": "multi_team_org",
"runtime_model": "multi_team_org",
}
if ownership is None:
return await self._execute_multi_team_org(plan, tasks)
with ownership.bind():
return await self._execute_multi_team_org(plan, tasks)
finally:
if ownership is not None:
ownership.release()
async def _execute_multi_team_org(
self,
@@ -4501,11 +4583,11 @@ class CompanyWorkItemExecutor:
# Claim whatever is immediately claimable and spawn each
# work item as an independent asyncio.Task so the loop no
# longer blocks on the slowest sibling.
claims = await self.runtime.claim_runnable_tasks(tasks, work_items=work_items)
for member_session, claimed_task in claims:
work_item_coro = self._run_claimed_work_item(member_session, claimed_task, {})
work_item_task = asyncio.create_task(work_item_coro)
active_work_item_tasks[work_item_task] = (member_session, claimed_task)
claims = await self._claim_and_create_work_item_tasks(
tasks,
work_items,
active_work_item_tasks,
)
# Termination: only when nothing is in-flight AND nothing
# else is runnable. If work items are still running, even an
# "empty runnable" snapshot may become non-empty within
@@ -4608,10 +4690,6 @@ class CompanyWorkItemExecutor:
self._schedule_kanban_notification()
except asyncio.CancelledError:
claimed_pairs = list(active_work_item_tasks.values())
claimed_tasks = [
claimed_task
for _member_session, claimed_task in claimed_pairs
]
for work_item_task in list(active_work_item_tasks.keys()):
if not work_item_task.done():
work_item_task.cancel()
@@ -4660,63 +4738,6 @@ class CompanyWorkItemExecutor:
)
except Exception:
logger.opt(exception=True).debug("company runtime cancellation: failed session idle reset")
for claimed_task in claimed_tasks:
if claimed_task.status in {TaskStatus.DONE, TaskStatus.FAILED, TaskStatus.CANCELLED}:
continue
claimed_task.metadata = dict(claimed_task.metadata or {})
claimed_task.metadata["company_runtime_suspended_at"] = datetime.now().isoformat()
claimed_task.metadata.setdefault("last_stop_reason", "runtime_cancelled")
claimed_task.metadata["company_runtime_stop_state"] = "suspended"
claimed_task.metadata["company_runtime_stop_marked_at"] = (
claimed_task.metadata.get("company_runtime_stop_marked_at") or datetime.now().isoformat()
)
claimed_task.metadata.setdefault(
"suspended_task_status",
claimed_task.status.value if isinstance(claimed_task.status, TaskStatus) else str(claimed_task.status or ""),
)
work_item_id = linked_work_item_id_for_task(claimed_task)
try:
if work_item_id and self._store_is_ready(self.store) and hasattr(self.store, "get_delegation_work_item"):
work_item = await self.store.get_delegation_work_item(work_item_id)
else:
work_item = None
if work_item is not None and getattr(work_item, "phase", None) not in {Phase.APPROVED, Phase.FAILED, Phase.CANCELLED}:
phase = getattr(work_item, "phase", Phase.RUNNING)
phase_value = phase.value if isinstance(phase, Phase) else str(phase or "")
original_claim = {
"claimed_by_role_runtime_session_id": str(getattr(work_item, "claimed_by_role_runtime_session_id", "") or ""),
"claimed_by_seat_id": str(getattr(work_item, "claimed_by_seat_id", "") or ""),
"claimed_by_role_session_id": str((getattr(work_item, "metadata", {}) or {}).get("claimed_by_role_session_id", "") or ""),
"claimed_task_id": str((getattr(work_item, "metadata", {}) or {}).get("claimed_task_id", "") or claimed_task.id),
}
await self.store.update_delegation_work_item(
work_item_id,
metadata_updates={
"dispatch_hold": "company_runtime_suspended",
"suspended_at": datetime.now().isoformat(),
"suspend_reason": claimed_task.metadata.get("last_stop_reason", "runtime_cancelled"),
"suspended_phase": phase_value,
"suspended_task_status": claimed_task.metadata.get("suspended_task_status", ""),
"suspended_claim": original_claim,
"claimed_by_role_session_id": "",
"claimed_task_id": "",
},
claimed_by_role_runtime_session_id="",
claimed_by_seat_id="",
)
claimed_task.metadata["dispatch_hold"] = "company_runtime_suspended"
claimed_task.metadata["suspended_phase"] = phase_value
except Exception:
logger.opt(exception=True).debug(
"company runtime cancellation: failed suspend hold release",
)
if self.save_task and self._store_is_ready(self.store):
try:
await self.save_task(claimed_task)
except Exception:
logger.opt(exception=True).debug(
"company runtime cancellation: failed suspended task save",
)
raise
finally:
# Drain any work items still running (shouldn't happen given the
@@ -4739,6 +4760,97 @@ class CompanyWorkItemExecutor:
active_work_item_tasks.clear()
return self._summarize_multi_team_org_results(tasks)
@staticmethod
def _runtime_scope_for_tasks(tasks: list[Task]) -> tuple[str, str]:
project_id = "default"
runtime_session_id = ""
for task in tasks:
project_id = str(task.project_id or project_id).strip() or "default"
metadata = dict(task.metadata or {})
runtime_session_id = str(
getattr(task, "parent_session_id", "")
or metadata.get("company_runtime_root_session_id")
or metadata.get("parent_session_id")
or ""
).strip()
if runtime_session_id:
return project_id, runtime_session_id
for task in tasks:
runtime_session_id = str(getattr(task, "session_id", "") or "").strip()
if runtime_session_id:
return project_id, runtime_session_id
return project_id, ""
async def _claim_and_create_work_item_tasks(
self,
tasks: list[Task],
work_items: list[DelegationWorkItem],
active_work_item_tasks: dict[
asyncio.Task[TaskResult | None],
tuple[CompanyMemberSession, Task],
],
) -> list[tuple[CompanyMemberSession, Task]]:
"""Keep durable claim and coroutine ownership in one scope boundary."""
async def claim_and_create() -> list[tuple[CompanyMemberSession, Task]]:
claims = await self.runtime.claim_runnable_tasks(
tasks,
work_items=work_items,
)
for member_session, claimed_task in claims:
work_item_task = self._create_claimed_work_item_task(
member_session,
claimed_task,
{},
)
active_work_item_tasks[work_item_task] = (
member_session,
claimed_task,
)
return claims
registry = self.active_task_run_registry
project_id, runtime_session_id = self._runtime_scope_for_tasks(tasks)
if registry is None or not runtime_session_id:
return await claim_and_create()
async with registry.scope_lock(project_id, runtime_session_id):
return await claim_and_create()
def _create_claimed_work_item_task(
self,
member_session: CompanyMemberSession,
task: Task,
task_by_projection_id: dict[str, Task],
) -> asyncio.Task[TaskResult | None]:
"""Register ownership before scheduling the full claimed-item coroutine."""
registry = self.active_task_run_registry
project_id = str(task.project_id or "default").strip() or "default"
attempt_token = ""
if registry is not None:
try:
attempt_token = registry.register(project_id, task.id)
except ActiveTaskRunAdmissionClosed as exc:
raise asyncio.CancelledError(str(exc)) from exc
async def run_owned() -> TaskResult | None:
try:
return await self._run_claimed_work_item(
member_session,
task,
task_by_projection_id,
)
finally:
if registry is not None and attempt_token:
registry.unregister(project_id, task.id, attempt_token)
try:
return asyncio.create_task(run_owned())
except BaseException:
if registry is not None and attempt_token:
registry.unregister(project_id, task.id, attempt_token)
raise
async def _run_claimed_work_item(
self,
member_session: CompanyMemberSession,
@@ -5298,69 +5410,10 @@ class CompanyWorkItemExecutor:
timeout=self.work_item_timeout,
)
except asyncio.CancelledError:
task.metadata = dict(task.metadata or {})
task.metadata["company_runtime_suspended_at"] = datetime.now().isoformat()
task.metadata.setdefault("last_stop_reason", "runtime_cancelled")
task.metadata["company_runtime_stop_state"] = "suspended"
task.metadata["company_runtime_stop_marked_at"] = (
task.metadata.get("company_runtime_stop_marked_at") or datetime.now().isoformat()
)
task.metadata.setdefault(
"suspended_task_status",
task.status.value if isinstance(task.status, TaskStatus) else str(task.status or ""),
)
work_item_id = linked_work_item_id_for_task(task)
store_ready = self._store_is_ready(self.store)
if work_item_id and self.store and store_ready:
task.metadata.pop("progress_log", None)
await append_work_item_progress(
self.store,
work_item_id,
"Work item suspended by runtime cancellation.",
)
else:
progress = list(task.metadata.get("progress_log", []) or [])
progress.append("Work item suspended by runtime cancellation.")
task.metadata["progress_log"] = progress[-20:]
try:
work_item = (
await self.store.get_delegation_work_item(work_item_id)
if work_item_id and store_ready and hasattr(self.store, "get_delegation_work_item")
else None
)
if work_item is not None and getattr(work_item, "phase", None) not in {Phase.APPROVED, Phase.FAILED, Phase.CANCELLED}:
phase = getattr(work_item, "phase", Phase.RUNNING)
phase_value = phase.value if isinstance(phase, Phase) else str(phase or "")
original_claim = {
"claimed_by_role_runtime_session_id": str(getattr(work_item, "claimed_by_role_runtime_session_id", "") or ""),
"claimed_by_seat_id": str(getattr(work_item, "claimed_by_seat_id", "") or ""),
"claimed_by_role_session_id": str((getattr(work_item, "metadata", {}) or {}).get("claimed_by_role_session_id", "") or ""),
"claimed_task_id": str((getattr(work_item, "metadata", {}) or {}).get("claimed_task_id", "") or task.id),
}
await self.store.update_delegation_work_item(
work_item_id,
metadata_updates={
"dispatch_hold": "company_runtime_suspended",
"suspended_at": datetime.now().isoformat(),
"suspend_reason": task.metadata.get("last_stop_reason", "runtime_cancelled"),
"suspended_phase": phase_value,
"suspended_task_status": task.metadata.get("suspended_task_status", ""),
"suspended_claim": original_claim,
"claimed_by_role_session_id": "",
"claimed_task_id": "",
},
claimed_by_role_runtime_session_id="",
claimed_by_seat_id="",
)
task.metadata["dispatch_hold"] = "company_runtime_suspended"
task.metadata["suspended_phase"] = phase_value
task.status = task_status_for_phase(phase) if isinstance(phase, Phase) else task.status
except Exception:
logger.opt(exception=True).debug(
"company runtime cancellation: failed to apply suspend hold",
)
if self.save_task and self._store_is_ready(self.store):
await self.save_task(task)
# Suspension is a checkpoint transition owned by OPCEngine.
# This task object may be stale by the time cancellation is
# observed, so persisting it here can erase the canonical
# checkpoint type, stop intent, or WorkItem hold.
raise
except asyncio.TimeoutError:
logger.error(f"Company work item {projection_id} timed out after {self.work_item_timeout}s")
opc/layer2_organization/company_runtime.py
+73 -35
View File
@@ -1373,7 +1373,6 @@ class CompanyRuntime:
_skip("no task materialized for work_item this tick",
session=session_label, work_item_id=work_item_id)
continue
self._claimed_work_item_ids.add(work_item_id)
else:
task_id = queued_item_id
self._queued_task_ids.discard(task_id)
@@ -1392,6 +1391,30 @@ class CompanyRuntime:
status=getattr(task, "status", None))
continue
self._claimed_task_ids.add(task_id)
if work_item is not None:
claimed = await self._claim_role_session_work_item(
session,
work_item,
task,
)
if not claimed:
fresh_work_item = None
get_work_item = getattr(
self.store,
"get_delegation_work_item",
None,
)
if callable(get_work_item):
fresh_work_item = await get_work_item(work_item_id)
if fresh_work_item is not None:
work_item_map[work_item_id] = fresh_work_item
_skip(
"atomic WorkItem claim lost to a phase/hold/owner update",
session=session_label,
work_item_id=work_item_id,
)
continue
self._claimed_work_item_ids.add(work_item_id)
if can_soft_wake and (
bool((task.metadata or {}).get("review_task", False))
or bool((task.metadata or {}).get("review_execution_work_item", False))
@@ -1412,7 +1435,6 @@ class CompanyRuntime:
self.prepare_task_for_session(session, task)
await self._sync_current_turn_mode_to_work_item(task, session.current_turn_mode)
if work_item is not None:
await self._claim_role_session_work_item(session, work_item, task)
# #7: mirror the claim onto the in-memory work_item so
# subsequent iterations in the same claim pass see it
# and is_dispatchable returns False (race safety after
@@ -1908,21 +1930,65 @@ class CompanyRuntime:
self.role_sessions[role_session_id] = role_session
return role_session
async def _claim_role_session_work_item(self, session: CompanyMemberSession, work_item: Any, task: Task) -> None:
async def _claim_role_session_work_item(
self,
session: CompanyMemberSession,
work_item: Any,
task: Task,
) -> bool:
"""Atomically claim ``work_item`` for the role-instance behind
``session``.
In the role-instance model the claim identity is
``role_runtime_session_id``. Seat / manager-seat columns are
still written for org-chart lookups but they are NOT part of
the claim key — only the role session is.
the claim key — only the role session is. Pure in-memory runtimes have
no durable race to arbitrate and keep the same local claim semantics.
"""
work_item_id = str(getattr(work_item, "work_item_id", "") or "").strip()
if not work_item_id:
return
return False
role_session = self._ensure_role_session(task)
if role_session is None:
return
return False
work_item_revision = 0
try:
work_item_revision = int((getattr(work_item, "metadata", {}) or {}).get("manager_mutation_revision") or 0)
except (TypeError, ValueError):
work_item_revision = 0
store_ready = self.store is not None and bool(
getattr(self.store, "is_ready", False)
)
claim = (
getattr(self.store, "claim_delegation_work_item_if_dispatchable", None)
if store_ready
else None
)
if store_ready:
# A durable runtime must win the store CAS before it mutates any
# in-memory scheduling state. This is the Stop/shutdown race
# boundary: a missing CAS API is a failed claim, not permission to
# fall back to the former best-effort update path.
if not callable(claim):
return False
persisted = await claim(
work_item_id,
expected_phase=getattr(work_item, "phase", Phase.READY),
role_runtime_session_id=role_session.role_session_id,
seat_id=str(getattr(session, "seat_id", "") or "").strip(),
task_id=task.id,
work_item_revision=work_item_revision,
)
if persisted is None:
return False
work_item.phase = persisted.phase
work_item.role_runtime_session_id = persisted.role_runtime_session_id
work_item.claimed_by_role_runtime_session_id = (
persisted.claimed_by_role_runtime_session_id
)
work_item.claimed_by_seat_id = persisted.claimed_by_seat_id
work_item.metadata = dict(persisted.metadata or {})
ready_background_ids = [
item_id
for item_id in list(role_session.background_work_item_ids or [])
@@ -1937,39 +2003,11 @@ class CompanyRuntime:
session.background_work_item_ids = list(role_session.background_work_item_ids)
task.metadata = dict(task.metadata)
task.metadata["delegation_role_session_id"] = role_session.role_session_id
work_item_revision = 0
try:
work_item_revision = int((getattr(work_item, "metadata", {}) or {}).get("manager_mutation_revision") or 0)
except (TypeError, ValueError):
work_item_revision = 0
task.metadata["started_work_item_revision"] = work_item_revision
task.metadata["claimed_work_item_revision"] = work_item_revision
if self.store and bool(getattr(self.store, "is_ready", False)) and hasattr(self.store, "update_delegation_work_item"):
# Do not regress a work item that is already in a review
# phase (AWAITING_MANAGER_REVIEW / AWAITING_HUMAN) back to
# RUNNING: the DB phase validator rejects that transition
# and the error bubbles up to the session loop. This
# happens when the reactivation sweeper wakes a task whose
# work item has already been promoted to review — treat
# the claim as "refresh the task/role-session bindings
# only" and leave the phase alone.
current_phase = getattr(work_item, "phase", None)
phase_to_write: Phase | None = Phase.RUNNING
if current_phase in IN_REVIEW_PHASES:
phase_to_write = None
await self.store.update_delegation_work_item(
work_item_id,
phase=phase_to_write,
role_runtime_session_id=role_session.role_session_id,
claimed_by_role_runtime_session_id=role_session.role_session_id,
metadata_updates={
"claimed_by_role_session_id": role_session.role_session_id,
"claimed_task_id": task.id,
"claimed_work_item_revision": work_item_revision,
},
)
if self.store and bool(getattr(self.store, "is_ready", False)) and hasattr(self.store, "save_delegation_role_session"):
await self.store.save_delegation_role_session(role_session)
return True
def ensure_role_instance_session(
self, task: Task
opc/layer2_organization/company_runtime_identity.py
+391
View File
@@ -0,0 +1,391 @@
"""Canonical company-runtime identity derived from durable records.
Company-mode Tasks are execution envelopes, not the identity of a run. A
runtime is owned by its root session and an active suspend checkpoint. This
module deliberately has no UI dependencies so every surface can resolve the
same scope without relying on process-local task maps.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any, Iterable
from opc.layer2_organization.work_item_links import linked_work_item_id_for_task
from opc.layer2_organization.work_item_runtime import is_work_item_runtime_metadata
COMPANY_RUNTIME_CHECKPOINT_TYPES: frozenset[str] = frozenset({
"company_runtime_suspended",
"company_runtime_interrupted",
})
ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES: frozenset[str] = frozenset({
"pending",
"resuming",
})
def _text(value: Any) -> str:
return str(value or "").strip()
def _metadata(task: Any) -> dict[str, Any]:
return dict(getattr(task, "metadata", {}) or {})
def _task_id(task: Any) -> str:
return _text(getattr(task, "id", ""))
def _task_session_id(task: Any) -> str:
return _text(getattr(task, "session_id", ""))
def _task_parent_session_id(task: Any) -> str:
metadata = _metadata(task)
return _text(
getattr(task, "parent_session_id", "")
or metadata.get("company_runtime_root_session_id")
or metadata.get("parent_session_id")
)
def _has_company_runtime_marker(task: Any) -> bool:
metadata = _metadata(task)
exec_mode = _text(metadata.get("exec_mode")).lower()
mode = _text(metadata.get("mode")).lower()
execution_mode = _text(metadata.get("execution_mode")).lower()
if exec_mode in {"company", "org", "custom"} or mode in {"company", "org", "custom"}:
return True
if execution_mode in {"company", "company_mode", "multi_team_org"}:
return True
if is_work_item_runtime_metadata(metadata):
return True
if linked_work_item_id_for_task(task):
return True
if any(
metadata.get(key) not in (None, "", [], {})
for key in (
"company_work_item_plan",
"company_runtime_root_session_id",
"delegation_run_id",
"work_item_projection_id",
"work_item_projection_ref",
"work_item_role_id",
"shared_role_session",
)
):
return True
# Old company records may predate exec_mode. An explicit task-mode marker
# wins over the legacy profile hint.
explicitly_task_mode = (
exec_mode in {"task", "project", "single"}
or mode == "task"
or execution_mode in {"task", "task_mode", "project"}
or _text(metadata.get("task_mode_contract")) == "single_full_capability_main_agent"
)
return not explicitly_task_mode and bool(_text(metadata.get("company_profile")))
def is_company_runtime_task(task: Any) -> bool:
"""Return whether durable Task metadata identifies company-owned work."""
return _has_company_runtime_marker(task)
def is_pure_company_ui_anchor(task: Any, runtime_session_id: str) -> bool:
"""Return whether *task* is the user-facing container for a runtime.
A shared final-decider Task can have the same ``session_id`` as the UI
anchor. Work-item, role, or parent links therefore disqualify a Task even
when its session id is an exact match.
"""
session_id = _text(runtime_session_id)
if not session_id or _task_session_id(task) != session_id:
return False
if _text(getattr(task, "parent_session_id", "")) or _text(getattr(task, "parent_id", "")):
return False
if linked_work_item_id_for_task(task):
return False
metadata = _metadata(task)
return not any(
metadata.get(key) not in (None, "", [], {}, False)
for key in (
"work_item_runtime",
"work_item_projection_id",
"work_item_projection_ref",
"work_item_id",
"work_item_role_id",
"delegation_role_session_id",
"shared_role_session",
"shared_role_id",
"company_runtime_root_session_id",
)
)
def _created_sort_key(value: Any) -> tuple[float, str]:
created_at = getattr(value, "created_at", None)
if isinstance(created_at, datetime):
timestamp = created_at.timestamp()
elif hasattr(created_at, "timestamp"):
try:
timestamp = float(created_at.timestamp())
except Exception:
timestamp = 0.0
else:
timestamp = 0.0
return timestamp, _task_id(value)
def _checkpoint_sort_key(checkpoint: Any) -> tuple[float, float, str]:
def _timestamp(value: Any) -> float:
if isinstance(value, datetime):
return value.timestamp()
if hasattr(value, "timestamp"):
try:
return float(value.timestamp())
except Exception:
return 0.0
return 0.0
return (
_timestamp(getattr(checkpoint, "updated_at", None)),
_timestamp(getattr(checkpoint, "created_at", None)),
_text(getattr(checkpoint, "checkpoint_id", "")),
)
def _checkpoint_runtime_session_id(checkpoint: Any) -> str:
payload = dict(getattr(checkpoint, "payload", {}) or {})
return _text(
getattr(checkpoint, "session_id", "")
or payload.get("parent_session_id")
or payload.get("session_id")
)
@dataclass(frozen=True)
class CompanyRuntimeIdentity:
"""Resolved identity for one company runtime scope."""
project_id: str
runtime_session_id: str
runtime_task_ids: tuple[str, ...]
ui_anchor_task_id: str = ""
config_source_task_id: str = ""
pending_checkpoint_id: str = ""
pending_checkpoint_type: str = ""
pending_checkpoint_status: str = ""
resumable: bool = False
checkpoint: Any | None = field(default=None, repr=False, compare=False)
class CompanyRuntimeIdentityIndex:
"""Session-first index over preloaded Tasks and checkpoints."""
def __init__(self, tasks: Iterable[Any], checkpoints: Iterable[Any] = ()) -> None:
self.tasks = tuple(tasks or ())
self.checkpoints = tuple(checkpoints or ())
self.tasks_by_id = {
_task_id(task): task
for task in self.tasks
if _task_id(task)
}
self.checkpoints_by_id = {
_text(getattr(checkpoint, "checkpoint_id", "")): checkpoint
for checkpoint in self.checkpoints
if _text(getattr(checkpoint, "checkpoint_id", ""))
}
self._identities_by_session = self._build_identities()
self._runtime_session_by_task_id: dict[str, str] = {}
runtime_sessions_by_task_session_id: dict[str, set[str]] = {}
for runtime_session_id, identity in self._identities_by_session.items():
for task_id in identity.runtime_task_ids:
self._runtime_session_by_task_id[task_id] = runtime_session_id
task_session_id = _task_session_id(self.tasks_by_id.get(task_id))
if task_session_id:
runtime_sessions_by_task_session_id.setdefault(
task_session_id,
set(),
).add(runtime_session_id)
self._runtime_session_by_task_session_id = {
task_session_id: next(iter(runtime_session_ids))
for task_session_id, runtime_session_ids in runtime_sessions_by_task_session_id.items()
if len(runtime_session_ids) == 1
}
@property
def identities(self) -> tuple[CompanyRuntimeIdentity, ...]:
return tuple(self._identities_by_session.values())
def task(self, task_id: str) -> Any | None:
return self.tasks_by_id.get(_text(task_id))
def resolve(
self,
*,
task_id: str = "",
task_session_id: str = "",
runtime_session_id: str = "",
checkpoint_id: str = "",
) -> CompanyRuntimeIdentity | None:
requested_task_id = _text(task_id)
requested_task_session_id = _text(task_session_id)
requested_session_id = _text(runtime_session_id)
requested_checkpoint_id = _text(checkpoint_id)
task_scope_id = self._runtime_session_by_task_id.get(requested_task_id, "")
session_scope_id = self._runtime_session_by_task_session_id.get(
requested_task_session_id,
"",
)
checkpoint = self.checkpoints_by_id.get(requested_checkpoint_id) if requested_checkpoint_id else None
checkpoint_session_id = _checkpoint_runtime_session_id(checkpoint) if checkpoint is not None else ""
candidates = {
value
for value in (
requested_session_id,
task_scope_id,
session_scope_id,
checkpoint_session_id,
)
if value
}
if requested_task_session_id and not session_scope_id:
return None
if len(candidates) != 1:
return None
resolved_session_id = next(iter(candidates))
identity = self._identities_by_session.get(resolved_session_id)
if identity is None:
return None
if requested_task_id and requested_task_id not in identity.runtime_task_ids:
return None
if requested_checkpoint_id and requested_checkpoint_id != identity.pending_checkpoint_id:
return None
return identity
def _build_identities(self) -> dict[str, CompanyRuntimeIdentity]:
active_checkpoints_by_session: dict[str, list[Any]] = {}
for checkpoint in self.checkpoints:
checkpoint_type = _text(getattr(checkpoint, "checkpoint_type", ""))
checkpoint_status = _text(getattr(checkpoint, "status", "")).lower()
if (
checkpoint_type not in COMPANY_RUNTIME_CHECKPOINT_TYPES
or checkpoint_status not in ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES
):
continue
runtime_session_id = _checkpoint_runtime_session_id(checkpoint)
if runtime_session_id:
active_checkpoints_by_session.setdefault(runtime_session_id, []).append(checkpoint)
known_sessions = set(active_checkpoints_by_session)
for task in self.tasks:
if not _has_company_runtime_marker(task):
continue
runtime_session_id = _task_parent_session_id(task) or _task_session_id(task)
if runtime_session_id:
known_sessions.add(runtime_session_id)
tasks_by_session: dict[str, list[Any]] = {session_id: [] for session_id in known_sessions}
for task in self.tasks:
task_id = _task_id(task)
if not task_id:
continue
parent_session_id = _task_parent_session_id(task)
own_session_id = _task_session_id(task)
runtime_session_id = parent_session_id or own_session_id
if runtime_session_id not in known_sessions:
continue
if not (
_has_company_runtime_marker(task)
or runtime_session_id in active_checkpoints_by_session
or is_pure_company_ui_anchor(task, runtime_session_id)
):
continue
tasks_by_session.setdefault(runtime_session_id, []).append(task)
identities: dict[str, CompanyRuntimeIdentity] = {}
for runtime_session_id in sorted(known_sessions):
group = sorted(tasks_by_session.get(runtime_session_id, []), key=_created_sort_key)
anchor = next(
(task for task in group if is_pure_company_ui_anchor(task, runtime_session_id)),
None,
)
def _has_runtime_config(task: Any) -> bool:
metadata = _metadata(task)
return any(
metadata.get(key) not in (None, "", [], {})
for key in (
"exec_mode",
"mode",
"company_profile",
"org_id",
"organization_id",
"preferred_agent",
"selected_execution_agent",
)
)
config_source = (
anchor if anchor is not None and _has_runtime_config(anchor) else None
) or next(
(
task for task in group
if _has_runtime_config(task)
),
anchor or (group[0] if group else None),
)
checkpoint_candidates = active_checkpoints_by_session.get(runtime_session_id, [])
checkpoint = max(checkpoint_candidates, key=_checkpoint_sort_key) if checkpoint_candidates else None
checkpoint_status = _text(getattr(checkpoint, "status", "")).lower() if checkpoint is not None else ""
project_id = _text(
getattr(checkpoint, "project_id", "") if checkpoint is not None else ""
) or _text(getattr(config_source, "project_id", "") if config_source is not None else "") or "default"
identities[runtime_session_id] = CompanyRuntimeIdentity(
project_id=project_id,
runtime_session_id=runtime_session_id,
runtime_task_ids=tuple(_task_id(task) for task in group if _task_id(task)),
ui_anchor_task_id=_task_id(anchor) if anchor is not None else "",
config_source_task_id=_task_id(config_source) if config_source is not None else "",
pending_checkpoint_id=_text(getattr(checkpoint, "checkpoint_id", "")) if checkpoint is not None else "",
pending_checkpoint_type=_text(getattr(checkpoint, "checkpoint_type", "")) if checkpoint is not None else "",
pending_checkpoint_status=checkpoint_status,
resumable=checkpoint_status == "pending",
checkpoint=checkpoint,
)
return identities
def build_company_runtime_identity_index(
tasks: Iterable[Any],
checkpoints: Iterable[Any] = (),
) -> CompanyRuntimeIdentityIndex:
return CompanyRuntimeIdentityIndex(tasks, checkpoints)
async def load_company_runtime_identity_index(
store: Any,
project_id: str,
) -> CompanyRuntimeIdentityIndex:
"""Load durable records once and build the canonical runtime index."""
tasks = await store.get_tasks(project_id=project_id)
checkpoint_getter = getattr(store, "get_execution_checkpoints", None)
if callable(checkpoint_getter):
checkpoints = await checkpoint_getter(
project_id=project_id,
checkpoint_types=sorted(COMPANY_RUNTIME_CHECKPOINT_TYPES),
statuses=sorted(ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES),
)
else:
checkpoint_getter = getattr(store, "get_pending_checkpoints", None)
checkpoints = await checkpoint_getter(
project_id=project_id,
checkpoint_types=sorted(COMPANY_RUNTIME_CHECKPOINT_TYPES),
) if callable(checkpoint_getter) else []
return build_company_runtime_identity_index(tasks, checkpoints)
opc/layer2_organization/custom_runtime.py
+2
View File
@@ -75,6 +75,8 @@ class CustomRuntimeRunner:
store=shared_store,
owns_store=shared_store is None,
run_startup_reconcile=shared_store is None,
active_task_run_registry=getattr(self.parent, "_active_task_run_registry", None),
owns_active_task_run_registry=False,
on_progress=self.parent.on_progress,
on_runtime_event=self.parent.on_runtime_event,
on_escalation=self.parent.on_escalation,
opc/layer2_organization/seat_executor.py
+4 -2
View File
@@ -144,8 +144,10 @@ class EngineSeatExecutor:
member_session: CompanyMemberSession | None = None,
) -> None:
_ = member_session
if hasattr(self.host, "_active_task_runs"):
self.host._active_task_runs.discard(task.id)
# The task coroutine owns its registry attempt token and removes it in
# ``finally``. Interrupt requests must not make a still-running
# coroutine appear inactive.
return None
async def shutdown(
self,
opc/plugins/cli_board/services/recovery.py
-297
View File
@@ -1,297 +0,0 @@
"""CLI-board-specific company runtime recovery manager.
Independent from office_ui/recovery_manager.py — same Core Engine APIs,
different notification path (TUI event bridge instead of WebSocket broadcast).
"""
from __future__ import annotations
import asyncio
import logging
import time
from dataclasses import dataclass, field
from datetime import datetime
from typing import TYPE_CHECKING, Any
from opc.layer2_organization.work_item_identity import work_item_projection_id_from_metadata
from opc.layer2_organization.work_item_transition import apply_task_status_transition
if TYPE_CHECKING:
from .engine_facade import EngineFacade
logger = logging.getLogger(__name__)
@dataclass
class RecoverableWorkItem:
projection_id: str
title: str
task_id: str
status: str
interrupted: bool
previous_status: str = ""
@dataclass
class InterruptedCompanyRuntime:
parent_session_id: str
parent_task_id: str
project_id: str
title: str
profile: str
interrupted_at: str
work_items: list[RecoverableWorkItem] = field(default_factory=list)
@dataclass
class RecoveryStatus:
interrupted: list[InterruptedCompanyRuntime] = field(default_factory=list)
active_recoveries: list[str] = field(default_factory=list)
scanned_at: float = 0.0
def _is_interrupted(task: Any) -> bool:
from opc.core.models import TaskStatus
if task.status != TaskStatus.FAILED:
return False
meta = getattr(task, "metadata", {}) or {}
if meta.get("interrupted_recovery"):
return True
result = getattr(task, "result", {}) or {}
artifacts = result.get("artifacts", {}) or {}
return bool(artifacts.get("interrupted"))
class CliRecoveryManager:
"""Scan for interrupted company runtimes and provide resume/cancel."""
_CACHE_TTL = 10.0
def __init__(self, facade: EngineFacade) -> None:
self._facade = facade
self._lock = asyncio.Lock()
self._active: dict[str, asyncio.Task[Any]] = {}
self._cached: RecoveryStatus | None = None
self._cache_until: float = 0.0
@property
def _project_id(self) -> str:
return self._facade.project_id or "default"
async def get_status(self) -> RecoveryStatus:
now = time.time()
if self._cached is not None and now < self._cache_until:
self._cached.active_recoveries = list(self._active.keys())
return self._cached
status = await self.scan()
self._cached = status
self._cache_until = now + self._CACHE_TTL
return status
async def scan(self) -> RecoveryStatus:
engine = await self._facade.ensure_ready()
if not engine.store:
return RecoveryStatus()
try:
all_tasks = await engine.store.get_tasks(project_id=self._project_id)
except Exception as exc:
logger.warning("Recovery scan failed: %s", exc)
return RecoveryStatus()
groups: dict[str, list[Any]] = {}
tasks_by_session: dict[str, Any] = {}
for task in all_tasks:
sid = str(getattr(task, "session_id", "") or "").strip()
if sid:
tasks_by_session[sid] = task
parent_sid = str(getattr(task, "parent_session_id", "") or "").strip()
projection_id = work_item_projection_id_from_metadata(getattr(task, "metadata", {}) or {})
if parent_sid and projection_id:
groups.setdefault(parent_sid, []).append(task)
from opc.core.models import TaskStatus
interrupted: list[InterruptedCompanyRuntime] = []
for parent_sid, tasks in groups.items():
if not any(_is_interrupted(t) for t in tasks):
continue
non_terminal = [t for t in tasks if t.status not in (TaskStatus.DONE, TaskStatus.CANCELLED)]
if not non_terminal:
continue
parent_task = tasks_by_session.get(parent_sid)
parent_task_id = parent_task.id if parent_task else parent_sid
title = parent_task.title if parent_task else "Unknown company runtime"
work_items: list[RecoverableWorkItem] = []
earliest = ""
for t in sorted(tasks, key=lambda x: (x.created_at, x.id)):
meta = dict(getattr(t, "metadata", {}) or {})
rmeta = meta.get("interrupted_recovery", {})
is_int = _is_interrupted(t)
if is_int and rmeta.get("detected_at", ""):
det = rmeta["detected_at"]
if not earliest or det < earliest:
earliest = det
work_items.append(RecoverableWorkItem(
projection_id=work_item_projection_id_from_metadata(meta, fallback=t.id),
title=t.title,
task_id=t.id,
status=t.status.value if hasattr(t.status, "value") else str(t.status),
interrupted=is_int,
previous_status=rmeta.get("previous_status", ""),
))
profile = ""
for t in tasks:
p = (getattr(t, "metadata", {}) or {}).get("company_profile", "")
if p:
profile = p
break
interrupted.append(InterruptedCompanyRuntime(
parent_session_id=parent_sid,
parent_task_id=parent_task_id,
project_id=self._project_id,
title=title,
profile=profile,
interrupted_at=earliest or datetime.now().isoformat(),
work_items=work_items,
))
return RecoveryStatus(
interrupted=interrupted,
active_recoveries=list(self._active.keys()),
scanned_at=time.time(),
)
async def resume(self, parent_task_id: str) -> dict[str, Any]:
async with self._lock:
if parent_task_id in self._active:
return {"ok": False, "error": "already_in_progress"}
status = await self.scan()
wf = next((w for w in status.interrupted if w.parent_task_id == parent_task_id), None)
if not wf:
return {"ok": False, "error": "not_found"}
engine = await self._facade.ensure_ready()
snapshot = await engine._load_company_runtime_snapshot(wf.parent_session_id)
if not snapshot:
return {"ok": False, "error": "snapshot_unavailable"}
plan, tasks = snapshot
await self._clean_checkpoints(wf, tasks)
from opc.core.models import TaskStatus
resumed_ids: list[str] = []
for task in tasks:
if task.status == TaskStatus.DONE:
continue
if task.status in (TaskStatus.FAILED, TaskStatus.BLOCKED):
task.result = None
task.execution_lock = False
task.execution_locked_at = None
meta = dict(task.metadata)
meta.pop("interrupted_recovery", None)
progress = list(meta.get("progress_log", []))
progress.append(f"[Recovery] Resumed at {datetime.now().isoformat()}")
meta["progress_log"] = progress[-20:]
task.metadata = meta
try:
await apply_task_status_transition(
engine.store,
task,
target_status_or_phase=TaskStatus.PENDING,
reason="cli_recovery_resume",
release_claim=True,
)
except Exception as exc:
logger.warning("Recovery resume skipped %s: %s", task.id, exc)
continue
if task.status != TaskStatus.PENDING:
logger.warning("Recovery resume preserved non-runnable phase for %s", task.id)
continue
await engine.store.save_task(task)
resumed_ids.append(work_item_projection_id_from_metadata(meta, fallback=task.id))
if not resumed_ids:
return {"ok": False, "error": "no_work_items_to_resume"}
self._cache_until = 0.0
bg = asyncio.create_task(self._execute(parent_task_id, plan, tasks))
self._active[parent_task_id] = bg
return {"ok": True, "resumed_work_item_projection_ids": resumed_ids}
async def cancel(self, parent_task_id: str) -> dict[str, Any]:
async with self._lock:
bg = self._active.pop(parent_task_id, None)
if bg and not bg.done():
bg.cancel()
status = await self.scan()
wf = next((w for w in status.interrupted if w.parent_task_id == parent_task_id), None)
if not wf:
return {"ok": False, "error": "not_found"}
engine = await self._facade.ensure_ready()
snapshot = await engine._load_company_runtime_snapshot(wf.parent_session_id)
if not snapshot:
return {"ok": False, "error": "snapshot_unavailable"}
_, tasks = snapshot
from opc.core.models import TaskStatus
cancelled = 0
for task in tasks:
if task.status not in (TaskStatus.DONE, TaskStatus.CANCELLED):
try:
await apply_task_status_transition(
engine.store,
task,
target_status_or_phase=TaskStatus.CANCELLED,
reason="cli_recovery_cancel",
release_claim=True,
)
except Exception as exc:
logger.warning("Recovery cancel skipped %s: %s", task.id, exc)
continue
if task.status != TaskStatus.CANCELLED:
logger.warning("Recovery cancel preserved non-cancelled phase for %s", task.id)
continue
cancelled += 1
await self._clean_checkpoints(wf, tasks)
self._cache_until = 0.0
return {"ok": True, "cancelled_count": cancelled}
async def _execute(self, parent_task_id: str, plan: Any, tasks: list[Any]) -> None:
try:
engine = await self._facade.ensure_ready()
executor = engine.company_executor
if not executor:
raise RuntimeError("company_executor not available")
await executor.execute(plan, tasks)
except asyncio.CancelledError:
pass
except Exception as exc:
logger.warning("Recovery execution failed for %s: %s", parent_task_id, exc)
finally:
self._active.pop(parent_task_id, None)
self._cache_until = 0.0
async def _clean_checkpoints(self, wf: InterruptedCompanyRuntime, tasks: list[Any]) -> None:
engine = await self._facade.ensure_ready()
if not engine.store:
return
session_ids = {str(getattr(t, "session_id", "") or "").strip() for t in tasks}
session_ids.add(wf.parent_session_id)
session_ids.discard("")
try:
pending = await engine.store.get_pending_checkpoints(project_id=wf.project_id)
for cp in pending:
if str(cp.session_id or "").strip() in session_ids:
await engine.store.resolve_execution_checkpoint(cp.checkpoint_id, status="cancelled")
except Exception as exc:
logger.debug("Checkpoint cleanup error: %s", exc)
opc/plugins/cli_board/tui/app.py
-39
View File
@@ -15,7 +15,6 @@ from opc.plugins.cli_board.state.store import BoardStateStore
from opc.plugins.cli_board.tui.screens.help import HelpScreen
from opc.plugins.cli_board.tui.screens.palette import CommandPaletteScreen, PaletteCommand
from opc.plugins.cli_board.tui.screens.prompt import PromptField, PromptScreen
from opc.plugins.cli_board.tui.screens.recovery import RecoveryAction, RecoveryScreen
from opc.plugins.cli_board.widgets.activity_pane import ActivityPaneWidget
from opc.plugins.cli_board.widgets.context_tabs import ContextTabsWidget
from opc.plugins.cli_board.widgets.detail_pane import DetailPaneWidget
@@ -36,7 +35,6 @@ if TYPE_CHECKING:
from opc.plugins.cli_board.services.engine_facade import EngineFacade
from opc.plugins.cli_board.services.event_bridge import CliBoardEventBridge
from opc.plugins.cli_board.services.reconcile import ReconcileLoop
from opc.plugins.cli_board.services.recovery import CliRecoveryManager
class CliBoardApp(App[None]):
@@ -74,7 +72,6 @@ class CliBoardApp(App[None]):
Binding("x", "cancel_task", "Cancel"),
Binding("t", "retry_selected", "Retry"),
Binding("e", "checkpoint_feedback", "Feedback"),
Binding("w", "recovery_scan", "Recovery"),
Binding("R", "rename_session", "Rename", show=False),
Binding("D", "delete_session", "Delete", show=False),
Binding("E", "switch_mode", "Mode", show=False),
@@ -115,7 +112,6 @@ class CliBoardApp(App[None]):
self.repository: BoardRepository | None = None
self.actions: BoardActions | None = None
self.event_bridge: CliBoardEventBridge | None = None
self.recovery_manager: CliRecoveryManager | None = None
self.reconcile_loop: ReconcileLoop | None = None
self.exec_mode = "task"
self.company_profile = "corporate"
@@ -143,13 +139,10 @@ class CliBoardApp(App[None]):
from opc.plugins.cli_board.services.engine_facade import EngineFacade
from opc.plugins.cli_board.services.event_bridge import CliBoardEventBridge
from opc.plugins.cli_board.services.recovery import CliRecoveryManager
self.facade = EngineFacade(project_id=self.project_id)
self.repository = BoardRepository(self.facade, project_id=self.project_id)
self.actions = BoardActions(self.facade, project_id=self.project_id)
self.event_bridge = CliBoardEventBridge(self._handle_board_event)
self.recovery_manager = CliRecoveryManager(self.facade)
def compose(self) -> ComposeResult:
yield Header(show_clock=True)
@@ -650,36 +643,6 @@ class CliBoardApp(App[None]):
success_message=f"{label} checkpoint for {task.title}{suffix}.",
)
def action_recovery_scan(self) -> None:
if self._readonly_guard():
return
self._action_recovery_scan()
@work(group="modal", exclusive=True)
async def _action_recovery_scan(self) -> None:
if self.recovery_manager is None:
self.status_widget.set_message("Recovery unavailable.")
return
status = await self.recovery_manager.get_status()
result = await self.push_screen_wait(RecoveryScreen(status))
if result is None:
return
if result.action == "resume":
self.status_widget.set_message(f"Resuming {result.parent_task_id}...")
outcome = await self.recovery_manager.resume(result.parent_task_id)
if outcome.get("ok"):
ids = outcome.get("resumed_work_item_projection_ids", [])
self.status_widget.set_message(f"Resumed {len(ids)} work item(s).")
else:
self.status_widget.set_message(f"Resume failed: {outcome.get('error', '?')}.")
elif result.action == "cancel":
outcome = await self.recovery_manager.cancel(result.parent_task_id)
if outcome.get("ok"):
self.status_widget.set_message(f"Cancelled {outcome.get('cancelled_count', 0)} task(s).")
else:
self.status_widget.set_message(f"Cancel failed: {outcome.get('error', '?')}.")
await self._refresh_snapshot(reason="recovery", silent=True)
def action_rename_session(self) -> None:
if self._readonly_guard():
return
@@ -1354,7 +1317,6 @@ class CliBoardApp(App[None]):
PaletteCommand("view_focus", "Switch to Focus", "Zoom into the selected task.", "3"),
PaletteCommand("view_pipeline", "Switch to Projection", "Show the read-only work-item projection for the selected company run.", "4"),
PaletteCommand("view_org", "Switch to Organisation", "Show read-only org structure.", "5"),
PaletteCommand("recovery_scan", "Runtime Recovery", "Scan and resume interrupted company runtimes.", "w"),
PaletteCommand("rename_session", "Rename Session", "Change the title of the selected task.", "R"),
PaletteCommand("delete_session", "Delete Session", "Cancel and remove the selected task.", "D"),
PaletteCommand("purge_cancelled", "Purge Cancelled Tasks", "Permanently delete all cancelled/failed tasks.", ""),
@@ -1382,7 +1344,6 @@ class CliBoardApp(App[None]):
"project_delete",
"session_config",
"org_add_role",
"recovery_scan",
"rename_session",
"delete_session",
"purge_cancelled",
opc/plugins/cli_board/tui/screens/help.py
+1 -1
View File
@@ -49,7 +49,7 @@ class HelpScreen(ModalScreen[None]):
" s: reply in session m: move between columns\n"
" a / d: approve / deny checkpoint\n"
" e: checkpoint feedback (approve/deny with message)\n"
" c: done x: cancel t: retry w: runtime recovery\n"
" c: done x: cancel t: retry\n"
"\n"
"Session Management\n"
" R: rename session D: delete session\n"
opc/plugins/cli_board/tui/screens/recovery.py
-126
View File
@@ -1,126 +0,0 @@
"""Recovery modal screen for the CLI board."""
from __future__ import annotations
from dataclasses import dataclass
from textual.app import ComposeResult
from textual.containers import Horizontal, Vertical, VerticalScroll
from textual.screen import ModalScreen
from textual.widgets import Button, Label, Static
from opc.plugins.cli_board.services.recovery import RecoveryStatus
@dataclass
class RecoveryAction:
action: str # "resume" | "cancel" | "dismiss"
parent_task_id: str = ""
class RecoveryScreen(ModalScreen[RecoveryAction | None]):
"""Show interrupted company runtimes with resume/cancel options."""
DEFAULT_CSS = """
RecoveryScreen {
align: center middle;
}
.recovery-dialog {
width: 88;
max-width: 90%;
height: auto;
max-height: 80%;
border: solid $primary;
background: $surface;
padding: 1 2;
}
.recovery-title {
text-style: bold;
margin-bottom: 1;
}
.recovery-runtime {
margin-bottom: 1;
padding: 1;
border: round $secondary;
}
.recovery-actions {
align-horizontal: right;
height: auto;
margin-top: 1;
}
.recovery-empty {
color: $text-muted;
margin: 1;
}
"""
BINDINGS = [("escape", "dismiss_screen", "Close")]
def __init__(self, status: RecoveryStatus) -> None:
super().__init__()
self.status = status
def compose(self) -> ComposeResult:
with Vertical(classes="recovery-dialog"):
yield Static("Interrupted Runtimes", classes="recovery-title")
if not self.status.interrupted:
yield Static("No interrupted runtimes found.", classes="recovery-empty")
else:
with VerticalScroll():
for wf in self.status.interrupted:
with Vertical(classes="recovery-runtime"):
# Runtime header
active = wf.parent_task_id in set(self.status.active_recoveries)
status_label = " (recovering...)" if active else ""
yield Label(f"{wf.title}{status_label}")
yield Static(
f" Profile: {wf.profile or 'unknown'} "
f"Interrupted: {wf.interrupted_at[:19] if wf.interrupted_at else '?'}"
)
# Work-item summary
done = sum(1 for s in wf.work_items if s.status == "done")
total = len(wf.work_items)
failed = sum(1 for s in wf.work_items if s.interrupted)
yield Static(
f" Work items: {done}/{total} done, {failed} interrupted"
)
if not active:
with Horizontal():
yield Button(
"Resume",
id=f"resume-{wf.parent_task_id}",
variant="primary",
)
yield Button(
"Cancel",
id=f"cancel-{wf.parent_task_id}",
variant="error",
)
with Horizontal(classes="recovery-actions"):
yield Button("Close", id="close-recovery")
def on_button_pressed(self, event: Button.Pressed) -> None:
btn_id = event.button.id or ""
if btn_id == "close-recovery":
self.dismiss(None)
return
if btn_id.startswith("resume-"):
task_id = btn_id[len("resume-"):]
self.dismiss(RecoveryAction(action="resume", parent_task_id=task_id))
return
if btn_id.startswith("cancel-"):
task_id = btn_id[len("cancel-"):]
self.dismiss(RecoveryAction(action="cancel", parent_task_id=task_id))
return
def action_dismiss_screen(self) -> None:
self.dismiss(None)
opc/plugins/office_ui/frontend_dist/assets/index-02tfsorH.js
-155
View File
File diff suppressed because one or more lines are too long
opc/plugins/office_ui/frontend_dist/assets/index-Cson66Y4.js
+155
View File
File diff suppressed because one or more lines are too long
opc/plugins/office_ui/frontend_dist/assets/index-BCWwLlJm.css → opc/plugins/office_ui/frontend_dist/assets/index-DEvLDWDw.css
+1 -1
View File
File diff suppressed because one or more lines are too long
opc/plugins/office_ui/frontend_dist/index.html
+2 -2
View File
@@ -5,9 +5,9 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="icon" href="data:," />
<title>OpenOPC Pixel Office</title>
<script type="module" crossorigin src="./assets/index-02tfsorH.js"></script>
<script type="module" crossorigin src="./assets/index-Cson66Y4.js"></script>
<link rel="modulepreload" crossorigin href="./assets/phaser-DFK5Ua9d.js">
<link rel="stylesheet" crossorigin href="./assets/index-BCWwLlJm.css">
<link rel="stylesheet" crossorigin href="./assets/index-DEvLDWDw.css">
</head>
<body>
<div id="root"></div>
opc/plugins/office_ui/frontend_src/App.tsx
+7 -23
View File
@@ -476,7 +476,6 @@ export default function App() {
const [globalCompanyProfile, setGlobalCompanyProfile] = useState<'corporate' | 'custom'>('corporate')
const [globalTaskPreferredAgent, setGlobalTaskPreferredAgent] = useState<TaskPreferredAgent>('native')
const [orgInfoData, setOrgInfoData] = useState<OrgInfoPayload | null>(null)
const [recoveryStatus, setRecoveryStatus] = useState<any>(null)
const [commsState, setCommsState] = useState<import('./lib/wsClient').CommsStatePayload | null>(null)
const [commsMessage, setCommsMessage] = useState<import('./lib/wsClient').CommsMessagePayload | null>(null)
const [talentTemplates, setTalentTemplates] = useState<TalentTemplate[]>([])
@@ -1239,7 +1238,6 @@ export default function App() {
runtimeControlState: String(payload.runtime_control_state ?? payload.runtimeControlState ?? 'idle') as any,
canStop: Boolean(payload.can_stop ?? payload.canStop),
canResume: Boolean(payload.can_resume ?? payload.canResume),
resumeParentTaskId: String(payload.resume_parent_task_id ?? payload.resumeParentTaskId ?? ''),
resumeParentSessionId: String(payload.resume_parent_session_id ?? payload.resumeParentSessionId ?? ''),
pendingRuntimeCheckpointId: String(payload.pending_runtime_checkpoint_id ?? payload.pendingRuntimeCheckpointId ?? ''),
stopIntentId: String(payload.stop_intent_id ?? payload.stopIntentId ?? ''),
@@ -1749,10 +1747,6 @@ export default function App() {
clientRef.current?.collabSync(getActiveProjectId(), undefined, projectViewGenerationRef.current)
}
},
onRecoveryStatus: (payload) => {
if (!payloadMatchesActiveProject(payload as unknown as Record<string, unknown>, false)) return
setRecoveryStatus(payload)
},
onCommsState: (payload) => {
if (!payloadMatchesActiveProject(payload as unknown as Record<string, unknown>, false)) return
setCommsState(payload)
@@ -1761,13 +1755,6 @@ export default function App() {
if (!payloadMatchesActiveProject(payload as unknown as Record<string, unknown>, true)) return
setCommsMessage(payload)
},
onRecoveryResult: (payload) => {
if (!payloadMatchesActiveProject(payload as unknown as Record<string, unknown>, false)) return
if (payload?.status === 'completed' || payload?.status === 'cancelled') {
// Trigger a re-scan
clientRef.current?.recoveryAction(getActiveProjectId(), 'scan')
}
},
onTalentList: (payload) => {
setTalentTemplates(payload.templates ?? [])
if (payload.talent_dir) setDefaultTalentDir(payload.talent_dir)
@@ -2242,18 +2229,13 @@ export default function App() {
) => {
const session = sessionStore.sessions.find(s => s.taskId === taskId)
const parentSessionId = session?.resumeParentSessionId ?? session?.parentSessionId ?? session?.sessionId
const parentTaskId = session?.resumeParentTaskId
?? (parentSessionId ? sessionStore.sessions.find(s => s.sessionId === parentSessionId && !s.parentSessionId)?.taskId : undefined)
?? taskId
for (const candidate of sessionStore.sessions) {
if (
candidate.taskId === taskId
|| candidate.taskId === parentTaskId
|| (!!parentSessionId && (candidate.parentSessionId === parentSessionId || candidate.sessionId === parentSessionId))
) {
sessionStore.updateSession(candidate.taskId, {
...patch,
resumeParentTaskId: parentTaskId,
resumeParentSessionId: parentSessionId,
})
}
@@ -2278,7 +2260,7 @@ export default function App() {
clientRef.current?.sessionStop(getActiveProjectId(), taskId)
}, [sessionStore.sessions, markRuntimeControlForTask, getActiveProjectId])
const handleSessionResume = useCallback((taskId: string) => {
const handleSessionResume = useCallback((taskId: string, runtimeSessionId?: string, checkpointId?: string) => {
const session = sessionStore.sessions.find(s => s.taskId === taskId)
const isCompanyRuntime = session?.execMode === 'company'
|| session?.execMode === 'org'
@@ -2293,7 +2275,12 @@ export default function App() {
canResume: false,
})
}
clientRef.current?.sessionResume(getActiveProjectId(), taskId)
clientRef.current?.sessionResume(
getActiveProjectId(),
taskId,
runtimeSessionId ?? session?.resumeParentSessionId ?? session?.parentSessionId ?? session?.sessionId,
checkpointId ?? session?.pendingRuntimeCheckpointId,
)
}, [sessionStore.sessions, markRuntimeControlForTask, getActiveProjectId])
const handleGlobalModeChange = useCallback((mode: 'task' | 'company' | 'org' | 'custom', profile?: string, orgId?: string) => {
@@ -2426,9 +2413,6 @@ export default function App() {
activeSavedOrg={activeSavedOrg}
onSavedOrgsList={handleSavedOrgsList}
onSavedOrgLoad={handleSavedOrgLoad}
recoveryStatus={recoveryStatus}
onRecoveryResume={(id) => clientRef.current?.recoveryAction(getActiveProjectId(), 'resume', id)}
onRecoveryCancel={(id) => clientRef.current?.recoveryAction(getActiveProjectId(), 'cancel', id)}
commsState={commsState}
commsMessage={commsMessage}
onCommsRefresh={(opts) => {
opc/plugins/office_ui/frontend_src/lib/collabSync.ts
-1
View File
@@ -676,7 +676,6 @@ export function mapBackendSession(raw: any): Session {
runtimeControlState: raw.runtime_control_state ?? raw.runtimeControlState,
canStop: raw.can_stop ?? raw.canStop,
canResume: raw.can_resume ?? raw.canResume,
resumeParentTaskId: raw.resume_parent_task_id ?? raw.resumeParentTaskId,
resumeParentSessionId: raw.resume_parent_session_id ?? raw.resumeParentSessionId,
pendingRuntimeCheckpointId: raw.pending_runtime_checkpoint_id ?? raw.pendingRuntimeCheckpointId,
stopIntentId: raw.stop_intent_id ?? raw.stopIntentId,
opc/plugins/office_ui/frontend_src/lib/workItemSessions.ts
-2
View File
@@ -387,7 +387,6 @@ export function getConversationSessionView(
runtimeControlState: runtimeSource.runtimeControlState ?? normalizedActiveSession.runtimeControlState,
canStop: runtimeSource.canStop ?? normalizedActiveSession.canStop,
canResume: runtimeSource.canResume ?? normalizedActiveSession.canResume,
resumeParentTaskId: runtimeSource.resumeParentTaskId ?? normalizedActiveSession.resumeParentTaskId,
resumeParentSessionId: runtimeSource.resumeParentSessionId ?? normalizedActiveSession.resumeParentSessionId,
pendingRuntimeCheckpointId: runtimeSource.pendingRuntimeCheckpointId ?? normalizedActiveSession.pendingRuntimeCheckpointId,
stopIntentId: runtimeSource.stopIntentId ?? normalizedActiveSession.stopIntentId,
@@ -447,7 +446,6 @@ export function getConversationHeaderSession(
runtimeControlState: runtimeSource.runtimeControlState ?? normalizedActiveSession.runtimeControlState,
canStop: runtimeSource.canStop ?? normalizedActiveSession.canStop,
canResume: runtimeSource.canResume ?? normalizedActiveSession.canResume,
resumeParentTaskId: runtimeSource.resumeParentTaskId ?? normalizedActiveSession.resumeParentTaskId,
resumeParentSessionId: runtimeSource.resumeParentSessionId ?? normalizedActiveSession.resumeParentSessionId,
pendingRuntimeCheckpointId: runtimeSource.pendingRuntimeCheckpointId ?? normalizedActiveSession.pendingRuntimeCheckpointId,
stopIntentId: runtimeSource.stopIntentId ?? normalizedActiveSession.stopIntentId,
opc/plugins/office_ui/frontend_src/lib/wsClient.test.ts
+11
View File
@@ -30,6 +30,17 @@ const flushPromises = async () => {
const client = new VisualSocketClient('ws://unit.test', {})
// Company Continue keeps the selected UI channel task separate from the
// durable runtime identity used by the checkpoint handoff.
client.sessionResume('project-a', 'ui-task', 'runtime-session', 'checkpoint-1')
const resumeEnvelope = JSON.parse(
(client as unknown as TestSocketClient).pendingQueue.pop() ?? '{}',
) as Record<string, unknown>
assert.equal(resumeEnvelope.type, 'session_resume')
assert.equal(resumeEnvelope.task_id, 'ui-task')
assert.equal(resumeEnvelope.runtime_session_id, 'runtime-session')
assert.equal(resumeEnvelope.checkpoint_id, 'checkpoint-1')
// A summary and a full request for the same task are distinct correlations.
// Neither Promise may settle merely because the request was queued locally.
const summaryPromise = client.sessionDetail('project-a', 'task-1', { detailLevel: 'summary' })
opc/plugins/office_ui/frontend_src/lib/wsClient.ts
+15 -16
View File
@@ -41,8 +41,6 @@ interface SocketHandlers {
onProjectSwitched?: (payload: { project_id: string; switch_seq?: string }) => void
onProjectDeleted?: (payload: { project_id: string }) => void
onOrgInfo?: (payload: OrgInfoPayload) => void
onRecoveryStatus?: (payload: any) => void
onRecoveryResult?: (payload: any) => void
onTalentList?: (payload: TalentListPayload) => void
onTalentScanLocal?: (payload: { templates: Array<{ template_id: string; name: string; description: string; category: string; domains: string[]; tags: string[] }> }) => void
onEmployeeDetail?: (payload: EmployeeDetailPayload) => void
@@ -169,7 +167,6 @@ const PROJECT_SCOPED_MESSAGE_TYPES = new Set([
'session_update_title',
'secretary_send',
'project_index',
'recovery_action',
'comms_state',
'comms_read_message',
])
@@ -420,9 +417,22 @@ export class VisualSocketClient {
this.send({ type: 'session_stop', project_id: pid, task_id: taskId })
}
sessionResume(projectId: string, taskId: string, content?: string): void {
sessionResume(
projectId: string,
taskId: string,
runtimeSessionId?: string,
checkpointId?: string,
content?: string,
): void {
const pid = this.requireProjectId(projectId, 'session_resume')
this.send({ type: 'session_resume', project_id: pid, task_id: taskId, content })
this.send({
type: 'session_resume',
project_id: pid,
task_id: taskId,
runtime_session_id: runtimeSessionId,
checkpoint_id: checkpointId,
content,
})
}
sessionComplete(projectId: string, taskId: string): void {
@@ -646,11 +656,6 @@ export class VisualSocketClient {
this.send({ type: 'org_saved_delete', name })
}
recoveryAction(projectId: string, action: 'resume' | 'cancel' | 'scan', parentTaskId?: string): void {
const pid = this.requireProjectId(projectId, 'recovery_action')
this.send({ type: 'recovery_action', project_id: pid, action, parent_task_id: parentTaskId })
}
commsState(projectId: string, opts?: { task_id?: string; session_id?: string }): void {
const pid = this.requireProjectId(projectId, 'comms_state')
this.send({ type: 'comms_state', project_id: pid, ...(opts || {}) })
@@ -801,12 +806,6 @@ export class VisualSocketClient {
if (projectId) this.commsState(projectId)
} catch { /* ignore */ }
break
case 'recovery_status':
this.handlers.onRecoveryStatus?.(parsed.payload)
break
case 'recovery_result':
this.handlers.onRecoveryResult?.(parsed.payload)
break
case 'talent_list':
this.handlers.onTalentList?.(parsed.payload)
break
opc/plugins/office_ui/frontend_src/stores/SessionStore.ts
-2
View File
@@ -244,7 +244,6 @@ function sessionReducer(state: Session[], action: SessionAction): Session[] {
runtimeControlState: guardedRuntimeControl.runtimeControlState ?? existing.runtimeControlState,
canStop: guardedRuntimeControl.canStop ?? existing.canStop,
canResume: guardedRuntimeControl.canResume ?? existing.canResume,
resumeParentTaskId: incoming.resumeParentTaskId ?? existing.resumeParentTaskId,
resumeParentSessionId: incoming.resumeParentSessionId ?? existing.resumeParentSessionId,
pendingRuntimeCheckpointId: guardedRuntimeControl.pendingRuntimeCheckpointId ?? existing.pendingRuntimeCheckpointId,
stopIntentId: guardedRuntimeControl.stopIntentId ?? existing.stopIntentId,
@@ -337,7 +336,6 @@ function sessionReducer(state: Session[], action: SessionAction): Session[] {
runtimeControlState: guarded.runtimeControlState ?? s.runtimeControlState,
canStop: guarded.canStop ?? s.canStop,
canResume: guarded.canResume ?? s.canResume,
resumeParentTaskId: guarded.resumeParentTaskId ?? s.resumeParentTaskId,
resumeParentSessionId: guarded.resumeParentSessionId ?? s.resumeParentSessionId,
pendingRuntimeCheckpointId: guarded.pendingRuntimeCheckpointId ?? s.pendingRuntimeCheckpointId,
stopIntentId: guarded.stopIntentId ?? s.stopIntentId,
opc/plugins/office_ui/frontend_src/types/kanban.ts
+3 -4
View File
@@ -316,10 +316,9 @@ export interface Session {
originChannel?: string
originTaskId?: string
runtimeControlState?: 'running' | 'suspending' | 'suspended' | 'resuming' | 'idle'
canStop?: boolean
canResume?: boolean
resumeParentTaskId?: string
resumeParentSessionId?: string
canStop?: boolean
canResume?: boolean
resumeParentSessionId?: string
pendingRuntimeCheckpointId?: string
stopIntentId?: string
// Handoff context from upstream work item (Company Mode)
opc/plugins/office_ui/frontend_src/types/visual.ts
-2
View File
@@ -100,8 +100,6 @@ export type SocketEnvelope =
| { type: 'work_item_batch_updated'; payload: { run_id?: string; work_items: RuntimeWorkItemInfo[]; frontier?: RuntimeFrontierSummary } }
| { type: 'project_recovery_updated'; payload: Record<string, unknown> }
| { type: 'project_revision_created'; payload: { run_id?: string; revision_links: SessionLinkInfo[] } }
| { type: 'recovery_status'; payload: Record<string, unknown> }
| { type: 'recovery_result'; payload: Record<string, unknown> }
| { type: 'talent_list'; payload: TalentListPayload }
| { type: 'talent_scan_local'; payload: { templates: Array<{ template_id: string; name: string; description: string; category: string; domains: string[]; tags: string[] }> } }
| { type: 'employee_detail'; payload: EmployeeDetailPayload }
opc/plugins/office_ui/frontend_src/workspace/ContextPanel.tsx
-3
View File
@@ -77,7 +77,6 @@ interface ContextPanelProps {
onCommsRefresh?: () => void
onCommsReadMessage?: (path: string) => void
orgInfoData?: OrgInfoPayload | null
recoveryStatus?: Record<string, unknown> | null
canShowTeamTab?: boolean
onTeamStopRun?: () => void
@@ -497,7 +496,6 @@ export function ContextPanel({
onCommsRefresh,
onCommsReadMessage,
orgInfoData,
recoveryStatus,
canShowTeamTab = false,
onTeamStopRun,
onTitleChange,
@@ -1306,7 +1304,6 @@ export function ContextPanel({
<div style={{ flex: 1, overflow: 'auto', minHeight: 0 }}>
<ProjectCockpit
orgInfoData={orgInfoData ?? null}
recoveryStatus={recoveryStatus ?? null}
commsState={commsState ?? null}
onStopRun={onTeamStopRun}
embedded
opc/plugins/office_ui/frontend_src/workspace/ProjectCockpit.tsx
+1 -4
View File
@@ -48,7 +48,6 @@ interface TeamCardInfo {
interface ProjectCockpitProps {
orgInfoData?: OrgInfoPayload | null
recoveryStatus?: Record<string, unknown> | null
commsState?: CommsStatePayload | null
onStopRun?: () => void
embedded?: boolean
@@ -56,7 +55,6 @@ interface ProjectCockpitProps {
export function ProjectCockpit({
orgInfoData,
recoveryStatus,
commsState,
onStopRun,
embedded = false,
@@ -78,7 +76,6 @@ export function ProjectCockpit({
count + asRecordList(asRecord(digest.manager_digest).notification_backlog).length
), 0)
const unreadCount = actionableCount + protocolCount + notificationCount
const interrupted = Array.isArray(recoveryStatus?.interrupted) ? recoveryStatus.interrupted.length : 0
const communicationItems = [
{ label: 'Actionable', value: actionableCount },
@@ -198,7 +195,7 @@ export function ProjectCockpit({
<span>Seats {runtimeView.runtimeSeats.length}</span>
<span>Approvals {pendingDecisionCount}</span>
<span>Unread {unreadCount}</span>
<span>Recovery {interrupted > 0 ? `${interrupted} interrupted` : summarizeText(asRecord(projectRun?.recovery_pointer).status, 'clean')}</span>
<span>Run state {summarizeText(asRecord(projectRun?.recovery_pointer).status, 'clean')}</span>
</div>
</div>
opc/plugins/office_ui/frontend_src/workspace/WorkItemRecoveryPanel.tsx
-116
View File
@@ -1,116 +0,0 @@
import { useState } from 'react'
export interface RecoverableWorkItem {
work_item_projection_id: string
title: string
task_id: string
status: string
interrupted: boolean
previous_status: string
}
export interface InterruptedWorkItemRuntime {
parent_session_id: string
parent_task_id: string
project_id: string
title: string
profile: string
interrupted_at: string
work_items: RecoverableWorkItem[]
}
export interface RecoveryStatusPayload {
interrupted: InterruptedWorkItemRuntime[]
active_recoveries: string[]
scanned_at: number
}
interface WorkItemRecoveryPanelProps {
data: RecoveryStatusPayload
onResume: (parentTaskId: string) => void
onCancel: (parentTaskId: string) => void
}
const STATUS_ICON: Record<string, string> = {
done: '\u2713',
failed: '\u2717',
pending: '\u25CB',
blocked: '\u25A0',
cancelled: '\u2014',
running: '\u25B6',
}
const STATUS_COLOR: Record<string, string> = {
done: 'var(--green, #27ae60)',
failed: 'var(--red, #e74c3c)',
pending: 'var(--text-secondary, #888)',
blocked: 'var(--yellow, #f39c12)',
cancelled: 'var(--text-dim, #555)',
running: 'var(--accent, #3498db)',
}
export function WorkItemRecoveryPanel({ data, onResume, onCancel }: WorkItemRecoveryPanelProps) {
const [dismissed, setDismissed] = useState<Set<string>>(new Set())
if (!data.interrupted.length && !data.active_recoveries.length) return null
const visible = data.interrupted.filter(w => !dismissed.has(w.parent_task_id))
if (!visible.length && !data.active_recoveries.length) return null
return (
<div className="wfr-panel">
{visible.map(wf => {
const isRecovering = data.active_recoveries.includes(wf.parent_task_id)
const doneCount = wf.work_items.filter(item => item.status === 'done').length
const failedCount = wf.work_items.filter(item => item.interrupted || item.status === 'failed').length
return (
<div key={wf.parent_task_id} className="wfr-card">
<div className="wfr-header">
<span className="wfr-icon">&#x26A0;</span>
<div className="wfr-header-text">
<span className="wfr-title">Interrupted: {wf.title}</span>
<span className="wfr-subtitle">
{doneCount}/{wf.work_items.length} work items done, {failedCount} interrupted
{wf.profile && <> &middot; {wf.profile}</>}
</span>
</div>
</div>
<div className="wfr-work-items">
{wf.work_items.map(item => (
<div key={item.work_item_projection_id} className={`wfr-work-item wfr-work-item--${item.status}`}>
<span className="wfr-work-item-icon" style={{ color: STATUS_COLOR[item.status] || STATUS_COLOR.pending }}>
{STATUS_ICON[item.status] || STATUS_ICON.pending}
</span>
<span className="wfr-work-item-title">{item.title}</span>
{item.interrupted && <span className="wfr-work-item-badge">interrupted</span>}
</div>
))}
</div>
<div className="wfr-actions">
{isRecovering ? (
<span className="wfr-recovering">
<span className="spinner-inline" /> Recovering...
</span>
) : (
<>
<button className="wfr-btn wfr-btn--resume" onClick={() => onResume(wf.parent_task_id)}>
Resume
</button>
<button className="wfr-btn wfr-btn--cancel" onClick={() => onCancel(wf.parent_task_id)}>
Cancel
</button>
<button className="wfr-btn wfr-btn--dismiss" onClick={() => setDismissed(prev => new Set(prev).add(wf.parent_task_id))}>
Dismiss
</button>
</>
)}
</div>
</div>
)
})}
</div>
)
}
opc/plugins/office_ui/frontend_src/workspace/WorkspacePage.tsx
+17 -21
View File
@@ -1,6 +1,5 @@
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react'
import type { AgentInfo, OrgInfoPayload, SavedOrgSummary } from '../types/visual'
import { WorkItemRecoveryPanel } from './WorkItemRecoveryPanel'
import type { ChatMessage, CheckpointReplyMetadata, OutgoingAttachmentPayload } from '../types/chat'
import type { KanbanTask, Session, TaskPreferredAgent } from '../types/kanban'
import type { BoardStoreState } from '../kanban/BoardStore'
@@ -253,7 +252,7 @@ interface WorkspacePageProps {
*/
onContinueInNewChat?: (mode: 'task' | 'company' | 'org' | 'custom', companyProfile?: 'corporate' | 'custom', orgId?: string) => void
onSessionStop?: (taskId: string) => void
onSessionResume?: (taskId: string) => void
onSessionResume?: (taskId: string, runtimeSessionId?: string, checkpointId?: string) => void
onSessionComplete?: (taskId: string) => void
onLoadSessionDetail?: (
taskId: string,
@@ -263,9 +262,6 @@ interface WorkspacePageProps {
onCollabSync?: () => void
orgInfoData?: OrgInfoPayload | null
onNavigateToOrg?: () => void
recoveryStatus?: any
onRecoveryResume?: (parentTaskId: string) => void
onRecoveryCancel?: (parentTaskId: string) => void
commsState?: import('../lib/wsClient').CommsStatePayload | null
commsMessage?: import('../lib/wsClient').CommsMessagePayload | null
onCommsRefresh?: (opts?: { task_id?: string; session_id?: string; project_id?: string }) => void
@@ -305,9 +301,6 @@ export function WorkspacePage({
onCollabSync,
orgInfoData,
onNavigateToOrg,
recoveryStatus,
onRecoveryResume,
onRecoveryCancel,
commsState,
commsMessage,
onCommsRefresh,
@@ -988,13 +981,21 @@ export function WorkspacePage({
const handleResume = useCallback(() => {
const targetSession = activeConversation.runtimeSession ?? activeConversation.displaySession ?? activeSession
const targetTaskId = targetSession?.resumeParentTaskId ?? targetSession?.taskId ?? activeSessionId
if (targetTaskId) onSessionResume?.(targetTaskId)
const uiTaskId = activeSessionId ?? targetSession?.taskId
const runtimeSessionId = targetSession?.resumeParentSessionId
?? targetSession?.parentSessionId
?? targetSession?.sessionId
if (uiTaskId) {
onSessionResume?.(uiTaskId, runtimeSessionId, targetSession?.pendingRuntimeCheckpointId)
}
}, [activeConversation.runtimeSession, activeConversation.displaySession, activeSession, activeSessionId, onSessionResume])
const handleResumeTask = useCallback((taskId: string) => {
const session = sessions.find(s => s.taskId === taskId)
onSessionResume?.(session?.resumeParentTaskId ?? taskId)
const runtimeSessionId = session?.resumeParentSessionId
?? session?.parentSessionId
?? session?.sessionId
onSessionResume?.(taskId, runtimeSessionId, session?.pendingRuntimeCheckpointId)
}, [sessions, onSessionResume])
const handleCompleteTask = useCallback((taskId: string) => {
@@ -1034,8 +1035,11 @@ export function WorkspacePage({
}
const targetTaskId = activeSessionId
if (!targetTaskId) return
const checkpointReplyId = String(latestPendingCheckpointReply?.response_to_checkpoint_id ?? '').trim()
const runtimeSession = activeConversation.runtimeSession ?? activeConversation.displaySession ?? activeSession
const runtimeCheckpointId = String(runtimeSession?.pendingRuntimeCheckpointId ?? '').trim()
let outgoingMetadata = latestPendingCheckpointReply
?? (runtimeCheckpointId ? { response_to_checkpoint_id: runtimeCheckpointId } : undefined)
const checkpointReplyId = String(outgoingMetadata?.response_to_checkpoint_id ?? '').trim()
if (!checkpointReplyId) {
const uiMessageId = makeOptimisticUserMessageId()
outgoingMetadata = { ...(latestPendingCheckpointReply ?? {}), ui_message_id: uiMessageId }
@@ -1050,7 +1054,7 @@ export function WorkspacePage({
}
dispatchSessionSend(targetTaskId, content, attachments, outgoingMetadata)
},
[effectiveView.kind, activeSessionId, activeConversation.displaySession, activeSession, latestPendingCheckpointReply, chatStore, dispatchSessionSend, onSecretarySend],
[effectiveView.kind, activeSessionId, activeConversation.runtimeSession, activeConversation.displaySession, activeSession, latestPendingCheckpointReply, chatStore, dispatchSessionSend, onSecretarySend],
)
// ── MessageList send (checkpoint replies) ──
@@ -1114,13 +1118,6 @@ export function WorkspacePage({
{/* Middle column: Kanban Board (hidden when panel maximized) */}
{panelState !== 'maximized' && (
<div className="workspace-board">
{recoveryStatus && onRecoveryResume && onRecoveryCancel && (
<WorkItemRecoveryPanel
data={recoveryStatus}
onResume={onRecoveryResume}
onCancel={onRecoveryCancel}
/>
)}
{agents.length > 0 && <AgentStatusBar agents={agents} tasks={boardStore.tasks} />}
{isCompanyMode ? (
boardStore.activeBoard && activeSession && (
@@ -1206,7 +1203,6 @@ export function WorkspacePage({
onCommsRefresh={onCommsRefresh ? () => onCommsRefresh({ session_id: activeSession?.sessionId || undefined, project_id: projectId || undefined }) : undefined}
onCommsReadMessage={onCommsReadMessage}
orgInfoData={orgInfoData ?? null}
recoveryStatus={recoveryStatus ?? null}
canShowTeamTab={canShowTeamTab}
onTeamStopRun={activeSessionId ? () => onSessionStop?.(activeSessionId) : undefined}
onTitleChange={onTitleChange}
opc/plugins/office_ui/frontend_src/workspace/workspace.css
-126
View File
@@ -1605,132 +1605,6 @@
}
}
/* ── Work Item Recovery Panel ────────────────────────────────────────── */
.wfr-panel {
padding: 0 8px;
}
.wfr-card {
background: color-mix(in srgb, var(--yellow, #f39c12) 8%, var(--bg-secondary));
border: 1px solid color-mix(in srgb, var(--yellow, #f39c12) 25%, transparent);
border-radius: 8px;
padding: 12px 14px;
margin-bottom: 8px;
}
.wfr-header {
display: flex;
align-items: flex-start;
gap: 8px;
margin-bottom: 8px;
}
.wfr-icon {
font-size: 16px;
line-height: 1;
flex-shrink: 0;
}
.wfr-header-text {
display: flex;
flex-direction: column;
gap: 2px;
min-width: 0;
}
.wfr-title {
font-size: 13px;
font-weight: 600;
color: var(--text);
}
.wfr-subtitle {
font-size: 11px;
color: var(--text-secondary);
}
.wfr-work-items {
display: flex;
flex-direction: column;
gap: 4px;
margin-bottom: 10px;
}
.wfr-work-item {
display: flex;
align-items: center;
gap: 6px;
font-size: 12px;
color: var(--text-secondary);
}
.wfr-work-item-icon {
font-size: 12px;
width: 14px;
text-align: center;
flex-shrink: 0;
font-weight: 700;
}
.wfr-work-item-title {
flex: 1;
min-width: 0;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.wfr-work-item-badge {
font-size: 9px;
padding: 1px 5px;
border-radius: 3px;
background: color-mix(in srgb, var(--red, #e74c3c) 15%, transparent);
color: var(--red, #e74c3c);
flex-shrink: 0;
}
.wfr-actions {
display: flex;
gap: 8px;
align-items: center;
}
.wfr-btn {
padding: 5px 14px;
font-size: 12px;
font-weight: 500;
border: none;
border-radius: 6px;
cursor: pointer;
}
.wfr-btn--resume {
background: var(--green, #27ae60);
color: #fff;
}
.wfr-btn--resume:hover { opacity: 0.85; }
.wfr-btn--cancel {
background: var(--red, #e74c3c);
color: #fff;
}
.wfr-btn--cancel:hover { opacity: 0.85; }
.wfr-btn--dismiss {
background: transparent;
color: var(--text-secondary);
}
.wfr-btn--dismiss:hover { color: var(--text); }
.wfr-recovering {
display: flex;
align-items: center;
gap: 6px;
font-size: 12px;
color: var(--accent);
}
/* ═══════════════════════════════════════════════════════
Kanban empty-state
═══════════════════════════════════════════════════════ */
opc/plugins/office_ui/recovery_manager.py
-552
View File
@@ -1,552 +0,0 @@
"""Pluggable work-item crash-recovery manager.
Detects interrupted company-mode work items after server restart and provides
deterministic Resume / Cancel operations — no LLM needed.
Hooks into existing engine capabilities without modifying core code:
- engine.store → task queries + persistence
- engine._load_company_runtime_snapshot() → reconstruct work-item plan + tasks
- engine.company_executor.execute() → resume execution
- ws_handler.broadcast() → push status to all clients
"""
from __future__ import annotations
import asyncio
import dataclasses
import logging
import time
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any, Awaitable, Callable
from opc.layer2_organization.work_item_identity import work_item_projection_id_from_metadata
from opc.layer2_organization.work_item_transition import apply_task_status_transition
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Data types
# ---------------------------------------------------------------------------
@dataclass
class RecoverableWorkItem:
work_item_projection_id: str
title: str
task_id: str
status: str # "done" | "failed" | "pending" | "blocked" | "cancelled"
interrupted: bool # has interrupted_recovery metadata
previous_status: str # what it was before reconciliation
@dataclass
class InterruptedWorkItemRun:
parent_session_id: str
parent_task_id: str
project_id: str
title: str
profile: str
interrupted_at: str
work_items: list[RecoverableWorkItem] = field(default_factory=list)
@dataclass
class RecoveryStatus:
interrupted: list[InterruptedWorkItemRun] = field(default_factory=list)
active_recoveries: list[str] = field(default_factory=list)
scanned_at: float = 0.0
# ---------------------------------------------------------------------------
# Manager
# ---------------------------------------------------------------------------
class RuntimeRecoveryManager:
"""Scans for interrupted work-item runs and provides deterministic recovery."""
_CACHE_TTL = 10.0 # seconds
def __init__(
self,
engine: Any,
broadcast_fn: Callable[[dict[str, Any]], Awaitable[None]],
) -> None:
self._engine = engine
self._broadcast = broadcast_fn
self._lock = asyncio.Lock()
self._active_recoveries: dict[str, asyncio.Task[Any]] = {}
self._cached: RecoveryStatus | None = None
self._cache_until: float = 0.0
# ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
async def get_recovery_status(self) -> RecoveryStatus:
"""Return cached scan results, re-scanning if stale."""
now = time.time()
if self._cached is not None and now < self._cache_until:
# Keep active_recoveries up to date even from cache
self._cached.active_recoveries = list(self._active_recoveries.keys())
return self._cached
status = await self.scan()
self._cached = status
self._cache_until = now + self._CACHE_TTL
return status
async def scan(self) -> RecoveryStatus:
"""Scan the database for recoverable interrupted work-item runs."""
store = self._engine.store
if not store:
return RecoveryStatus()
project_id = self._engine.project_id or "default"
try:
all_tasks = await store.get_tasks(project_id=project_id)
except Exception as exc:
logger.warning(f"Recovery scan failed: {exc}")
return RecoveryStatus()
# Group projected work-item tasks by parent_session_id.
groups: dict[str, list[Any]] = {}
all_tasks_by_session: dict[str, Any] = {}
for task in all_tasks:
session_id = str(getattr(task, "session_id", "") or "").strip()
if session_id:
all_tasks_by_session[session_id] = task
parent_sid = str(getattr(task, "parent_session_id", "") or "").strip()
projection_id = work_item_projection_id_from_metadata(getattr(task, "metadata", {}) or {})
if parent_sid and projection_id:
groups.setdefault(parent_sid, []).append(task)
interrupted: list[InterruptedWorkItemRun] = []
for parent_sid, tasks in groups.items():
# Check if any task has interrupted_recovery metadata
has_interrupted = any(
_is_interrupted(t) for t in tasks
)
if not has_interrupted:
continue
# Skip if all tasks are terminal (DONE or CANCELLED)
from opc.core.models import TaskStatus
non_terminal = [
t for t in tasks
if t.status not in (TaskStatus.DONE, TaskStatus.CANCELLED)
]
if not non_terminal:
continue
# Find the parent (primary) task
parent_task = all_tasks_by_session.get(parent_sid)
parent_task_id = parent_task.id if parent_task else parent_sid
title = parent_task.title if parent_task else "Unknown work-item run"
# Build work-item list.
work_items: list[RecoverableWorkItem] = []
earliest_interrupt = ""
for t in sorted(tasks, key=lambda x: (x.created_at, x.id)):
meta = dict(getattr(t, "metadata", {}) or {})
recovery_meta = meta.get("interrupted_recovery", {})
is_int = _is_interrupted(t)
if is_int and recovery_meta.get("detected_at", ""):
detected = recovery_meta["detected_at"]
if not earliest_interrupt or detected < earliest_interrupt:
earliest_interrupt = detected
work_items.append(RecoverableWorkItem(
work_item_projection_id=work_item_projection_id_from_metadata(meta, fallback=t.id),
title=t.title,
task_id=t.id,
status=t.status.value if hasattr(t.status, "value") else str(t.status),
interrupted=is_int,
previous_status=recovery_meta.get("previous_status", ""),
))
profile = ""
for t in tasks:
p = (getattr(t, "metadata", {}) or {}).get("company_profile", "")
if p:
profile = p
break
interrupted.append(InterruptedWorkItemRun(
parent_session_id=parent_sid,
parent_task_id=parent_task_id,
project_id=project_id,
title=title,
profile=profile,
interrupted_at=earliest_interrupt or datetime.now().isoformat(),
work_items=work_items,
))
return RecoveryStatus(
interrupted=interrupted,
active_recoveries=list(self._active_recoveries.keys()),
scanned_at=time.time(),
)
async def resume(self, parent_task_id: str) -> dict[str, Any]:
"""Deterministically resume an interrupted work-item run."""
async with self._lock:
if parent_task_id in self._active_recoveries:
return {"ok": False, "error": "already_in_progress"}
# Find the interrupted work-item run.
status = await self.scan()
wf = next((w for w in status.interrupted if w.parent_task_id == parent_task_id), None)
if not wf:
return {"ok": False, "error": "not_found"}
# Load snapshot
snapshot = await self._engine._load_company_runtime_snapshot(wf.parent_session_id)
if not snapshot:
return {"ok": False, "error": "snapshot_unavailable"}
plan, tasks = snapshot
# Clean orphaned checkpoints
await self._clean_orphaned_checkpoints(wf, tasks)
# Reset interrupted/failed/blocked tasks → PENDING
from opc.core.models import TaskStatus
resumed_ids: list[str] = []
failed_ids: list[str] = []
for task in tasks:
if task.status == TaskStatus.DONE:
continue
if task.status in (TaskStatus.FAILED, TaskStatus.BLOCKED):
task.result = None
task.execution_lock = False
task.execution_locked_at = None
meta = dict(task.metadata)
meta.pop("interrupted_recovery", None)
progress = list(meta.get("progress_log", []))
progress.append(f"[Recovery] Resumed at {datetime.now().isoformat()}")
meta["progress_log"] = progress[-20:]
task.metadata = meta
projection_id = work_item_projection_id_from_metadata(meta, fallback=task.id)
try:
await apply_task_status_transition(
self._engine.store,
task,
target_status_or_phase=TaskStatus.PENDING,
reason="office_recovery_resume",
release_claim=True,
)
except Exception as exc:
logger.warning("Recovery resume skipped %s: %s", task.id, exc)
failed_ids.append(projection_id)
continue
if task.status != TaskStatus.PENDING:
logger.warning("Recovery resume preserved non-runnable phase for %s", task.id)
failed_ids.append(projection_id)
continue
await self._engine.store.save_task(task)
resumed_ids.append(projection_id)
if not resumed_ids:
return {
"ok": False,
"error": "no_work_items_to_resume",
"failed_work_item_projection_ids": failed_ids,
}
# Invalidate cache
self._cache_until = 0.0
# Launch execution in background
await self._set_run_recovery_state(
wf.parent_session_id,
status="resuming",
lifecycle_status="active",
)
bg_task = asyncio.create_task(
self._execute_recovery(parent_task_id, wf.parent_session_id, plan, tasks)
)
self._active_recoveries[parent_task_id] = bg_task
return {
"ok": True,
"resumed_work_item_projection_ids": resumed_ids,
"failed_work_item_projection_ids": failed_ids,
}
async def cancel(self, parent_task_id: str) -> dict[str, Any]:
"""Cancel an interrupted work-item run and clean up."""
async with self._lock:
# Cancel active recovery if running
bg = self._active_recoveries.pop(parent_task_id, None)
if bg and not bg.done():
bg.cancel()
# Find the interrupted work-item run.
status = await self.scan()
wf = next((w for w in status.interrupted if w.parent_task_id == parent_task_id), None)
if not wf:
return {"ok": False, "error": "not_found"}
# Load tasks and cancel non-terminal ones
snapshot = await self._engine._load_company_runtime_snapshot(wf.parent_session_id)
if not snapshot:
return {"ok": False, "error": "snapshot_unavailable"}
_, tasks = snapshot
from opc.core.models import TaskStatus
cancelled_count = 0
failed_ids: list[str] = []
for task in tasks:
if task.status not in (TaskStatus.DONE, TaskStatus.CANCELLED):
projection_id = work_item_projection_id_from_metadata(
getattr(task, "metadata", {}) or {},
fallback=task.id,
)
try:
await apply_task_status_transition(
self._engine.store,
task,
target_status_or_phase=TaskStatus.CANCELLED,
reason="office_recovery_cancel",
release_claim=True,
)
except Exception as exc:
logger.warning("Recovery cancel skipped %s: %s", task.id, exc)
failed_ids.append(projection_id)
continue
if task.status != TaskStatus.CANCELLED:
logger.warning("Recovery cancel preserved non-cancelled phase for %s", task.id)
failed_ids.append(projection_id)
continue
cancelled_count += 1
# Clean orphaned checkpoints
await self._clean_orphaned_checkpoints(wf, tasks)
# Invalidate cache
self._cache_until = 0.0
await self._set_run_recovery_state(
wf.parent_session_id,
status="cancelled",
lifecycle_status="cancelled",
)
await self._broadcast_status()
return {
"ok": True,
"cancelled_count": cancelled_count,
"failed_work_item_projection_ids": failed_ids,
}
# ------------------------------------------------------------------
# Internal helpers
# ------------------------------------------------------------------
async def _execute_recovery(
self,
parent_task_id: str,
parent_session_id: str,
plan: Any,
tasks: list[Any],
) -> None:
"""Run the work-item executor in the background."""
project_id = self._engine.project_id or "default"
try:
await self._set_run_recovery_state(
parent_session_id,
status="started",
lifecycle_status="active",
)
await self._broadcast({"type": "recovery_result", "payload": {
"project_id": project_id,
"parent_task_id": parent_task_id, "status": "started",
}})
executor = self._engine.company_executor
if not executor:
raise RuntimeError("company_executor not available")
result = await executor.execute(plan, tasks)
await self._broadcast({"type": "recovery_result", "payload": {
"project_id": project_id,
"parent_task_id": parent_task_id, "status": "completed",
"summary": result[:500] if result else "",
}})
await self._set_run_recovery_state(
parent_session_id,
status="completed",
lifecycle_status="active",
)
except asyncio.CancelledError:
await self._broadcast({"type": "recovery_result", "payload": {
"project_id": project_id,
"parent_task_id": parent_task_id, "status": "cancelled",
}})
await self._set_run_recovery_state(
parent_session_id,
status="cancelled",
lifecycle_status="cancelled",
)
except Exception as exc:
logger.warning(f"Recovery execution failed for {parent_task_id}: {exc}")
await self._broadcast({"type": "recovery_result", "payload": {
"project_id": project_id,
"parent_task_id": parent_task_id, "status": "failed",
"error": str(exc),
}})
await self._set_run_recovery_state(
parent_session_id,
status="failed",
lifecycle_status="blocked",
extra={"error": str(exc)},
)
finally:
self._active_recoveries.pop(parent_task_id, None)
self._cache_until = 0.0
await self._broadcast_status()
async def _clean_orphaned_checkpoints(
self,
wf: InterruptedWorkItemRun,
tasks: list[Any],
) -> int:
"""Resolve pending checkpoints whose tasks are no longer active."""
store = self._engine.store
if not store:
return 0
session_ids = {
str(getattr(t, "session_id", "") or "").strip()
for t in tasks
}
session_ids.add(wf.parent_session_id)
session_ids.discard("")
cleaned = 0
try:
pending = await store.get_pending_checkpoints(
project_id=wf.project_id,
)
for cp in pending:
cp_session = str(cp.session_id or "").strip()
if cp_session in session_ids:
await store.resolve_execution_checkpoint(
cp.checkpoint_id, status="cancelled"
)
cleaned += 1
except Exception as exc:
logger.debug(f"Checkpoint cleanup error: {exc}")
return cleaned
async def _broadcast_status(self) -> None:
"""Push updated recovery status to all connected clients."""
try:
status = await self.get_recovery_status()
await self._broadcast({"type": "recovery_status", "payload":
_serialize_status(status, project_id=self._engine.project_id or "default")
})
except Exception as exc:
logger.debug(f"Recovery status broadcast failed: {exc}")
def _invalidate_cache(self) -> None:
"""Force next get_recovery_status to re-scan."""
self._cache_until = 0.0
self._cached = None
async def _set_run_recovery_state(
self,
key: str,
*,
status: str,
lifecycle_status: str | None = None,
match_task_id: bool = False,
extra: dict[str, Any] | None = None,
) -> None:
store = getattr(self._engine, "store", None)
if not store or not hasattr(store, "list_delegation_runs") or not hasattr(store, "save_delegation_run"):
return
runs = await store.list_delegation_runs(project_id=self._engine.project_id or "default")
target = None
if match_task_id:
for run in runs:
metadata = dict(getattr(run, "metadata", {}) or {})
if str(metadata.get("origin_task_id", "") or "").strip() == key:
target = run
break
if target is None:
for run in runs:
if str(run.session_id or "").strip() == key:
target = run
break
if target is None:
return
target.recovery_pointer = {
**dict(getattr(target, "recovery_pointer", {}) or {}),
"status": status,
"updated_at": datetime.now().isoformat(),
**dict(extra or {}),
}
if lifecycle_status:
target.lifecycle_status = lifecycle_status
target.updated_at = datetime.now()
await store.save_delegation_run(target)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _is_interrupted(task: Any) -> bool:
"""Check if a task was interrupted by crash."""
from opc.core.models import TaskStatus
if task.status != TaskStatus.FAILED:
return False
meta = getattr(task, "metadata", {}) or {}
if meta.get("interrupted_recovery"):
return True
result = getattr(task, "result", {}) or {}
artifacts = result.get("artifacts", {}) or {}
return bool(artifacts.get("interrupted"))
def _serialize_status(status: RecoveryStatus, *, project_id: str | None = None) -> dict[str, Any]:
"""Convert RecoveryStatus to JSON-safe dict."""
resolved_project_id = str(project_id or "").strip()
if not resolved_project_id:
for item in status.interrupted:
if item.project_id:
resolved_project_id = item.project_id
break
payload = {
"interrupted": [
{
"parent_session_id": w.parent_session_id,
"parent_task_id": w.parent_task_id,
"project_id": w.project_id,
"title": w.title,
"profile": w.profile,
"interrupted_at": w.interrupted_at,
"work_items": [
{
"work_item_projection_id": s.work_item_projection_id,
"title": s.title,
"task_id": s.task_id,
"status": s.status,
"interrupted": s.interrupted,
"previous_status": s.previous_status,
}
for s in w.work_items
],
}
for w in status.interrupted
],
"active_recoveries": status.active_recoveries,
"scanned_at": status.scanned_at,
}
if resolved_project_id:
payload["project_id"] = resolved_project_id
return payload
opc/plugins/office_ui/server.py
-11
View File
@@ -144,17 +144,6 @@ async def create_app(
engine.event_bus.subscribe_all(_root_engine_event)
# ── Runtime crash recovery (pluggable, no engine modifications) ──
from opc.plugins.office_ui.recovery_manager import RuntimeRecoveryManager
recovery_manager = RuntimeRecoveryManager(engine, ws_handler.broadcast)
ws_handler.recovery_manager = recovery_manager
# ── Startup self-heal for tasks abandoned by a prior process ─────
# Must run before restoring persisted mode and before any WS client can
# connect, so orphaned running/locked rows do not block new Continue /
# session_send acquisitions.
await ws_handler.heal_orphan_tasks_on_boot()
# ── Restore persisted mode and load matching agents on startup ───
await ws_handler.restore_persisted_mode()
startup_preset = ws_handler._resolve_preset_name()
opc/plugins/office_ui/services/models.py
+10 -1
View File
@@ -32,4 +32,13 @@ class ServiceError(Exception):
self.payload = dict(payload or {})
def to_payload(self) -> dict[str, Any]:
return {"error": self.message, "code": self.code, **self.payload}
# Transport envelope fields are authoritative. Business details may
# add context, but must never turn an error acknowledgement into a
# success or replace the exception's code/message.
payload = {
key: value
for key, value in self.payload.items()
if key not in {"ok", "error", "code"}
}
payload.update({"error": self.message, "code": self.code})
return payload
opc/plugins/office_ui/services/runtime.py
-45
View File
@@ -200,48 +200,3 @@ class RuntimeService:
payload["display_text"] = " | ".join(display_parts)
payload["event_type"] = event_type
return payload
async def recovery_scan(self, *, project_id: str) -> ServiceResult:
manager = await self._recovery_manager(project_id)
from opc.plugins.office_ui.recovery_manager import _serialize_status
status = await manager.get_recovery_status()
return ServiceResult(_serialize_status(status, project_id=project_id))
async def recovery_action(self, *, project_id: str, action: str, parent_task_id: str) -> ServiceResult:
manager = await self._recovery_manager(project_id)
normalized = str(action or "").strip().lower()
if normalized == "scan":
return await self.recovery_scan(project_id=project_id)
if not str(parent_task_id or "").strip():
raise ServiceError("parent_task_id_required", "parent_task_id required")
if normalized in {"resume", "retry"}:
payload = await manager.resume(parent_task_id)
elif normalized == "cancel":
payload = await manager.cancel(parent_task_id)
else:
raise ServiceError("unknown_recovery_action", f"unknown action: {action}", {"action": action})
payload = {**dict(payload), "project_id": project_id, "parent_task_id": parent_task_id, "action": normalized}
if not payload.get("ok", False):
raise ServiceError(str(payload.get("error") or "recovery_failed"), str(payload.get("error") or "recovery_failed"), payload)
return ServiceResult(payload)
async def _recovery_manager(self, project_id: str) -> Any:
engine = await self.context.engine_for_project(project_id)
async def _noop_broadcast(_event: dict[str, Any]) -> None:
return None
managers = getattr(self.context, "recovery_managers", None)
if managers is None:
managers = {}
setattr(self.context, "recovery_managers", managers)
key = self.context.normalize_project_id(project_id)
existing = managers.get(key)
if existing is not None and getattr(existing, "_engine", None) is engine:
return existing
from opc.plugins.office_ui.recovery_manager import RuntimeRecoveryManager
manager = RuntimeRecoveryManager(engine, _noop_broadcast)
managers[key] = manager
return manager
opc/plugins/office_ui/services/session.py
+305 -182
View File
@@ -12,6 +12,12 @@ from typing import Any
from loguru import logger
from opc.core.models import Task, TaskStatus
from opc.layer2_organization.company_runtime_identity import (
ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES,
COMPANY_RUNTIME_CHECKPOINT_TYPES,
is_company_runtime_task,
load_company_runtime_identity_index,
)
from opc.plugins.office_ui.execution_identity import (
ExecutionIdentity,
canonicalize_execution_identity,
@@ -156,6 +162,32 @@ class SessionService:
task_project = self.context.normalize_project_id(getattr(task, "project_id", None))
if task_project != pid:
raise ServiceError("target_wrong_project", "Target belongs to a different project", {"project_id": task_project})
# Company control is resolved from durable session/checkpoint scope
# before any generic Task/session fallback. This prevents a shared
# final-decider row from winning merely because it was loaded first.
identity_index = await load_company_runtime_identity_index(store, pid)
if task is not None:
identity = identity_index.resolve(task_id=raw_target)
else:
identity = (
identity_index.resolve(runtime_session_id=raw_target)
or identity_index.resolve(task_session_id=raw_target)
)
if identity is not None:
# A concrete Task id is the caller's UI/chat channel, never the
# company execution identity. Keep that channel stable while the
# shared identity supplies the durable runtime session/checkpoint.
if task is not None:
return task, identity.runtime_session_id
resolved_task = (
identity_index.task(identity.ui_anchor_task_id)
or identity_index.task(identity.config_source_task_id)
)
if resolved_task is not None:
return resolved_task, identity.runtime_session_id
if task is not None:
return task, str(getattr(task, "session_id", "") or getattr(task, "parent_session_id", "") or "")
session = await store.get_session(raw_target) if hasattr(store, "get_session") else None
@@ -165,139 +197,64 @@ class SessionService:
if session_project != pid:
raise ServiceError("target_wrong_project", "Target belongs to a different project", {"project_id": session_project})
tasks = await store.get_tasks(project_id=pid) if hasattr(store, "get_tasks") else []
tasks = list(identity_index.tasks)
session_tasks = [
candidate for candidate in tasks
if str(getattr(candidate, "session_id", "") or "") == raw_target
]
if not session_tasks:
raise ServiceError("session_not_task_backed", "Session is not linked to a task-backed runtime", {"session_id": raw_target})
session_tasks.sort(key=lambda item: bool(str(getattr(item, "parent_session_id", "") or "")))
return session_tasks[0], raw_target
task_mode_anchor = min(
session_tasks,
key=lambda item: (
bool(str(getattr(item, "parent_session_id", "") or "")),
str(getattr(item, "created_at", "") or ""),
str(getattr(item, "id", "") or ""),
),
)
return task_mode_anchor, raw_target
async def _resolve_company_runtime_target(self, engine: Any, task: Any) -> dict[str, Any]:
store = getattr(engine, "store", None)
parent_session_id = str(
getattr(task, "parent_session_id", "")
or getattr(task, "session_id", "")
or ""
).strip()
parent_task_id = str(self.context.session_to_task.get(parent_session_id) or "").strip()
project_id = self.context.normalize_project_id(getattr(task, "project_id", None) or getattr(engine, "project_id", None))
try:
project_tasks = await store.get_tasks(project_id=project_id) if hasattr(store, "get_tasks") else [task]
except Exception:
project_tasks = [task]
for candidate in project_tasks:
candidate_id = str(getattr(candidate, "id", "") or "").strip()
candidate_session_id = str(getattr(candidate, "session_id", "") or "").strip()
candidate_parent_session_id = str(getattr(candidate, "parent_session_id", "") or "").strip()
if candidate_session_id == parent_session_id and not candidate_parent_session_id:
parent_task_id = candidate_id
break
if not parent_task_id:
parent_task_id = str(
self.context.active_runtime_children.get(str(getattr(task, "id", "") or ""))
or getattr(task, "id", "")
or ""
).strip()
affected_task_ids: list[str] = []
for candidate in project_tasks:
candidate_id = str(getattr(candidate, "id", "") or "").strip()
if not candidate_id:
continue
candidate_session_id = str(getattr(candidate, "session_id", "") or "").strip()
candidate_parent_session_id = str(getattr(candidate, "parent_session_id", "") or "").strip()
if (
candidate_id == str(getattr(task, "id", "") or "")
or candidate_id == parent_task_id
or candidate_session_id == parent_session_id
or candidate_parent_session_id == parent_session_id
):
if candidate_id not in affected_task_ids:
affected_task_ids.append(candidate_id)
for child_id, origin_id in list(self.context.active_runtime_children.items()):
if origin_id == parent_task_id or child_id == str(getattr(task, "id", "") or ""):
if child_id not in affected_task_ids:
affected_task_ids.append(child_id)
if parent_task_id and parent_task_id not in affected_task_ids:
affected_task_ids.insert(0, parent_task_id)
return {
"parent_session_id": parent_session_id,
"parent_task_id": parent_task_id or str(getattr(task, "id", "") or ""),
"origin_task_id": parent_task_id or str(getattr(task, "id", "") or ""),
"affected_task_ids": affected_task_ids or [str(getattr(task, "id", "") or "")],
}
async def _mark_company_runtime_stop_state(
async def _resolve_company_runtime_target(
self,
*,
engine: Any,
task_ids: list[str],
state: str,
stop_intent_id: str,
checkpoint_type: str = "",
) -> None:
task: Any,
*,
runtime_session_id: str = "",
checkpoint_id: str = "",
) -> dict[str, Any]:
store = getattr(engine, "store", None)
project_id = self.context.normalize_project_id(getattr(task, "project_id", None) or getattr(engine, "project_id", None))
if not self.context.store_is_ready(store):
return
for task_id in task_ids:
try:
task = await store.get_task(str(task_id))
except Exception:
task = None
if not task or self._is_terminal_status(task):
continue
metadata = dict(getattr(task, "metadata", {}) or {})
metadata["company_runtime_stop_state"] = state
metadata["company_runtime_stop_intent_id"] = stop_intent_id
metadata["company_runtime_stop_marked_at"] = datetime.now().isoformat()
metadata["dispatch_hold"] = "company_runtime_suspended"
metadata["company_runtime_suspended_at"] = datetime.now().isoformat()
if checkpoint_type:
metadata["company_runtime_suspend_checkpoint_type"] = checkpoint_type
metadata.setdefault("suspended_task_status", self._task_status_value(task))
task.metadata = metadata
task.status = TaskStatus.BLOCKED
if hasattr(task, "execution_lock"):
task.execution_lock = False
if hasattr(task, "execution_locked_at"):
task.execution_locked_at = None
try:
await store.save_task(task)
except Exception:
logger.opt(exception=True).debug("failed to mark company runtime stop state")
async def _clear_company_runtime_stop_state(self, *, engine: Any, task_ids: list[str]) -> None:
store = getattr(engine, "store", None)
if not self.context.store_is_ready(store):
return
for task_id in task_ids:
try:
task = await store.get_task(str(task_id))
except Exception:
task = None
if not task:
continue
metadata = dict(getattr(task, "metadata", {}) or {})
for key in (
"dispatch_hold",
"company_runtime_stop_state",
"company_runtime_stop_intent_id",
"company_runtime_stop_marked_at",
"company_runtime_suspend_checkpoint_type",
"company_runtime_suspended_at",
"suspended_task_status",
):
metadata.pop(key, None)
task.metadata = metadata
if self._task_status_value(task) == "blocked":
task.status = TaskStatus.IDLE
try:
await store.save_task(task)
except Exception:
logger.opt(exception=True).debug("failed to clear company runtime stop state")
raise ServiceError("store_not_ready", "store_not_ready", {"project_id": project_id})
index = await load_company_runtime_identity_index(store, project_id)
identity = index.resolve(
task_id=str(getattr(task, "id", "") or ""),
runtime_session_id=runtime_session_id,
checkpoint_id=checkpoint_id,
)
if identity is None:
raise ServiceError(
"company_runtime_identity_mismatch",
"Company runtime identity does not match the requested task/session/checkpoint",
{
"task_id": str(getattr(task, "id", "") or ""),
"runtime_session_id": str(runtime_session_id or ""),
"checkpoint_id": str(checkpoint_id or ""),
},
)
config_task = index.task(identity.config_source_task_id) or task
ui_anchor_task_id = identity.ui_anchor_task_id
return {
"identity": identity,
"runtime_session_id": identity.runtime_session_id,
"ui_channel_task_id": str(getattr(task, "id", "") or ""),
"ui_anchor_task_id": ui_anchor_task_id,
"config_source_task_id": identity.config_source_task_id,
"config_task": config_task,
"origin_task_id": ui_anchor_task_id,
"affected_task_ids": list(identity.runtime_task_ids),
"checkpoint": identity.checkpoint,
}
def _normalize_requested_config(
self,
@@ -743,8 +700,54 @@ class SessionService:
if getattr(engine, "memory", None):
await engine.memory.ensure_session(task.session_id, project_id=project_id, title=task.title, mode="primary", metadata={"source": "service"})
await store.save_task(task)
# A Task selected by the UI/CLI is only the chat channel for company
# mode. Resolve the durable runtime scope before choosing execution
# configuration, session, origin, or checkpoint. In particular, a
# role/work-item Task must never become the parent execution identity.
company_target: dict[str, Any] | None = None
task_is_company_runtime = is_company_runtime_task(task)
try:
company_target = await self._resolve_company_runtime_target(engine, task)
except ServiceError as exc:
if exc.code != "company_runtime_identity_mismatch" or task_is_company_runtime:
raise
if task.status == TaskStatus.CANCELLED:
company_identity = (
company_target.get("identity")
if company_target is not None
else None
)
checkpoint = (
company_target.get("checkpoint")
if company_target is not None
else None
)
active_cancelled_anchor = bool(
company_identity is not None
and str(getattr(company_identity, "ui_anchor_task_id", "") or "").strip()
== str(getattr(task, "id", "") or "").strip()
and checkpoint is not None
and str(getattr(checkpoint, "checkpoint_type", "") or "").strip()
in COMPANY_RUNTIME_CHECKPOINT_TYPES
and str(getattr(checkpoint, "status", "") or "").strip().lower()
in ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES
)
if not active_cancelled_anchor:
raise ServiceError(
"session_ended",
"session_ended",
{"project_id": project_id, "task_id": task.id},
)
config_task = (
company_target.get("config_task")
if company_target is not None
else task
) or task
identity = self.resolve_task_identity(
task,
config_task,
default_exec_mode=mode,
default_company_profile=company_profile if company_profile is not None else "corporate",
default_preferred_agent=preferred_agent if preferred_agent is not None else "native",
@@ -752,26 +755,85 @@ class SessionService:
)
if identity.is_custom_org and not identity.org_id:
raise ServiceError("org_id_required", "org_id_required", {"project_id": project_id, "task_id": task.id})
await self.persist_session_config(
task,
exec_mode=identity.exec_mode,
company_profile=identity.company_profile,
preferred_agent=identity.preferred_agent,
org_id=identity.org_id,
engine=engine,
)
# Existing company scopes read configuration from the resolver's
# config source. Only persist when that source is the selected Task;
# this preserves normal session configuration while avoiding writes to
# an internal work item merely because it was used as the UI channel.
if (
company_target is None
or str(getattr(config_task, "id", "") or "").strip()
== str(getattr(task, "id", "") or "").strip()
):
await self.persist_session_config(
task,
exec_mode=identity.exec_mode,
company_profile=identity.company_profile,
preferred_agent=identity.preferred_agent,
org_id=identity.org_id,
engine=engine,
)
execution_session_id = str(getattr(task, "session_id", "") or "").strip()
origin_task_id = str(getattr(task, "id", "") or "").strip() or None
message_metadata: dict[str, Any] | None = None
if company_target is not None:
execution_session_id = str(
company_target.get("runtime_session_id", "") or ""
).strip()
origin_task_id = str(
company_target.get("ui_anchor_task_id", "") or ""
).strip() or None
if not execution_session_id:
raise ServiceError(
"company_runtime_identity_mismatch",
"Company runtime has no canonical runtime session",
{"project_id": project_id, "task_id": task.id},
)
checkpoint = company_target.get("checkpoint")
if checkpoint is not None:
checkpoint_id = str(
getattr(checkpoint, "checkpoint_id", "") or ""
).strip()
checkpoint_status = str(
getattr(checkpoint, "status", "") or ""
).strip().lower()
if checkpoint_status != "pending":
raise ServiceError(
"company_runtime_checkpoint_not_pending",
"Company runtime checkpoint is not pending",
{
"project_id": project_id,
"task_id": task.id,
"checkpoint_id": checkpoint_id,
"checkpoint_status": checkpoint_status,
},
)
message_metadata = {
"response_to_checkpoint_id": checkpoint_id,
"response_to_checkpoint_type": str(
getattr(checkpoint, "checkpoint_type", "") or ""
).strip(),
}
response = await engine.process_message(
str(content or "").strip(),
project_id=project_id,
session_id=task.session_id,
session_id=execution_session_id,
mode=identity.exec_mode,
org_id=identity.org_id or None,
company_profile=identity.company_profile if identity.is_company_runtime else None,
preferred_agent=identity.preferred_agent if identity.is_task else None,
domains=list(domains or []),
origin_task_id=task.id,
origin_task_id=origin_task_id,
message_metadata=message_metadata,
)
return ServiceResult({"project_id": project_id, "task_id": task.id, "session_id": task.session_id, "response": response})
return ServiceResult({
"project_id": project_id,
"task_id": task.id,
"session_id": execution_session_id,
"response": response,
})
async def rename(self, *, project_id: str, task_id: str = "", session_id: str = "", title: str) -> ServiceResult:
pid = self.context.normalize_project_id(project_id)
@@ -935,50 +997,44 @@ class SessionService:
default_payload=default_payload,
)
engine = await self.context.engine_for_project(project_id)
exec_mode, _company_profile = self.resolve_task_session_config(task)
if exec_mode in {"company", "org", "custom"}:
try:
target_info = await self._resolve_company_runtime_target(engine, task)
except ServiceError as exc:
if exc.code != "company_runtime_identity_mismatch":
raise
target_info = None
if target_info is None and is_company_runtime_task(task):
raise ServiceError(
"company_runtime_identity_mismatch",
"Company runtime identity could not be resolved; refusing task-mode cancellation",
{"task_id": resolved_task_id, "session_id": resolved_session_id},
)
if target_info is not None:
stop_intent_id = str(uuid.uuid4())
affected_task_ids = list(target_info.get("affected_task_ids", []) or [resolved_task_id])
suspended: dict[str, Any] | None = None
suspend = getattr(engine, "suspend_company_runtime", None)
await self._mark_company_runtime_stop_state(
engine=engine,
task_ids=affected_task_ids,
state="suspending",
stop_intent_id=stop_intent_id,
)
if callable(suspend):
try:
suspended = await suspend(
origin_task_id=str(target_info.get("origin_task_id", "") or resolved_task_id),
session_id=(str(target_info.get("parent_session_id", "") or resolved_session_id).strip() or None),
session_id=(str(target_info.get("runtime_session_id", "") or resolved_session_id).strip() or None),
reason="user_stop",
checkpoint_type="company_runtime_suspended",
stop_intent_id=stop_intent_id,
)
except Exception:
logger.opt(exception=True).warning("suspend_company_runtime failed during service stop")
if suspended is not None:
for candidate in list(suspended.get("task_ids", []) or []):
candidate_id = str(candidate or "").strip()
if candidate_id and candidate_id not in affected_task_ids:
affected_task_ids.append(candidate_id)
await self._mark_company_runtime_stop_state(
engine=engine,
task_ids=affected_task_ids,
state="suspended",
stop_intent_id=stop_intent_id,
checkpoint_type=str(suspended.get("checkpoint_type", "") or "company_runtime_suspended"),
)
else:
await self._mark_company_runtime_stop_state(
engine=engine,
task_ids=affected_task_ids,
state="suspended",
stop_intent_id=stop_intent_id,
checkpoint_type="company_runtime_suspended",
if suspended is None:
raise ServiceError(
"company_runtime_suspend_failed",
"Company runtime could not be suspended",
{"runtime_session_id": target_info.get("runtime_session_id", "")},
)
for candidate in list(suspended.get("task_ids", []) or []):
candidate_id = str(candidate or "").strip()
if candidate_id and candidate_id not in affected_task_ids:
affected_task_ids.append(candidate_id)
self.context.stop_requested_task_ids.update(affected_task_ids)
if self.context.cancel_session_tasks is not None:
for tid in affected_task_ids:
@@ -1009,8 +1065,7 @@ class SessionService:
"stop_intent_id": stop_intent_id,
"checkpoint_id": str((suspended or {}).get("checkpoint_id", "") or ""),
"task_ids": affected_task_ids,
"resume_parent_task_id": str(target_info.get("parent_task_id", "") or resolved_task_id),
"resume_parent_session_id": str(target_info.get("parent_session_id", "") or resolved_session_id),
"resume_parent_session_id": str(target_info.get("runtime_session_id", "") or resolved_session_id),
}
return ServiceResult(payload, [ServiceEvent("session_runtime_control", payload), ServiceEvent("session_updated", payload)])
@@ -1039,6 +1094,8 @@ class SessionService:
project_id: str,
task_id: str = "",
session_id: str = "",
runtime_session_id: str = "",
checkpoint_id: str = "",
target: str = "",
content: str = "",
) -> ServiceResult:
@@ -1066,45 +1123,111 @@ class SessionService:
default_payload=default_payload,
)
engine = await self.context.engine_for_project(project_id)
exec_mode, company_profile = self.resolve_task_session_config(task)
target_info = await self._resolve_company_runtime_target(engine, task) if exec_mode in {"company", "org", "custom"} else {
try:
company_target = await self._resolve_company_runtime_target(
engine,
task,
runtime_session_id=runtime_session_id,
checkpoint_id=checkpoint_id,
)
except ServiceError as exc:
if exc.code != "company_runtime_identity_mismatch" or runtime_session_id or checkpoint_id:
raise
if is_company_runtime_task(task):
raise
company_target = None
target_info = company_target or {
"affected_task_ids": [resolved_task_id],
"parent_task_id": resolved_task_id,
"parent_session_id": resolved_session_id,
"ui_anchor_task_id": resolved_task_id,
"runtime_session_id": resolved_session_id,
"config_task": task,
"checkpoint": None,
}
affected_task_ids = list(target_info.get("affected_task_ids", []) or [resolved_task_id])
await self._clear_company_runtime_stop_state(engine=engine, task_ids=affected_task_ids)
message = str(content or "").strip() or "Resume the existing runtime."
engine_mode = "company" if exec_mode == "company" else ("org" if exec_mode in {"org", "custom"} else "task")
org_id = self.resolve_task_org_id(task) if engine_mode == "org" else ""
config_task = target_info.get("config_task") or task
config_exec_mode, config_company_profile = self.resolve_task_session_config(config_task)
engine_mode = "company" if config_exec_mode == "company" else (
"org" if config_exec_mode in {"org", "custom"} else "task"
)
if engine_mode in {"company", "org"}:
checkpoint = target_info.get("checkpoint")
if checkpoint is None:
raise ServiceError("company_runtime_checkpoint_not_found", "No active company runtime checkpoint")
if str(getattr(checkpoint, "status", "") or "").strip().lower() != "pending":
raise ServiceError(
"company_runtime_checkpoint_not_pending",
"Company runtime checkpoint is not pending",
{"checkpoint_id": str(getattr(checkpoint, "checkpoint_id", "") or "")},
)
else:
checkpoint = None
org_id = self.resolve_task_org_id(config_task) if engine_mode == "org" else ""
message_metadata: dict[str, Any] = {"ui_force_resume": True}
if checkpoint is not None:
message_metadata.update({
"response_to_checkpoint_id": str(getattr(checkpoint, "checkpoint_id", "") or ""),
"response_to_checkpoint_type": str(getattr(checkpoint, "checkpoint_type", "") or ""),
})
response = await engine.process_message(
message,
project_id=self.context.normalize_project_id(project_id),
session_id=str(target_info.get("parent_session_id", "") or resolved_session_id),
session_id=str(target_info.get("runtime_session_id", "") or resolved_session_id),
mode=engine_mode,
org_id=org_id or None,
company_profile=company_profile if engine_mode == "company" else None,
preferred_agent=self.resolve_task_preferred_agent(task) if engine_mode == "task" else None,
origin_task_id=str(target_info.get("parent_task_id", "") or resolved_task_id),
message_metadata={"ui_force_resume": True},
company_profile=config_company_profile if engine_mode == "company" else None,
preferred_agent=self.resolve_task_preferred_agent(config_task) if engine_mode == "task" else None,
origin_task_id=(str(target_info.get("ui_anchor_task_id", "") or "").strip() or None),
message_metadata=message_metadata,
)
runtime_control_state = "idle"
pending_runtime_checkpoint_id = ""
if engine_mode in {"company", "org"}:
refreshed_index = await load_company_runtime_identity_index(
engine.store,
self.context.normalize_project_id(project_id),
)
refreshed_identity = refreshed_index.resolve(
runtime_session_id=str(
target_info.get("runtime_session_id", "") or resolved_session_id
),
)
if refreshed_identity is not None and refreshed_identity.pending_checkpoint_id:
pending_runtime_checkpoint_id = refreshed_identity.pending_checkpoint_id
runtime_control_state = (
"suspended"
if refreshed_identity.pending_checkpoint_status == "pending"
else "resuming"
)
payload = {
**default_payload,
"status": "resuming",
"runtime_control_state": "resuming",
"can_resume": False,
"status": runtime_control_state,
"runtime_control_state": runtime_control_state,
"can_resume": runtime_control_state == "suspended",
"response": response,
"task_ids": affected_task_ids,
"resume_parent_task_id": str(target_info.get("parent_task_id", "") or resolved_task_id),
"resume_parent_session_id": str(target_info.get("parent_session_id", "") or resolved_session_id),
"resume_parent_session_id": str(target_info.get("runtime_session_id", "") or resolved_session_id),
"pending_runtime_checkpoint_id": pending_runtime_checkpoint_id,
}
return ServiceResult(payload, [ServiceEvent("session_runtime_control", payload), ServiceEvent("session_updated", payload)])
async def resume(self, *, project_id: str, task_id: str = "", session_id: str = "", target: str = "", content: str = "") -> ServiceResult:
async def resume(
self,
*,
project_id: str,
task_id: str = "",
session_id: str = "",
runtime_session_id: str = "",
checkpoint_id: str = "",
target: str = "",
content: str = "",
) -> ServiceResult:
return await self.continue_run(
project_id=project_id,
task_id=task_id,
session_id=session_id,
runtime_session_id=runtime_session_id,
checkpoint_id=checkpoint_id,
target=target,
content=content,
)
opc/plugins/office_ui/snapshot_builder.py
+71 -80
View File
@@ -39,6 +39,11 @@ from opc.layer2_organization.phase import (
should_hide_work_item_from_company_kanban,
verdict,
)
from opc.layer2_organization.company_runtime_identity import (
ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES,
COMPANY_RUNTIME_CHECKPOINT_TYPES,
build_company_runtime_identity_index,
)
from opc.layer2_organization.work_item_context_view import WorkItemContextView
from opc.layer2_organization.work_item_identity import (
WORK_ITEM_PROJECTION_ID_KEY,
@@ -1503,6 +1508,11 @@ def _primary_session_tasks_by_session_id(
*,
task_meta_map: dict[str, dict[str, Any]] | None = None,
) -> tuple[dict[str, Any], list[str]]:
identity_index = build_company_runtime_identity_index(tasks)
company_identities = {
identity.runtime_session_id: identity
for identity in identity_index.identities
}
primary_tasks_by_session_id: dict[str, Any] = {}
ordered_session_ids: list[str] = []
for task in tasks:
@@ -1513,6 +1523,19 @@ def _primary_session_tasks_by_session_id(
if bool(task_meta.get("review_task", False)):
continue
session_id = str(getattr(task, "session_id", "") or "").strip()
company_identity = company_identities.get(session_id)
if company_identity is not None:
anchor = identity_index.task(company_identity.ui_anchor_task_id)
if anchor is not None and session_id not in primary_tasks_by_session_id:
primary_tasks_by_session_id[session_id] = anchor
ordered_session_ids.append(session_id)
if anchor is not None:
# A shared final-decider/work-item Task must never replace a
# pure UI anchor that owns the same session id.
continue
# Without a pure anchor this scope has no primary chat container.
# Never synthesize one from a role/work-item Task.
continue
if not session_id or _task_parent_session_link(task, task_meta):
continue
current = primary_tasks_by_session_id.get(session_id)
@@ -1530,31 +1553,16 @@ def _primary_session_tasks_by_session_id(
return primary_tasks_by_session_id, ordered_session_ids
def _shared_role_identity_tasks_by_session_id(
def _company_config_source_tasks_by_session_id(
tasks: list[Any],
*,
task_meta_map: dict[str, dict[str, Any]] | None = None,
) -> dict[str, Any]:
identity_tasks_by_session_id: dict[str, Any] = {}
for task in tasks:
task_id = str(getattr(task, "id", "") or "").strip()
task_meta = (
task_meta_map.get(task_id, {}) if task_meta_map is not None and task_id else _task_metadata(task)
)
session_id = _shared_role_session_key(task, task_meta)
if not session_id:
continue
current = identity_tasks_by_session_id.get(session_id)
if current is None:
identity_tasks_by_session_id[session_id] = task
continue
current_id = str(getattr(current, "id", "") or "").strip()
current_meta = (
task_meta_map.get(current_id, {}) if task_meta_map is not None and current_id else _task_metadata(current)
)
if _session_representative_rank(task, task_meta) > _session_representative_rank(current, current_meta):
identity_tasks_by_session_id[session_id] = task
return identity_tasks_by_session_id
identity_index = build_company_runtime_identity_index(tasks)
return {
identity.runtime_session_id: task
for identity in identity_index.identities
if identity.config_source_task_id
and (task := identity_index.task(identity.config_source_task_id)) is not None
}
async def build_company_kanban_projection(
@@ -2594,31 +2602,7 @@ async def _build_company_runtime_control_by_task(
if not store:
return {}
parent_task_by_session: dict[str, str] = {}
tasks_by_parent_session: dict[str, list[Any]] = {}
for task in tasks:
metadata = dict(getattr(task, "metadata", {}) or {})
mode = str(metadata.get("mode", "") or metadata.get("exec_mode", "") or "").strip().lower()
is_company_runtime_task = bool(
mode in {"company", "org", "custom"}
or str(getattr(task, "parent_session_id", "") or "").strip()
or metadata.get("company_profile")
or metadata.get("company_work_item_plan")
or metadata.get("work_item_runtime")
or metadata.get("work_item_projection_id")
)
if not is_company_runtime_task:
continue
session_id = str(getattr(task, "session_id", "") or "").strip()
parent_session_id = str(getattr(task, "parent_session_id", "") or "").strip()
task_id = str(getattr(task, "id", "") or "").strip()
if session_id and not parent_session_id:
parent_task_by_session[session_id] = task_id
runtime_parent_session_id = parent_session_id or session_id
if runtime_parent_session_id:
tasks_by_parent_session.setdefault(runtime_parent_session_id, []).append(task)
checkpoints_by_session: dict[str, Any] = {}
checkpoints: list[Any] = []
getter = getattr(store, "get_execution_checkpoints", None)
if not callable(getter):
getter = getattr(store, "get_pending_checkpoints", None)
@@ -2626,29 +2610,25 @@ async def _build_company_runtime_control_by_task(
try:
kwargs = {
"project_id": project_id,
"checkpoint_types": ["company_runtime_suspended", "company_runtime_interrupted"],
"checkpoint_types": sorted(COMPANY_RUNTIME_CHECKPOINT_TYPES),
}
if getattr(getter, "__name__", "") == "get_execution_checkpoints":
kwargs["statuses"] = ["pending", "resuming"]
kwargs["statuses"] = sorted(ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES)
checkpoints = await getter(**kwargs)
for checkpoint in checkpoints:
sid = str(getattr(checkpoint, "session_id", "") or "").strip()
if sid and sid not in checkpoints_by_session:
checkpoints_by_session[sid] = checkpoint
except Exception:
logger.opt(exception=True).debug("snapshot: failed to load company runtime checkpoints")
checkpoints = []
identity_index = build_company_runtime_identity_index(tasks, checkpoints)
result: dict[str, dict[str, Any]] = {}
for parent_session_id, group in tasks_by_parent_session.items():
checkpoint = checkpoints_by_session.get(parent_session_id)
parent_task_id = parent_task_by_session.get(parent_session_id, "")
if not parent_task_id:
for task in group:
if not str(getattr(task, "parent_session_id", "") or "").strip():
parent_task_id = str(getattr(task, "id", "") or "").strip()
break
if not parent_task_id and group:
parent_task_id = str(getattr(group[0], "id", "") or "").strip()
for identity in identity_index.identities:
group = [
task
for task_id in identity.runtime_task_ids
if (task := identity_index.task(task_id)) is not None
]
checkpoint = identity.checkpoint
def _task_status_value(task: Any) -> str:
status = getattr(task, "status", "")
@@ -2660,10 +2640,19 @@ async def _build_company_runtime_control_by_task(
task for task in group
if _task_status_value(task) not in {"done", "failed", "cancelled"}
]
has_running_task = any(
_task_status_value(task) == "running"
for task in non_terminal_group
)
# Persisted RUNNING is only a projection. The controller-local
# execution registry is the sole proof that this process still owns a
# coroutine capable of monitoring and persisting the run.
runtime_is_live = getattr(engine, "_task_runtime_is_live", None)
has_running_task = False
if callable(runtime_is_live):
for task in non_terminal_group:
live_result = runtime_is_live(task)
if inspect.isawaitable(live_result):
live_result = await live_result
if live_result is True:
has_running_task = True
break
any_stop_in_progress = any(
str((getattr(task, "metadata", {}) or {}).get("company_runtime_stop_state", "") or "").strip()
in {"suspending", "suspended", "resuming_after_suspending"}
@@ -2708,8 +2697,7 @@ async def _build_company_runtime_control_by_task(
"runtime_control_state": state,
"can_stop": state == "running",
"can_resume": state == "suspended",
"resume_parent_task_id": parent_task_id,
"resume_parent_session_id": parent_session_id,
"resume_parent_session_id": identity.runtime_session_id,
"pending_runtime_checkpoint_id": pending_checkpoint_id,
"stop_intent_id": str(checkpoint_payload.get("stop_intent_id", "") or ""),
}
@@ -3014,9 +3002,8 @@ async def build_project_index_sync(
session_tasks,
task_meta_map=task_meta_map,
)
shared_identity_tasks_by_session_id = _shared_role_identity_tasks_by_session_id(
company_config_tasks_by_session_id = _company_config_source_tasks_by_session_id(
session_tasks,
task_meta_map=task_meta_map,
)
child_tasks_by_parent: dict[str, list[Any]] = {}
for task in session_tasks:
@@ -3087,11 +3074,12 @@ async def build_project_index_sync(
representative_task = primary_tasks_by_session_id.get(session_id)
representative_task_id = str(getattr(representative_task, "id", "") or "").strip()
shared_session_id = _shared_role_session_key(t, t_meta)
if shared_session_id and representative_task_id and representative_task_id != task_id:
continue
if shared_session_id:
if not representative_task_id or representative_task_id != task_id:
continue
identity_task = t
identity_meta = t_meta
shared_identity_task = shared_identity_tasks_by_session_id.get(session_id)
shared_identity_task = company_config_tasks_by_session_id.get(session_id)
shared_identity_task_id = str(getattr(shared_identity_task, "id", "") or "").strip()
if shared_identity_task_id and shared_identity_task_id != task_id:
identity_task = shared_identity_task
@@ -3569,9 +3557,8 @@ async def build_collab_sync(
session_tasks,
task_meta_map=task_meta_map,
)
shared_identity_tasks_by_session_id = _shared_role_identity_tasks_by_session_id(
company_config_tasks_by_session_id = _company_config_source_tasks_by_session_id(
session_tasks,
task_meta_map=task_meta_map,
)
child_tasks_by_parent: dict[str, list[Any]] = {}
for task in session_tasks:
@@ -3616,11 +3603,15 @@ async def build_collab_sync(
representative_task = primary_tasks_by_session_id.get(session_id)
representative_task_id = str(getattr(representative_task, "id", "") or "").strip()
shared_session_id = _shared_role_session_key(t, t_meta)
if shared_session_id and representative_task_id and representative_task_id != str(getattr(t, "id", "") or "").strip():
continue
if shared_session_id:
if (
not representative_task_id
or representative_task_id != str(getattr(t, "id", "") or "").strip()
):
continue
identity_task = t
identity_meta = t_meta
shared_identity_task = shared_identity_tasks_by_session_id.get(session_id)
shared_identity_task = company_config_tasks_by_session_id.get(session_id)
shared_identity_task_id = str(getattr(shared_identity_task, "id", "") or "").strip()
if shared_identity_task_id and shared_identity_task_id != str(getattr(t, "id", "") or "").strip():
identity_task = shared_identity_task
opc/plugins/office_ui/tests/test_snapshot_builder_company_kanban.py
+19 -14
View File
@@ -51,8 +51,11 @@ class CompanyKanbanProjectionTests(unittest.IsolatedAsyncioTestCase):
},
)
store = MagicMock()
store.get_pending_checkpoints = AsyncMock(return_value=[])
engine = SimpleNamespace(store=store)
store.get_execution_checkpoints = AsyncMock(return_value=[])
engine = SimpleNamespace(
store=store,
_task_runtime_is_live=AsyncMock(return_value=True),
)
control = await _build_company_runtime_control_by_task(
engine,
@@ -95,8 +98,11 @@ class CompanyKanbanProjectionTests(unittest.IsolatedAsyncioTestCase):
},
)
store = MagicMock()
store.get_pending_checkpoints = AsyncMock(return_value=[])
engine = SimpleNamespace(store=store)
store.get_execution_checkpoints = AsyncMock(return_value=[])
engine = SimpleNamespace(
store=store,
_task_runtime_is_live=AsyncMock(return_value=True),
)
control = await _build_company_runtime_control_by_task(
engine,
@@ -106,8 +112,9 @@ class CompanyKanbanProjectionTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(control["child-task"]["runtime_control_state"], "running")
self.assertTrue(control["child-task"]["can_stop"])
engine._task_runtime_is_live.assert_awaited()
async def test_runtime_control_running_status_does_not_require_live_heartbeat(self) -> None:
async def test_runtime_control_running_status_requires_controller_registry_ownership(self) -> None:
created_at = datetime.now(timezone.utc)
parent_task = SimpleNamespace(
id="parent-task",
@@ -122,7 +129,7 @@ class CompanyKanbanProjectionTests(unittest.IsolatedAsyncioTestCase):
},
)
store = MagicMock()
store.get_pending_checkpoints = AsyncMock(return_value=[])
store.get_execution_checkpoints = AsyncMock(return_value=[])
engine = SimpleNamespace(
store=store,
_task_runtime_is_live=AsyncMock(return_value=False),
@@ -134,9 +141,9 @@ class CompanyKanbanProjectionTests(unittest.IsolatedAsyncioTestCase):
"proj1",
)
self.assertEqual(control["parent-task"]["runtime_control_state"], "running")
self.assertTrue(control["parent-task"]["can_stop"])
engine._task_runtime_is_live.assert_not_awaited()
self.assertEqual(control["parent-task"]["runtime_control_state"], "idle")
self.assertFalse(control["parent-task"]["can_stop"])
engine._task_runtime_is_live.assert_awaited_once_with(parent_task)
async def test_runtime_control_treats_dispatch_hold_as_suspending_without_checkpoint(self) -> None:
created_at = datetime.now(timezone.utc)
@@ -1482,7 +1489,7 @@ class CollabSyncCompanyModeTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(session["company_profile"], "custom")
self.assertEqual(session["preferred_agent"], "codex")
async def test_build_collab_sync_deduplicates_shared_role_sessions(self) -> None:
async def test_build_collab_sync_does_not_promote_shared_role_session_without_ui_anchor(self) -> None:
created_at = datetime.now(timezone.utc)
shared_session_id = "root-session:role:cto"
pending_task = SimpleNamespace(
@@ -1519,6 +1526,7 @@ class CollabSyncCompanyModeTests(unittest.IsolatedAsyncioTestCase):
engine = MagicMock()
engine.store = MagicMock()
engine.store.get_tasks = AsyncMock(return_value=[pending_task, running_task])
engine.store.get_execution_checkpoints = AsyncMock(return_value=[])
engine.project_id = "proj-shared"
engine.llm = None
@@ -1586,10 +1594,7 @@ class CollabSyncCompanyModeTests(unittest.IsolatedAsyncioTestCase):
)
sessions = result.get("sessions", [])
self.assertEqual(len(sessions), 1)
self.assertEqual(sessions[0]["project_id"], "proj-shared")
self.assertEqual(sessions[0]["task_id"], "task-running")
self.assertEqual(sessions[0]["session_id"], shared_session_id)
self.assertEqual(sessions, [])
async def test_build_collab_sync_keeps_root_session_visible_when_final_decider_shares_session(self) -> None:
created_at = datetime.now(timezone.utc)
opc/plugins/office_ui/ws_handler.py
+703 -565
View File
File diff suppressed because it is too large Load Diff