fix: unify company runtime recovery lifecycle

This commit is contained in:
LZH-YS1998
2026-07-14 14:35:43 +08:00
parent 5e02364eb4
commit b8202bbe9e
56 changed files with 8753 additions and 3542 deletions
+1 -2
View File
@@ -385,7 +385,7 @@ Company Mode turns one brief into a runtime session plus role-owned work items.
| `Agents` | Role rollup: active/waiting/pending/done roles, current tool, role work items, filters, search, and links to detailed execution progress. |
| `Info` | Status, assignees, role identity, employee assignment, selected execution agent, timing, and developer details. |
| `Comms` | Role inboxes, unread/read/sent messages, meetings, decisions, and recent communication failures. |
| `Team` | Runtime cockpit: teams, seats, approvals, unread communication, recovery state, and stop controls for the current run. |
| `Team` | Runtime cockpit: teams, seats, approvals, unread communication, run state, and stop controls for the current run. |
To inspect the detailed workflow for a role, open a company-mode session and click a role/work item in the `Chat` progress card or `Agents` tab. The Execution Progress panel shows each work item, its status, activity sections, tool progress, handoffs, review targets, and execution turn metadata.
@@ -522,7 +522,6 @@ See [`docs/cli-chat-slash.md`](docs/cli-chat-slash.md) for the full command tabl
| `opc talent` | `list`, `employees`, `import`, `hire`, `scan`, `import-selected`, `employee-detail`, `import-agent` |
| `opc market` | `presets`, `browse`, `preview`, `apply-preset`, `export`, `install`, `list`, `uninstall --yes` |
| `opc runtime` | `status`, `checkpoints`, `logs`, `run` |
| `opc recovery` | `scan`, `resume`, `cancel --yes`, `retry` |
| `opc channels` | `status`, `login`, `start`, `stop` |
Most service-style commands accept `--project/-p` and `--json`.
-1
View File
@@ -522,7 +522,6 @@ opc talent hire <template_id> <role_id> -p demo
| `opc talent` | `list``employees``import``hire``scan``import-selected``employee-detail``import-agent` |
| `opc market` | `presets``browse``preview``apply-preset``export``install``list``uninstall --yes` |
| `opc runtime` | `status``checkpoints``logs``run` |
| `opc recovery` | `scan``resume``cancel --yes``retry` |
| `opc channels` | `status``login``start``stop` |
大多数服务类命令都支持 `--project/-p``--json`
+1 -3
View File
@@ -13,7 +13,7 @@ The source of truth for this table is `_SLASH_COMMANDS` in `opc/cli/app.py`.
- `/mode [task|company] [corporate|custom]` changes how future natural-language messages run.
- `/agent [native|codex|claude_code|cursor|opencode|none]` sets or clears the preferred execution agent.
- `/domains [domain ...|clear]` sets or clears domain hints.
- Aliases: `/p` is `/project`, `/s` is `/session`, `/t` is `/task`, `/checkpoint` is `/checkpoints`, `/recovery` is `/recover`.
- Aliases: `/p` is `/project`, `/s` is `/session`, `/t` is `/task`, and `/checkpoint` is `/checkpoints`.
## Project And Session
@@ -30,8 +30,6 @@ The source of truth for this table is `_SLASH_COMMANDS` in `opc/cli/app.py`.
- `/tasks [status] [--limit N] [--full]` lists project tasks.
- `/task show|move|done|rename|delete` inspects and updates persisted tasks.
- `/runtime [--limit N] [--full]` shows live runtime, active tasks, external sessions, and checkpoints.
- `/recover [--limit N] [--full]` lists interrupted runtimes and resumable checkpoints.
- `/recover resume|cancel|retry <parent_task_id>` acts on interrupted company runtimes.
- `/logs <task_id|session_id> [--limit N] [--full]` shows execution logs, runtime events, tools, and transcript.
- `/comms <task_id> [--limit N] [--full]` shows company-mode messages, handoffs, review notes, and handoff context.
- `/attachments [--limit N] [--full]` lists current-session attachment references.
+256 -232
View File
@@ -2118,33 +2118,6 @@ def runtime_run(task_id: str = typer.Argument(...), project: Optional[str] = typ
asyncio.run(_run_service_command(project, lambda svc: svc.runtime.run_task(project_id=project or "default", task_id=task_id), json_output=json_output))
recovery_app = typer.Typer(help="Recover interrupted company runtimes")
app.add_typer(recovery_app, name="recovery")
@recovery_app.command("scan")
def recovery_scan(project: Optional[str] = typer.Option(None, "--project", "-p"), json_output: bool = typer.Option(False, "--json")):
asyncio.run(_run_service_command(project, lambda svc: svc.runtime.recovery_scan(project_id=project or "default"), json_output=json_output))
@recovery_app.command("resume")
def recovery_resume(parent_task_id: str = typer.Argument(...), project: Optional[str] = typer.Option(None, "--project", "-p"), json_output: bool = typer.Option(False, "--json")):
asyncio.run(_run_service_command(project, lambda svc: svc.runtime.recovery_action(project_id=project or "default", action="resume", parent_task_id=parent_task_id), json_output=json_output))
@recovery_app.command("cancel")
def recovery_cancel(parent_task_id: str = typer.Argument(...), yes: bool = typer.Option(False, "--yes", "-y"), project: Optional[str] = typer.Option(None, "--project", "-p"), json_output: bool = typer.Option(False, "--json")):
if not yes:
console.print("[warning]Destructive command requires --yes.[/warning]")
raise typer.Exit(code=1)
asyncio.run(_run_service_command(project, lambda svc: svc.runtime.recovery_action(project_id=project or "default", action="cancel", parent_task_id=parent_task_id), json_output=json_output))
@recovery_app.command("retry")
def recovery_retry(parent_task_id: str = typer.Argument(...), project: Optional[str] = typer.Option(None, "--project", "-p"), json_output: bool = typer.Option(False, "--json")):
asyncio.run(_run_service_command(project, lambda svc: svc.runtime.recovery_action(project_id=project or "default", action="retry", parent_task_id=parent_task_id), json_output=json_output))
comms_app = typer.Typer(help="Inspect company-mode comms")
app.add_typer(comms_app, name="comms")
@@ -2834,6 +2807,17 @@ class ChatTurnController:
self._closing = True
self.queue.clear()
await self.stop_kanban_watch(silent=True)
prepare = getattr(
self.state.engine,
"prepare_active_company_runtimes_for_shutdown",
None,
)
if callable(prepare):
# Persist company checkpoints and put the engine into
# infrastructure-shutdown mode before cancelling the active turn.
# Engine.shutdown() repeats this idempotently when it closes the
# remaining stores and subsystems.
await prepare()
task = self.active_task
if task is None or task.done():
return
@@ -2994,8 +2978,6 @@ _SLASH_COMMANDS: tuple[_SlashCommandSpec, ...] = (
_SlashCommandSpec("Tasks", "/task rename <task_id> <title>", "Rename a task and its session title."),
_SlashCommandSpec("Tasks", "/task delete <task_id> --yes", "Hard-delete a task and its persisted lifecycle data."),
_SlashCommandSpec("Runtime", "/runtime [--limit N] [--full]", "Show live runtime, active tasks, external sessions, and checkpoints."),
_SlashCommandSpec("Runtime", "/recover [--limit N] [--full]", "Show interrupted runtime and resumable checkpoints.", ("scan", "resume", "cancel", "retry")),
_SlashCommandSpec("Runtime", "/recover resume|cancel|retry <parent_task_id>", "Act on an interrupted company runtime."),
_SlashCommandSpec("Runtime", "/logs <task_id|session_id> [--limit N] [--full]", "Show execution logs, runtime events, tools, and transcript."),
_SlashCommandSpec("Runtime", "/comms <task_id> [--limit N] [--full]", "Show company-mode messages, handoffs, and review notes."),
_SlashCommandSpec("Runtime", "/attachments [--limit N] [--full]", "List current session attachment references."),
@@ -3027,7 +3009,7 @@ _SLASH_COMMANDS: tuple[_SlashCommandSpec, ...] = (
_SlashCommandSpec("Diagnostics", "/checkpoints [--limit N] [--full]", "List pending execution checkpoints."),
)
CommandSpec = _SlashCommandSpec
_SLASH_ALIASES = {"p": "project", "s": "session", "t": "task", "checkpoint": "checkpoints", "recovery": "recover", "work-item": "work-items", "workitems": "work-items"}
_SLASH_ALIASES = {"p": "project", "s": "session", "t": "task", "checkpoint": "checkpoints", "work-item": "work-items", "workitems": "work-items"}
def _initial_company_profile(config: OPCConfig) -> str:
@@ -3862,6 +3844,8 @@ async def _resolve_session_or_task(state: _InteractiveChatState, token: str) ->
async def _resolve_runtime_control_target(state: _InteractiveChatState, target: str = "") -> tuple[str, str]:
from opc.layer2_organization.company_runtime_identity import load_company_runtime_identity_index
store = _require_chat_store(state, label="Session store")
if store is None:
return "", ""
@@ -3869,24 +3853,38 @@ async def _resolve_runtime_control_target(state: _InteractiveChatState, target:
if not raw_target:
console.print("[warning]No current session. Use /session list or /session create first.[/warning]")
return "", ""
project_id = _current_project_id(state.engine)
identity_index = await load_company_runtime_identity_index(store, project_id)
task = await store.get_task(raw_target) if hasattr(store, "get_task") else None
if task is not None:
project_id = str(getattr(task, "project_id", "") or "default")
if project_id != _current_project_id(state.engine):
console.print(f"[warning]Target belongs to project '{project_id}'. Switch project first.[/warning]")
task_project_id = str(getattr(task, "project_id", "") or "default")
if task_project_id != project_id:
console.print(f"[warning]Target belongs to project '{task_project_id}'. Switch project first.[/warning]")
return "", ""
runtime_identity = identity_index.resolve(task_id=raw_target)
if runtime_identity is not None:
return raw_target, runtime_identity.runtime_session_id
return str(getattr(task, "id", "") or ""), str(getattr(task, "session_id", "") or getattr(task, "parent_session_id", "") or "")
runtime_identity = (
identity_index.resolve(runtime_session_id=raw_target)
or identity_index.resolve(task_session_id=raw_target)
)
if runtime_identity is not None:
control_task_id = (
runtime_identity.ui_anchor_task_id
or runtime_identity.config_source_task_id
)
if control_task_id:
return control_task_id, runtime_identity.runtime_session_id
session = await store.get_session(raw_target) if hasattr(store, "get_session") else None
if session is None:
console.print(f"[warning]Task or session not found: {raw_target}[/warning]")
return "", ""
project_id = str(getattr(session, "project_id", "") or "default")
if project_id != _current_project_id(state.engine):
console.print(f"[warning]Target belongs to project '{project_id}'. Switch project first.[/warning]")
session_project_id = str(getattr(session, "project_id", "") or "default")
if session_project_id != project_id:
console.print(f"[warning]Target belongs to project '{session_project_id}'. Switch project first.[/warning]")
return "", ""
if raw_target in state.session_to_task:
return state.session_to_task[raw_target], raw_target
tasks = await store.get_tasks(project_id=project_id) if hasattr(store, "get_tasks") else []
tasks = list(identity_index.tasks)
candidates = [
item for item in tasks
if str(getattr(item, "session_id", "") or "") == raw_target
@@ -3894,8 +3892,15 @@ async def _resolve_runtime_control_target(state: _InteractiveChatState, target:
if not candidates:
console.print(f"[warning]Session is not task-backed: {raw_target}[/warning]")
return "", raw_target
candidates.sort(key=lambda item: bool(str(getattr(item, "parent_session_id", "") or "")))
return str(getattr(candidates[0], "id", "") or ""), raw_target
task_mode_anchor = min(
candidates,
key=lambda item: (
bool(str(getattr(item, "parent_session_id", "") or "")),
str(getattr(item, "created_at", "") or ""),
str(getattr(item, "id", "") or ""),
),
)
return str(getattr(task_mode_anchor, "id", "") or ""), raw_target
def _make_cli_runtime_control_context(state: _InteractiveChatState, controller: ChatTurnController | None = None) -> Any:
@@ -3923,35 +3928,66 @@ def _make_cli_runtime_control_context(state: _InteractiveChatState, controller:
return context
async def _latest_company_suspend_checkpoint(state: _InteractiveChatState) -> Any | None:
session_id = str(state.session_id or "").strip()
if not session_id:
return None
for name in ("get_active_company_runtime_suspend_checkpoint", "get_pending_company_runtime_suspend_checkpoint"):
getter = getattr(state.engine, name, None)
if callable(getter):
try:
checkpoint = await getter(session_id)
except Exception:
checkpoint = None
if checkpoint is not None and str(getattr(checkpoint, "status", "") or "pending") == "pending":
return checkpoint
async def _company_runtime_identity_for_session(
state: _InteractiveChatState,
session_id: str | None = None,
) -> Any | None:
from opc.layer2_organization.company_runtime_identity import (
load_company_runtime_identity_index,
)
runtime_session_id = str(session_id or state.session_id or "").strip()
store = getattr(state.engine, "store", None)
if store is not None and hasattr(store, "get_pending_checkpoints"):
try:
checkpoints = await store.get_pending_checkpoints(
project_id=_current_project_id(state.engine),
session_id=session_id,
checkpoint_types=["company_runtime_suspended", "company_runtime_interrupted"],
)
except TypeError:
checkpoints = await store.get_pending_checkpoints(project_id=_current_project_id(state.engine), session_id=session_id)
except Exception:
checkpoints = []
for checkpoint in list(checkpoints or []):
if str(getattr(checkpoint, "checkpoint_type", "") or "") in {"company_runtime_suspended", "company_runtime_interrupted"}:
return checkpoint
return None
if not runtime_session_id or store is None:
return None
try:
identity_index = await load_company_runtime_identity_index(
store,
_current_project_id(state.engine),
)
# The interactive session may be a role/work-item session. Resolve it
# as a task session after trying the canonical root session so both UI
# roots and child channels converge on the same company runtime.
identity = (
identity_index.resolve(runtime_session_id=runtime_session_id)
or identity_index.resolve(task_session_id=runtime_session_id)
)
except Exception:
return None
return identity
async def _latest_company_suspend_checkpoint(
state: _InteractiveChatState,
session_id: str | None = None,
) -> Any | None:
identity = await _company_runtime_identity_for_session(state, session_id)
return identity.checkpoint if identity is not None and identity.resumable else None
async def _company_runtime_execution_identity(
state: _InteractiveChatState,
runtime_identity: Any,
) -> Any:
"""Read execution configuration from the durable runtime config source."""
from opc.plugins.office_ui.execution_identity import execution_identity_from_task
config_source_task_id = str(
getattr(runtime_identity, "config_source_task_id", "") or ""
).strip()
store = getattr(state.engine, "store", None)
if not config_source_task_id or store is None or not hasattr(store, "get_task"):
raise RuntimeError("Company runtime has no durable configuration source.")
config_task = await store.get_task(config_source_task_id)
if config_task is None:
raise RuntimeError("Company runtime configuration source no longer exists.")
return execution_identity_from_task(
config_task,
default_exec_mode=state.mode,
default_company_profile=state.company_profile,
default_preferred_agent=state.preferred_agent or "native",
default_org_id=state.org_id,
)
async def _handle_stop_slash(state: _InteractiveChatState, args: list[str], controller: ChatTurnController | None = None) -> None:
@@ -3972,22 +4008,35 @@ async def _handle_stop_slash(state: _InteractiveChatState, args: list[str], cont
return
payload = dict(result.payload)
state.runtime_control_state = str(payload.get("runtime_control_state") or payload.get("status") or "stopped")
state.runtime_control_task_id = str(payload.get("resume_parent_task_id") or payload.get("task_id") or task_id)
state.runtime_control_task_id = str(payload.get("task_id") or task_id)
state.runtime_control_session_id = str(payload.get("resume_parent_session_id") or payload.get("session_id") or session_id)
state.runtime_control_checkpoint_id = str(payload.get("checkpoint_id") or "")
console.print("[success]Stopped.[/success] [dim]Send a message to revise, or /continue to resume.[/dim]")
async def _runtime_control_identity_for_task(state: _InteractiveChatState, task_id: str) -> Any:
async def _runtime_control_execution_identity(state: _InteractiveChatState, task_id: str) -> Any:
from opc.layer2_organization.company_runtime_identity import load_company_runtime_identity_index
from opc.plugins.office_ui.execution_identity import execution_identity_from_task
store = getattr(state.engine, "store", None)
task = None
if store is not None and hasattr(store, "get_task") and task_id:
if store is not None and task_id:
try:
task = await store.get_task(task_id)
identity_index = await load_company_runtime_identity_index(
store,
_current_project_id(state.engine),
)
runtime_identity = identity_index.resolve(task_id=task_id)
task = identity_index.task(
runtime_identity.config_source_task_id if runtime_identity is not None else task_id
)
except Exception:
task = None
if task is None and hasattr(store, "get_task"):
try:
task = await store.get_task(task_id)
except Exception:
task = None
return execution_identity_from_task(
task,
default_exec_mode=state.mode,
@@ -4014,13 +4063,24 @@ async def _handle_continue_slash(state: _InteractiveChatState, args: list[str],
task_id, session_id = await _resolve_runtime_control_target(state, target)
if not task_id:
return
identity = await _runtime_control_identity_for_task(state, task_id)
identity = await _runtime_control_execution_identity(state, task_id)
content = " ".join(message_parts).strip() or "Resume the existing runtime."
checkpoint = await _latest_company_suspend_checkpoint(state, session_id)
if identity.exec_mode in {"company", "org", "custom"} and checkpoint is None:
console.print(
"[warning]No suspended or interrupted company runtime is available to continue.[/warning]"
)
return
if controller is not None and controller.is_busy:
checkpoint = await _latest_company_suspend_checkpoint(state)
if checkpoint is None and state.runtime_control_state not in {"suspended", "stopped"}:
console.print("[warning]Busy: wait for the current turn or /stop it before /continue.[/warning]")
return
metadata: dict[str, Any] = {"ui_force_resume": True}
if checkpoint is not None:
metadata["response_to_checkpoint_id"] = str(getattr(checkpoint, "checkpoint_id", "") or "")
metadata["response_to_checkpoint_type"] = str(
getattr(checkpoint, "checkpoint_type", "") or "company_runtime_suspended"
)
item = QueuedChatInput(
text=content,
project_id=_current_project_id(state.engine),
@@ -4030,26 +4090,29 @@ async def _handle_continue_slash(state: _InteractiveChatState, args: list[str],
org_id=identity.org_id,
preferred_agent=identity.preferred_agent,
domains=list(state.domains),
message_metadata={"ui_force_resume": True},
message_metadata=metadata,
)
state.runtime_control_state = "resuming"
state.runtime_control_task_id = task_id
state.runtime_control_session_id = session_id or state.session_id
state.runtime_control_checkpoint_id = ""
state.runtime_control_checkpoint_id = str(getattr(checkpoint, "checkpoint_id", "") or "")
if controller is not None:
await controller.submit_item(item)
else:
previous_session_id = state.session_id
previous_mode = state.mode
previous_profile = state.company_profile
previous_org_id = state.org_id
previous_agent = state.preferred_agent
state.session_id = session_id or state.session_id
state.mode = identity.exec_mode
state.company_profile = identity.company_profile
state.org_id = identity.org_id
state.preferred_agent = identity.preferred_agent
try:
await _process_interactive_chat_message(state, content, message_metadata={"ui_force_resume": True})
await _process_interactive_chat_message(state, content, message_metadata=metadata)
finally:
state.session_id = previous_session_id
state.mode = previous_mode
state.company_profile = previous_profile
state.org_id = previous_org_id
@@ -5630,140 +5693,6 @@ async def _handle_runtime_slash(state: _InteractiveChatState, args: list[str]) -
_render_checkpoint_table(checkpoints[:limit], title="Pending Checkpoints", full=full)
class _ChatRecoveryFacade:
def __init__(self, state: _InteractiveChatState) -> None:
self._state = state
self.project_id = _current_project_id(state.engine)
@property
def store(self): # noqa: ANN201 - preserve recovery facade shape
return getattr(self._state.engine, "store", None)
@property
def opc_home(self) -> Path:
return Path(getattr(self._state.engine, "opc_home", get_opc_home()))
async def ensure_ready(self):
return self._state.engine
def _get_chat_recovery_manager(state: _InteractiveChatState) -> Any:
cached = getattr(state, "_cli_chat_recovery_manager", None)
engine_id = id(state.engine)
if cached and cached[0] == engine_id:
return cached[1]
from opc.plugins.cli_board.services.recovery import CliRecoveryManager
manager = CliRecoveryManager(_ChatRecoveryFacade(state))
setattr(state, "_cli_chat_recovery_manager", (engine_id, manager))
return manager
def _render_recovery_status(status: Any, *, limit: int, full: bool = False) -> None:
workflows = list(getattr(status, "interrupted", []) or [])[:limit]
if workflows:
table = Table(title=f"Interrupted Company Runtimes ({len(workflows)})")
table.add_column("Parent Task")
table.add_column("Parent Session")
table.add_column("Title")
table.add_column("Profile")
table.add_column("Interrupted")
table.add_column("Work Items")
for workflow in workflows:
interrupted_items = [
item for item in list(getattr(workflow, "work_items", []) or [])
if bool(getattr(item, "interrupted", False))
]
table.add_row(
str(getattr(workflow, "parent_task_id", "") or ""),
str(getattr(workflow, "parent_session_id", "") or ""),
_clip_text(getattr(workflow, "title", "") or "", 70, full=full),
str(getattr(workflow, "profile", "") or ""),
str(getattr(workflow, "interrupted_at", "") or ""),
f"{len(interrupted_items)}/{len(list(getattr(workflow, 'work_items', []) or []))}",
)
console.print(table)
else:
console.print("[info]No interrupted company runtimes found.[/info]")
active = list(getattr(status, "active_recoveries", []) or [])
if active:
console.print(f"[info]Active recoveries: {', '.join(active)}[/info]")
async def _find_checkpoint_by_id(store: Any, project_id: str, checkpoint_id: str) -> Any | None:
if hasattr(store, "get_execution_checkpoints"):
checkpoints = await store.get_execution_checkpoints(project_id=project_id)
else:
checkpoints = await store.get_pending_checkpoints(project_id=project_id)
return next((item for item in checkpoints if str(getattr(item, "checkpoint_id", "") or "") == checkpoint_id), None)
async def _handle_recover_slash(state: _InteractiveChatState, args: list[str]) -> None:
store = _require_chat_store(state, label="Recovery store")
if store is None:
return
if args and args[0].lower() == "resume":
if len(args) != 2:
console.print("[warning]Usage: /recover resume <parent_task_id>. Try /recover.[/warning]")
return
manager = _get_chat_recovery_manager(state)
result = await manager.resume(args[1])
if result.get("ok"):
resumed = result.get("resumed_work_item_projection_ids", []) or []
console.print(f"[success]Recovery started for {args[1]}: {', '.join(resumed) or 'runtime queued'}.[/success]")
return
if result.get("error") == "not_found":
checkpoint = await _find_checkpoint_by_id(store, _current_project_id(state.engine), args[1])
if checkpoint:
session_id = str(getattr(checkpoint, "session_id", "") or "")
suffix = f" Try /session resume {session_id}." if session_id else " Try /checkpoints."
console.print(f"[warning]Checkpoint {args[1]} is not resumed directly.{suffix}[/warning]")
return
console.print(f"[warning]Recovery could not start: {result.get('error', 'unknown_error')}[/warning]")
return
if args and args[0].lower() in {"cancel", "retry"}:
action = args[0].lower()
if len(args) < 2:
console.print(f"[warning]Usage: /recover {action} <parent_task_id>{' --yes' if action == 'cancel' else ''}. Try /recover.[/warning]")
return
if action == "cancel":
remaining, ok = _require_yes_arg(args[2:], usage="/recover cancel <parent_task_id> --yes")
if not ok or remaining:
return
elif len(args) > 2:
console.print(f"[warning]Usage: /recover {action} <parent_task_id>[/warning]")
return
payload = await _run_chat_office_service(
state,
lambda svc: svc.runtime.recovery_action(project_id=_current_project_id(state.engine), action=action, parent_task_id=args[1]),
)
if payload:
_emit_payload(payload)
return
try:
if args and args[0].lower() == "scan":
args = args[1:]
args, limit, full = _parse_view_args(args)
except ValueError as exc:
console.print(f"[warning]{exc}. Try /recover --limit 20.[/warning]")
return
if args:
console.print("[warning]Usage: /recover [--limit N] [--full] or /recover resume <parent_task_id>.[/warning]")
return
manager = _get_chat_recovery_manager(state)
status = await manager.get_status()
_render_recovery_status(status, limit=limit, full=full)
if hasattr(store, "get_execution_checkpoints"):
checkpoints = await store.get_execution_checkpoints(
project_id=_current_project_id(state.engine),
statuses=["pending", "resuming"],
)
else:
checkpoints = await store.get_pending_checkpoints(project_id=_current_project_id(state.engine))
_render_checkpoint_table(checkpoints[:limit], title="Recovery Checkpoints", full=full)
async def _resolve_logs_target(
state: _InteractiveChatState,
target: str,
@@ -7487,10 +7416,6 @@ def _busy_slash_policy(command: str, args: list[str]) -> BusyCommandPolicy:
}
if command in readonly_roots:
return BusyCommandPolicy.IMMEDIATE_READONLY
if command == "recover":
if not args or args[0].lower() in {"scan", "status", "list"}:
return BusyCommandPolicy.IMMEDIATE_READONLY
return BusyCommandPolicy.BLOCKED_WHEN_BUSY
if command == "task":
if args and args[0].lower() == "show":
return BusyCommandPolicy.IMMEDIATE_READONLY
@@ -7810,8 +7735,6 @@ async def _handle_chat_slash_command(state: _InteractiveChatState, user_input: s
await _handle_reorg_slash(state, args)
elif command == "runtime":
await _handle_runtime_slash(state, args)
elif command == "recover":
await _handle_recover_slash(state, args)
elif command in {"work-items", "work-item"}:
await _handle_work_items_slash(state, args)
elif command == "logs":
@@ -7835,6 +7758,13 @@ async def _sync_runtime_checkpoint_hint(state: _InteractiveChatState) -> None:
display = state.runtime_display
if not hasattr(display, "set_checkpoint_hint"):
return
runtime_identity = await _company_runtime_identity_for_session(state)
if runtime_identity is not None and runtime_identity.checkpoint is not None:
checkpoint = runtime_identity.checkpoint
checkpoint_type = str(getattr(checkpoint, "checkpoint_type", "") or "pending")
checkpoint_id = str(getattr(checkpoint, "checkpoint_id", "") or "")
display.set_checkpoint_hint(checkpoint_type or checkpoint_id)
return
getter = getattr(state.engine, "get_latest_pending_checkpoint_for_session", None)
if not callable(getter) or not state.session_id:
display.set_checkpoint_hint("")
@@ -7862,24 +7792,110 @@ async def _process_interactive_chat_message(
try:
state.runtime_display.begin_turn()
effective_metadata = message_metadata
if effective_metadata is None:
suspend_checkpoint = await _latest_company_suspend_checkpoint(state)
if suspend_checkpoint is not None:
effective_metadata = {
"response_to_checkpoint_id": str(getattr(suspend_checkpoint, "checkpoint_id", "") or ""),
"response_to_checkpoint_type": str(getattr(suspend_checkpoint, "checkpoint_type", "") or "company_runtime_suspended"),
}
elif state.mode == "company":
execution_session_id = state.session_id
execution_origin_task_id: str | None = None
execution_mode = state.mode
execution_company_profile = state.company_profile
execution_org_id = state.org_id
execution_preferred_agent = state.preferred_agent
handoff_identity = None
runtime_identity = await _company_runtime_identity_for_session(state)
runtime_checkpoint = (
runtime_identity.checkpoint
if runtime_identity is not None
else None
)
explicit_runtime_type = str(
(effective_metadata or {}).get("response_to_checkpoint_type", "") or ""
).strip()
explicit_runtime_id = str(
(effective_metadata or {}).get("response_to_checkpoint_id", "") or ""
).strip()
is_explicit_runtime_handoff = explicit_runtime_type in {
"company_runtime_suspended",
"company_runtime_interrupted",
}
if is_explicit_runtime_handoff:
if runtime_identity is None or runtime_checkpoint is None:
raise RuntimeError(
"Company runtime checkpoint does not match the current session."
)
checkpoint_id = str(
getattr(runtime_checkpoint, "checkpoint_id", "") or ""
).strip()
checkpoint_type = str(
getattr(runtime_checkpoint, "checkpoint_type", "") or ""
).strip()
checkpoint_status = str(
getattr(runtime_checkpoint, "status", "") or ""
).strip().lower()
if (
not explicit_runtime_id
or explicit_runtime_id != checkpoint_id
or explicit_runtime_type != checkpoint_type
):
raise RuntimeError(
"Company runtime checkpoint identity mismatch; refresh before continuing."
)
if checkpoint_status != "pending":
raise RuntimeError(
f"Company runtime checkpoint is {checkpoint_status or 'not pending'}."
)
execution_session_id = runtime_identity.runtime_session_id
execution_origin_task_id = runtime_identity.ui_anchor_task_id or None
handoff_identity = runtime_identity
elif effective_metadata is None and runtime_checkpoint is not None:
checkpoint_status = str(
getattr(runtime_checkpoint, "status", "") or ""
).strip().lower()
if checkpoint_status != "pending":
raise RuntimeError(
f"Company runtime checkpoint is {checkpoint_status or 'not pending'}."
)
if runtime_identity is None:
raise RuntimeError(
"Company runtime checkpoint does not match the current session."
)
effective_metadata = {
"response_to_checkpoint_id": str(getattr(runtime_checkpoint, "checkpoint_id", "") or ""),
"response_to_checkpoint_type": str(getattr(runtime_checkpoint, "checkpoint_type", "") or "company_runtime_suspended"),
}
execution_session_id = runtime_identity.runtime_session_id
execution_origin_task_id = runtime_identity.ui_anchor_task_id or None
handoff_identity = runtime_identity
elif effective_metadata is None:
if runtime_identity is not None and runtime_identity.pending_checkpoint_id:
# An active checkpoint may only be pending or resuming. Never
# let a malformed identity/status fall through as a new turn.
raise RuntimeError(
"Company runtime checkpoint is not available for a new turn."
)
if state.mode == "company":
effective_metadata = {"company_preflight": "manual"}
if handoff_identity is not None:
runtime_execution_identity = await _company_runtime_execution_identity(
state,
handoff_identity,
)
execution_mode = runtime_execution_identity.exec_mode
execution_company_profile = runtime_execution_identity.company_profile
execution_org_id = runtime_execution_identity.org_id
execution_preferred_agent = runtime_execution_identity.preferred_agent
response = await state.engine.process_message(
user_input,
project_id=getattr(state.engine, "project_id", None),
session_id=state.session_id,
mode=state.mode,
org_id=state.org_id or None,
company_profile=state.company_profile if state.mode == "company" else None,
preferred_agent=state.preferred_agent,
session_id=execution_session_id,
mode=execution_mode,
org_id=execution_org_id or None,
company_profile=(
execution_company_profile
if execution_mode == "company"
else None
),
preferred_agent=execution_preferred_agent,
domains=list(state.domains),
origin_task_id=execution_origin_task_id,
message_metadata=effective_metadata,
)
await state.runtime_display.flush()
@@ -7910,7 +7926,15 @@ async def _process_interactive_chat_message(
"company_runtime_interrupted",
}
):
state.runtime_control_state = "running"
remaining_checkpoint = await _latest_company_suspend_checkpoint(state)
if remaining_checkpoint is not None:
state.runtime_control_state = "suspended"
state.runtime_control_checkpoint_id = str(
getattr(remaining_checkpoint, "checkpoint_id", "") or ""
)
else:
state.runtime_control_state = "idle"
state.runtime_control_checkpoint_id = ""
except KeyboardInterrupt:
console.print("\n[warning]Interrupted. Type /quit to exit.[/warning]")
except Exception as e:
+269
View File
@@ -0,0 +1,269 @@
"""Controller-local ownership of active task execution attempts.
Persisted task rows describe durable workflow state; they cannot prove that
the controller which owns an execution coroutine is still alive. This
registry intentionally stays in memory and is shared by all engines owned by
one controller.
"""
from __future__ import annotations
import asyncio
from contextlib import contextmanager
from contextvars import ContextVar
import uuid
from collections.abc import Iterator
_CURRENT_HANDOFF: ContextVar[tuple[object, str] | None] = ContextVar(
"opc_active_task_run_handoff",
default=None,
)
_CURRENT_DRIVER_ATTEMPT: ContextVar[tuple[object, str] | None] = ContextVar(
"opc_active_task_run_driver_attempt",
default=None,
)
class ActiveTaskRunAdmissionClosed(RuntimeError):
"""Raised when execution registration starts after shutdown admission closes."""
class ActiveTaskRunRegistry:
"""Track active execution attempts by ``(project_id, task_id)``.
A task can briefly have overlapping attempts while cancellation and a new
dispatch cross. Each registration therefore receives its own token and a
task remains active until its last token is removed.
"""
def __init__(self) -> None:
self._attempts: dict[tuple[str, str], set[str]] = {}
self._scope_locks: dict[tuple[str, str], asyncio.Lock] = {}
self._handoff_refs: dict[str, int] = {}
self._handoffs_drained = asyncio.Event()
self._handoffs_drained.set()
self._admission_closed = False
@staticmethod
def _key(project_id: str | None, task_id: str | None) -> tuple[str, str]:
project = str(project_id or "default").strip() or "default"
task = str(task_id or "").strip()
if not task:
raise ValueError("task_id is required")
return project, task
def register(self, project_id: str | None, task_id: str | None) -> str:
handoff_token = self._current_pending_handoff_token()
driver_attempt_token = self._current_driver_attempt_token()
if (
self._admission_closed
and handoff_token is None
and driver_attempt_token is None
):
raise ActiveTaskRunAdmissionClosed(
"task execution admission is closed for controller shutdown"
)
key = self._key(project_id, task_id)
attempt_token = uuid.uuid4().hex
self._attempts.setdefault(key, set()).add(attempt_token)
# A pre-shutdown WS request is handed off once its first real execution
# coroutine is registered. The reservation itself is deliberately not
# reported by is_active()/active_task_ids(); only this attempt is.
if handoff_token is not None:
self._settle_handoff(handoff_token)
return attempt_token
@contextmanager
def bind_driver_attempt(self, attempt_token: str) -> Iterator[None]:
"""Allow nested attempts while their live scheduler owns the scope.
Closing controller admission rejects new ingress, but a scheduler that
was already running may be between its atomic WorkItem claim and child
coroutine creation. Its nested registrations remain admissible until
that scheduler attempt ends, so shutdown can snapshot the still-live
scope instead of creating an orphan RUNNING claim.
"""
if not self._attempt_token_is_active(attempt_token):
raise ValueError("driver attempt is not active")
context_token = _CURRENT_DRIVER_ATTEMPT.set((self, attempt_token))
try:
yield
finally:
_CURRENT_DRIVER_ATTEMPT.reset(context_token)
def reserve_handoff(self) -> str:
"""Reserve one accepted ingress request until execution is registered.
Reservations bridge the short scheduling gap between the WS router and
``register``. They are controller-local synchronization only and never
become a second liveness source.
"""
if self._admission_closed:
raise ActiveTaskRunAdmissionClosed(
"task execution admission is closed for controller shutdown"
)
token = uuid.uuid4().hex
self._handoff_refs[token] = 1
self._handoffs_drained.clear()
return token
@contextmanager
def bind_handoff(self, handoff_token: str) -> Iterator[None]:
"""Propagate a reservation through tasks spawned by an ingress handler."""
if handoff_token not in self._handoff_refs:
raise ValueError("handoff reservation is not pending")
context_token = _CURRENT_HANDOFF.set((self, handoff_token))
try:
yield
finally:
_CURRENT_HANDOFF.reset(context_token)
def retain_current_handoff(self) -> str | None:
"""Retain the bound reservation for a newly scheduled coroutine."""
handoff_token = self._current_pending_handoff_token()
if handoff_token is None:
return None
self._handoff_refs[handoff_token] += 1
return handoff_token
def release_current_handoff(self) -> bool:
"""Release an accepted request that will not start an execution."""
handoff_token = self._current_pending_handoff_token()
if handoff_token is None:
return False
return self.release_handoff(handoff_token)
def release_handoff(self, handoff_token: str) -> bool:
"""Release one owner, draining a request that exited before execution."""
refs = self._handoff_refs.get(handoff_token)
if refs is None:
return False
if refs > 1:
self._handoff_refs[handoff_token] = refs - 1
return True
self._settle_handoff(handoff_token)
return True
def revoke_handoff(self, handoff_token: str) -> bool:
"""Invalidate every retained owner of a queued ingress handoff.
Controller shutdown uses this after synchronously cancelling a request
which has not registered its first execution attempt. Revocation is
intentionally stronger than ``release_handoff``: callbacks may be
delayed by cancellation cleanup, but the revoked request must neither
keep the shutdown barrier open nor register work after admission has
closed.
"""
if handoff_token not in self._handoff_refs:
return False
self._settle_handoff(handoff_token)
return True
def _current_pending_handoff_token(self) -> str | None:
binding = _CURRENT_HANDOFF.get()
if binding is None or binding[0] is not self:
return None
token = binding[1]
return token if token in self._handoff_refs else None
def _current_driver_attempt_token(self) -> str | None:
binding = _CURRENT_DRIVER_ATTEMPT.get()
if binding is None or binding[0] is not self:
return None
token = binding[1]
return token if self._attempt_token_is_active(token) else None
def _attempt_token_is_active(self, attempt_token: str) -> bool:
return any(
attempt_token in attempts
for attempts in self._attempts.values()
)
def _settle_handoff(self, handoff_token: str) -> None:
self._handoff_refs.pop(handoff_token, None)
if not self._handoff_refs:
self._handoffs_drained.set()
@property
def admission_closed(self) -> bool:
return self._admission_closed
def close_admission(self) -> None:
"""Reject future attempts without dropping attempts already in flight."""
self._admission_closed = True
async def close_admission_and_wait_for_handoffs(self) -> None:
"""Close ingress and wait until every already-accepted request hands off.
A bound pending reservation may still call ``register`` after admission
closes. That registration atomically drains the reservation and turns
the real coroutine into the sole active fact. This wait therefore ends
at handoff, never at completion of the potentially long execution.
"""
self.close_admission()
while self._handoff_refs:
await self._handoffs_drained.wait()
@property
def pending_handoff_count(self) -> int:
return len(self._handoff_refs)
def is_handoff_pending(self, handoff_token: str | None) -> bool:
return bool(handoff_token and handoff_token in self._handoff_refs)
def scope_lock(
self,
project_id: str | None,
runtime_session_id: str | None,
) -> asyncio.Lock:
"""Return the controller-shared lock for one durable runtime scope."""
project = str(project_id or "default").strip() or "default"
session = str(runtime_session_id or "").strip()
if not session:
raise ValueError("runtime_session_id is required")
key = (project, session)
lock = self._scope_locks.get(key)
if lock is None:
lock = asyncio.Lock()
self._scope_locks[key] = lock
return lock
def unregister(
self,
project_id: str | None,
task_id: str | None,
attempt_token: str,
) -> bool:
key = self._key(project_id, task_id)
attempts = self._attempts.get(key)
if not attempts or attempt_token not in attempts:
return False
attempts.remove(attempt_token)
if not attempts:
self._attempts.pop(key, None)
return True
def is_active(self, project_id: str | None, task_id: str | None) -> bool:
return bool(self._attempts.get(self._key(project_id, task_id)))
def active_task_ids(self, project_id: str | None) -> set[str]:
project = str(project_id or "default").strip() or "default"
return {
task_id
for (candidate_project, task_id), attempts in self._attempts.items()
if candidate_project == project and attempts
}
def attempt_count(self, project_id: str | None, task_id: str | None) -> int:
return len(self._attempts.get(self._key(project_id, task_id), ()))
+358 -57
View File
@@ -2862,32 +2862,6 @@ class OPCStore:
await self._db.commit()
return cursor.rowcount > 0
async def renew_task_lock(self, task_id: str) -> bool:
"""Refresh ``execution_locked_at`` for a live running task.
This is the heartbeat used by the WS handler while it is actively
processing a session message. It only updates rows whose ``status`` is
still ``running`` (anything else has already ended and must not keep a
live timestamp), and deliberately ignores ``execution_lock``: that bit
is set only by delegation checkout, while the office_ui dispatch path
serializes through an in-memory asyncio.Lock and never flips it. The
refreshed ``execution_locked_at`` is what ``reset_orphan_running_tasks``
compares against on startup to distinguish live work from abandoned
``running`` rows.
Returns ``True`` when the row was still ``status=running`` (heartbeat
extended), ``False`` otherwise (task ended or gone caller should
stop heartbeating).
"""
assert self._db
now_iso = datetime.now().isoformat()
async with self._db.execute(
"UPDATE tasks SET execution_locked_at = ? WHERE id = ? AND status = 'running'",
(now_iso, task_id),
) as cursor:
await self._db.commit()
return cursor.rowcount > 0
async def release_task_lock(self, task_id: str) -> None:
assert self._db
await self._db.execute(
@@ -2896,37 +2870,6 @@ class OPCStore:
)
await self._db.commit()
async def reset_orphan_running_tasks(self, *, lease_seconds: int = 300) -> dict[str, int]:
"""Reset ``status=running`` tasks abandoned by a prior process.
Safe single-process assumption: when this runs during server startup,
any task that still claims ``status=running`` must be an orphan its
worker coroutine died with the previous process. We revert it to
``idle`` so the UI can Continue it, and clear any stale execution lock
regardless of status when the lease has expired.
Returns a summary dict with keys ``statuses_reset`` and
``locks_cleared`` giving the affected row counts.
"""
assert self._db
cutoff_iso = (datetime.now() - timedelta(seconds=int(lease_seconds))).isoformat()
async with self._db.execute(
"UPDATE tasks SET status = 'idle', execution_lock = 0, execution_locked_at = NULL "
"WHERE status = 'running' AND ("
"execution_locked_at IS NULL OR execution_locked_at < ?"
")",
(cutoff_iso,),
) as cursor:
statuses_reset = cursor.rowcount or 0
async with self._db.execute(
"UPDATE tasks SET execution_lock = 0, execution_locked_at = NULL "
"WHERE execution_lock = 1 AND execution_locked_at IS NOT NULL AND execution_locked_at < ?",
(cutoff_iso,),
) as cursor:
locks_cleared = cursor.rowcount or 0
await self._db.commit()
return {"statuses_reset": statuses_reset, "locks_cleared": locks_cleared}
def _row_to_task(self, row: Any, description: Any) -> Task:
cols = [d[0] for d in description]
data = dict(zip(cols, row))
@@ -5509,6 +5452,108 @@ class OPCStore:
await self.save_delegation_work_item(item)
return item
async def claim_delegation_work_item_if_dispatchable(
self,
work_item_id: str,
*,
expected_phase: Phase | str,
role_runtime_session_id: str,
seat_id: str,
task_id: str,
work_item_revision: int = 0,
) -> DelegationWorkItem | None:
"""Atomically claim an unheld, unowned dispatchable WorkItem.
The dispatcher may be operating on a snapshot loaded before a Stop or
shutdown transition. Keeping the phase, claim, queue, and durable
hold predicates in the same UPDATE prevents that stale snapshot from
resurrecting a suspended WorkItem.
"""
phase = coerce_phase(expected_phase)
dispatchable_phases = {
Phase.READY,
Phase.READY_FOR_REWORK,
*IN_PROGRESS_PHASES,
}
if phase not in dispatchable_phases:
return None
if phase != Phase.RUNNING:
validate_transition(phase, Phase.RUNNING)
role_session_id = str(role_runtime_session_id or "").strip()
claimed_task_id = str(task_id or "").strip()
if not work_item_id or not role_session_id or not claimed_task_id:
return None
updated_at = datetime.now()
db = self._require_db()
cursor = await db.execute(
"""UPDATE delegation_work_items
SET phase = ?,
role_runtime_session_id = ?,
claimed_by_role_runtime_session_id = ?,
claimed_by_seat_id = ?,
metadata = json_set(
COALESCE(NULLIF(metadata, ''), '{}'),
'$.claimed_by_role_session_id', ?,
'$.claimed_task_id', ?,
'$.claimed_work_item_revision', ?
),
updated_at = ?
WHERE work_item_id = ?
AND phase = ?
AND COALESCE(claimed_by_role_runtime_session_id, '') = ''
AND COALESCE(claimed_by_seat_id, '') = ''
AND COALESCE(json_extract(metadata, '$.claimed_by_role_session_id'), '') = ''
AND COALESCE(json_extract(metadata, '$.claimed_task_id'), '') = ''
AND COALESCE(json_extract(metadata, '$.dispatch_hold'), '') = ''
AND COALESCE(json_extract(metadata, '$.queued_behind_session'), '') = ''
AND COALESCE(
CAST(json_extract(metadata, '$.manager_mutation_revision') AS INTEGER),
0
) = ?""",
(
Phase.RUNNING.value,
role_session_id,
role_session_id,
str(seat_id or "").strip(),
role_session_id,
claimed_task_id,
int(work_item_revision or 0),
updated_at.isoformat(),
str(work_item_id or "").strip(),
phase.value,
int(work_item_revision or 0),
),
)
await db.commit()
if not (getattr(cursor, "rowcount", 0) or 0):
return None
persisted = await self.get_delegation_work_item(work_item_id)
if persisted is None:
return None
persisted_metadata = dict(persisted.metadata or {})
if (
persisted.phase != Phase.RUNNING
or str(persisted.claimed_by_role_runtime_session_id or "").strip()
!= role_session_id
or str(persisted_metadata.get("claimed_by_role_session_id", "") or "").strip()
!= role_session_id
or str(persisted_metadata.get("claimed_task_id", "") or "").strip()
!= claimed_task_id
or str(persisted_metadata.get("dispatch_hold", "") or "").strip()
or str(persisted_metadata.get("queued_behind_session", "") or "").strip()
):
# Stop/shutdown may have won immediately after the CAS commit and
# cleared this claim while adding a durable hold. The committed
# UPDATE is not permission to spawn once that newer state exists.
return None
# Do not fire the generic phase hooks here. The executor's first
# idempotent RUNNING transition performs projection. Deferring it
# avoids a post-commit hook racing after Stop and projecting the Task
# back to RUNNING from an already-held WorkItem.
return persisted
async def apply_delegation_review_resolution(
self,
work_item_id: str,
@@ -7148,6 +7193,262 @@ class OPCStore:
)
await self._db.commit()
async def get_or_create_active_execution_checkpoint(
self,
checkpoint: ExecutionCheckpoint,
*,
checkpoint_types: list[str] | tuple[str, ...] | set[str],
create_if_missing: bool = True,
) -> tuple[ExecutionCheckpoint | None, bool]:
"""Atomically reuse or create one active checkpoint for a session scope.
``BEGIN IMMEDIATE`` serializes checkpoint creation across independent
controller/store connections. It also repairs historical duplicate
active rows in the same transaction, so concurrent startup/shutdown
reconcilers can never supersede each other's newly-created checkpoint
and leave the scope without a durable recovery point.
"""
assert self._db
clean_types = sorted(
{
str(item).strip()
for item in checkpoint_types
if str(item).strip()
}
)
project_id = str(checkpoint.project_id or "default").strip() or "default"
session_id = str(checkpoint.session_id or "").strip()
checkpoint_type = str(checkpoint.checkpoint_type or "").strip()
if not session_id:
raise ValueError("active execution checkpoint requires session_id")
if checkpoint_type not in clean_types:
raise ValueError(
"checkpoint_type must be included in the active checkpoint scope"
)
placeholders = ", ".join("?" for _ in clean_types)
try:
await self._db.execute("BEGIN IMMEDIATE")
async with self._db.execute(
f"""SELECT * FROM execution_checkpoints
WHERE project_id = ?
AND session_id = ?
AND checkpoint_type IN ({placeholders})
AND status IN ('pending', 'resuming')
ORDER BY updated_at DESC, created_at DESC, checkpoint_id DESC""",
(project_id, session_id, *clean_types),
) as cursor:
rows = await cursor.fetchall()
cols = [description[0] for description in cursor.description]
if rows:
decoded = [dict(zip(cols, row)) for row in rows]
winner_data = decoded[0]
winner_id = str(winner_data["checkpoint_id"])
now = datetime.now().isoformat()
for duplicate in decoded[1:]:
duplicate_id = str(duplicate.get("checkpoint_id", "") or "").strip()
if not duplicate_id:
continue
payload = _json_loads(duplicate.get("payload"), {})
payload["superseded_at"] = now
payload["superseded_by_checkpoint_id"] = winner_id
await self._db.execute(
"""UPDATE execution_checkpoints
SET status = 'superseded', payload = ?, updated_at = ?
WHERE checkpoint_id = ? AND status IN ('pending', 'resuming')""",
(_json_dumps(payload), now, duplicate_id),
)
await self._db.commit()
return (
ExecutionCheckpoint(
checkpoint_id=winner_id,
project_id=str(winner_data["project_id"]),
session_id=winner_data.get("session_id"),
checkpoint_type=str(winner_data["checkpoint_type"]),
status=str(winner_data["status"]),
task_id=winner_data.get("task_id"),
payload=_json_loads(winner_data.get("payload"), {}),
created_at=datetime.fromisoformat(str(winner_data["created_at"])),
updated_at=datetime.fromisoformat(str(winner_data["updated_at"])),
),
False,
)
if not create_if_missing:
await self._db.commit()
return None, False
await self._db.execute(
"""INSERT INTO execution_checkpoints
(checkpoint_id, project_id, session_id, checkpoint_type, status,
task_id, payload, created_at, updated_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
(
checkpoint.checkpoint_id,
project_id,
session_id,
checkpoint_type,
checkpoint.status,
checkpoint.task_id,
_json_dumps(checkpoint.payload),
checkpoint.created_at.isoformat(),
checkpoint.updated_at.isoformat(),
),
)
await self._db.commit()
return checkpoint, True
except Exception:
await self._db.rollback()
raise
async def normalize_active_execution_checkpoints(
self,
*,
project_id: str,
session_id: str,
checkpoint_types: list[str] | tuple[str, ...] | set[str],
) -> ExecutionCheckpoint | None:
"""Return one active scope owner while superseding duplicate rows.
Unlike creation, startup reconciliation must not manufacture a
checkpoint merely because none exists. This wrapper reuses the same
serialized transaction and duplicate winner rule with insertion
explicitly disabled.
"""
clean_types = sorted(
{
str(item).strip()
for item in checkpoint_types
if str(item).strip()
}
)
if not clean_types or not str(session_id or "").strip():
return None
candidate = ExecutionCheckpoint(
project_id=str(project_id or "default").strip() or "default",
session_id=str(session_id or "").strip(),
checkpoint_type=clean_types[0],
)
checkpoint, _created = await self.get_or_create_active_execution_checkpoint(
candidate,
checkpoint_types=clean_types,
create_if_missing=False,
)
return checkpoint
async def compare_and_set_execution_checkpoint(
self,
checkpoint_id: str,
*,
expected_statuses: list[str] | tuple[str, ...] | set[str],
status: str,
payload: dict[str, Any],
updated_at: datetime | None = None,
) -> bool:
"""Atomically transition a checkpoint only from an expected state.
The conditional UPDATE is the cross-controller guard for checkpoint
consumption. In-memory scope locks serialize one Office controller;
this CAS prevents a standalone CLI/controller from claiming the same
pending runtime concurrently.
"""
assert self._db
expected = [
str(item).strip()
for item in expected_statuses
if str(item).strip()
]
if not checkpoint_id or not expected:
return False
placeholders = ", ".join("?" for _ in expected)
cursor = await self._db.execute(
f"""UPDATE execution_checkpoints
SET status = ?, payload = ?, updated_at = ?
WHERE checkpoint_id = ? AND status IN ({placeholders})""",
(
str(status or "").strip(),
_json_dumps(dict(payload or {})),
(updated_at or datetime.now()).isoformat(),
checkpoint_id,
*expected,
),
)
await self._db.commit()
return cursor.rowcount == 1
async def complete_execution_checkpoint_and_reopen_ui_anchor(
self,
checkpoint_id: str,
*,
project_id: str,
session_id: str,
expected_status: str,
status: str,
payload: dict[str, Any],
ui_anchor_task_id: str = "",
updated_at: datetime | None = None,
) -> bool:
"""Atomically complete a runtime handoff and reopen its UI anchor.
The anchor must never become chat-runnable if a concurrent Stop already
took checkpoint ownership back. Keeping both writes in one SQLite
transaction makes the checkpoint CAS the gate for the UI projection.
"""
assert self._db
checkpoint_id = str(checkpoint_id or "").strip()
project_id = str(project_id or "default").strip() or "default"
session_id = str(session_id or "").strip()
expected_status = str(expected_status or "").strip()
status = str(status or "").strip()
ui_anchor_task_id = str(ui_anchor_task_id or "").strip()
if not checkpoint_id or not session_id or not expected_status or not status:
return False
now = updated_at or datetime.now()
try:
await self._db.execute("BEGIN IMMEDIATE")
cursor = await self._db.execute(
"""UPDATE execution_checkpoints
SET status = ?, payload = ?, updated_at = ?
WHERE checkpoint_id = ?
AND project_id = ?
AND session_id = ?
AND status = ?""",
(
status,
_json_dumps(dict(payload or {})),
now.isoformat(),
checkpoint_id,
project_id,
session_id,
expected_status,
),
)
if cursor.rowcount != 1:
await self._db.rollback()
return False
if ui_anchor_task_id:
await self._db.execute(
"""UPDATE tasks
SET status = ?, execution_lock = 0, execution_locked_at = NULL
WHERE id = ? AND project_id = ? AND status = ?""",
(
TaskStatus.IDLE.value,
ui_anchor_task_id,
project_id,
TaskStatus.CANCELLED.value,
),
)
await self._db.commit()
return True
except Exception:
await self._db.rollback()
raise
async def get_execution_checkpoints(
self,
project_id: str = "default",
+1470 -550
View File
File diff suppressed because it is too large Load Diff
+195 -142
View File
@@ -17,6 +17,10 @@ from typing import Any, Awaitable, Callable
from loguru import logger
from opc.core.active_task_runs import (
ActiveTaskRunAdmissionClosed,
ActiveTaskRunRegistry,
)
from opc.core.config import DEFAULT_EXTERNAL_AGENT_STARTUP_TIMEOUT_SECONDS, DEFAULT_ORGANIZATION_ID
from opc.core.models import (
AdaptiveRoleProfile,
@@ -456,6 +460,26 @@ class WorkItemOutputBundle:
summary: str = ""
@dataclass(frozen=True)
class CompanyExecutorDriverOwnership:
"""One registry attempt covering a complete company scheduler run."""
registry: ActiveTaskRunRegistry
project_id: str
task_id: str
attempt_token: str
def bind(self):
return self.registry.bind_driver_attempt(self.attempt_token)
def release(self) -> bool:
return self.registry.unregister(
self.project_id,
self.task_id,
self.attempt_token,
)
def serialize_company_runtime_spec(spec: CompanyRuntimeSpec | None) -> dict[str, Any]:
if spec is None:
return {}
@@ -1354,6 +1378,7 @@ class CompanyWorkItemExecutor:
store: Any | None = None,
llm: Any | None = None,
role_prompt_runner: Callable[[Task, str, dict[str, Any], str, bool], Awaitable[str | None]] | None = None,
active_task_run_registry: ActiveTaskRunRegistry | None = None,
) -> None:
self.org_engine = org_engine
self.communication = communication
@@ -1373,6 +1398,7 @@ class CompanyWorkItemExecutor:
self.on_kanban_changed = on_kanban_changed
self.work_item_timeout = work_item_timeout
self.role_prompt_runner = role_prompt_runner
self.active_task_run_registry = active_task_run_registry
self._default_run_state = CompanyExecutorRunState()
self._run_state_var: ContextVar[CompanyExecutorRunState | None] = ContextVar(
f"company-executor-run-state:{id(self)}",
@@ -3368,12 +3394,6 @@ class CompanyWorkItemExecutor:
"cell_id": work_item.cell_id,
"parent_work_item_id": work_item.parent_work_item_id,
}
if work_item.phase == Phase.PAUSED:
task.metadata.setdefault("interrupted_recovery", {
"reason": "work_item_interrupted",
"detected_at": datetime.now().isoformat(),
})
if task.metadata != before_metadata:
changed = True
return changed
@@ -4359,14 +4379,76 @@ class CompanyWorkItemExecutor:
return "dispatch_required"
return "worker_execute"
async def execute(self, plan: CompanyWorkItemRuntimePlan, tasks: list[Task]) -> str:
plan = _coerce_company_work_item_runtime_plan(plan) or CompanyWorkItemRuntimePlan()
plan.metadata = {
**dict(plan.metadata or {}),
"execution_model": "multi_team_org",
"runtime_model": "multi_team_org",
@staticmethod
def _driver_ownership_task(
tasks: list[Task],
*,
preferred_task_ids: set[str] | None = None,
) -> Task | None:
preferred = {
str(task_id or "").strip()
for task_id in set(preferred_task_ids or set())
if str(task_id or "").strip()
}
return await self._execute_multi_team_org(plan, tasks)
candidates = [
task
for task in tasks
if str(getattr(task, "id", "") or "").strip()
and (not preferred or task.id in preferred)
]
for task in candidates:
if linked_work_item_id_for_task(task):
return task
for task in candidates:
if is_work_item_runtime_metadata(dict(task.metadata or {})):
return task
return candidates[0] if candidates else None
def acquire_driver_ownership(
self,
tasks: list[Task],
*,
preferred_task_ids: set[str] | None = None,
) -> CompanyExecutorDriverOwnership | None:
registry = self.active_task_run_registry
task = self._driver_ownership_task(
tasks,
preferred_task_ids=preferred_task_ids,
)
if registry is None or task is None:
return None
project_id = str(task.project_id or "default").strip() or "default"
try:
attempt_token = registry.register(project_id, task.id)
except ActiveTaskRunAdmissionClosed as exc:
raise asyncio.CancelledError(str(exc)) from exc
return CompanyExecutorDriverOwnership(
registry=registry,
project_id=project_id,
task_id=task.id,
attempt_token=attempt_token,
)
async def execute(
self,
plan: CompanyWorkItemRuntimePlan,
tasks: list[Task],
) -> str:
ownership = self.acquire_driver_ownership(tasks)
try:
plan = _coerce_company_work_item_runtime_plan(plan) or CompanyWorkItemRuntimePlan()
plan.metadata = {
**dict(plan.metadata or {}),
"execution_model": "multi_team_org",
"runtime_model": "multi_team_org",
}
if ownership is None:
return await self._execute_multi_team_org(plan, tasks)
with ownership.bind():
return await self._execute_multi_team_org(plan, tasks)
finally:
if ownership is not None:
ownership.release()
async def _execute_multi_team_org(
self,
@@ -4501,11 +4583,11 @@ class CompanyWorkItemExecutor:
# Claim whatever is immediately claimable and spawn each
# work item as an independent asyncio.Task so the loop no
# longer blocks on the slowest sibling.
claims = await self.runtime.claim_runnable_tasks(tasks, work_items=work_items)
for member_session, claimed_task in claims:
work_item_coro = self._run_claimed_work_item(member_session, claimed_task, {})
work_item_task = asyncio.create_task(work_item_coro)
active_work_item_tasks[work_item_task] = (member_session, claimed_task)
claims = await self._claim_and_create_work_item_tasks(
tasks,
work_items,
active_work_item_tasks,
)
# Termination: only when nothing is in-flight AND nothing
# else is runnable. If work items are still running, even an
# "empty runnable" snapshot may become non-empty within
@@ -4608,10 +4690,6 @@ class CompanyWorkItemExecutor:
self._schedule_kanban_notification()
except asyncio.CancelledError:
claimed_pairs = list(active_work_item_tasks.values())
claimed_tasks = [
claimed_task
for _member_session, claimed_task in claimed_pairs
]
for work_item_task in list(active_work_item_tasks.keys()):
if not work_item_task.done():
work_item_task.cancel()
@@ -4660,63 +4738,6 @@ class CompanyWorkItemExecutor:
)
except Exception:
logger.opt(exception=True).debug("company runtime cancellation: failed session idle reset")
for claimed_task in claimed_tasks:
if claimed_task.status in {TaskStatus.DONE, TaskStatus.FAILED, TaskStatus.CANCELLED}:
continue
claimed_task.metadata = dict(claimed_task.metadata or {})
claimed_task.metadata["company_runtime_suspended_at"] = datetime.now().isoformat()
claimed_task.metadata.setdefault("last_stop_reason", "runtime_cancelled")
claimed_task.metadata["company_runtime_stop_state"] = "suspended"
claimed_task.metadata["company_runtime_stop_marked_at"] = (
claimed_task.metadata.get("company_runtime_stop_marked_at") or datetime.now().isoformat()
)
claimed_task.metadata.setdefault(
"suspended_task_status",
claimed_task.status.value if isinstance(claimed_task.status, TaskStatus) else str(claimed_task.status or ""),
)
work_item_id = linked_work_item_id_for_task(claimed_task)
try:
if work_item_id and self._store_is_ready(self.store) and hasattr(self.store, "get_delegation_work_item"):
work_item = await self.store.get_delegation_work_item(work_item_id)
else:
work_item = None
if work_item is not None and getattr(work_item, "phase", None) not in {Phase.APPROVED, Phase.FAILED, Phase.CANCELLED}:
phase = getattr(work_item, "phase", Phase.RUNNING)
phase_value = phase.value if isinstance(phase, Phase) else str(phase or "")
original_claim = {
"claimed_by_role_runtime_session_id": str(getattr(work_item, "claimed_by_role_runtime_session_id", "") or ""),
"claimed_by_seat_id": str(getattr(work_item, "claimed_by_seat_id", "") or ""),
"claimed_by_role_session_id": str((getattr(work_item, "metadata", {}) or {}).get("claimed_by_role_session_id", "") or ""),
"claimed_task_id": str((getattr(work_item, "metadata", {}) or {}).get("claimed_task_id", "") or claimed_task.id),
}
await self.store.update_delegation_work_item(
work_item_id,
metadata_updates={
"dispatch_hold": "company_runtime_suspended",
"suspended_at": datetime.now().isoformat(),
"suspend_reason": claimed_task.metadata.get("last_stop_reason", "runtime_cancelled"),
"suspended_phase": phase_value,
"suspended_task_status": claimed_task.metadata.get("suspended_task_status", ""),
"suspended_claim": original_claim,
"claimed_by_role_session_id": "",
"claimed_task_id": "",
},
claimed_by_role_runtime_session_id="",
claimed_by_seat_id="",
)
claimed_task.metadata["dispatch_hold"] = "company_runtime_suspended"
claimed_task.metadata["suspended_phase"] = phase_value
except Exception:
logger.opt(exception=True).debug(
"company runtime cancellation: failed suspend hold release",
)
if self.save_task and self._store_is_ready(self.store):
try:
await self.save_task(claimed_task)
except Exception:
logger.opt(exception=True).debug(
"company runtime cancellation: failed suspended task save",
)
raise
finally:
# Drain any work items still running (shouldn't happen given the
@@ -4739,6 +4760,97 @@ class CompanyWorkItemExecutor:
active_work_item_tasks.clear()
return self._summarize_multi_team_org_results(tasks)
@staticmethod
def _runtime_scope_for_tasks(tasks: list[Task]) -> tuple[str, str]:
project_id = "default"
runtime_session_id = ""
for task in tasks:
project_id = str(task.project_id or project_id).strip() or "default"
metadata = dict(task.metadata or {})
runtime_session_id = str(
getattr(task, "parent_session_id", "")
or metadata.get("company_runtime_root_session_id")
or metadata.get("parent_session_id")
or ""
).strip()
if runtime_session_id:
return project_id, runtime_session_id
for task in tasks:
runtime_session_id = str(getattr(task, "session_id", "") or "").strip()
if runtime_session_id:
return project_id, runtime_session_id
return project_id, ""
async def _claim_and_create_work_item_tasks(
self,
tasks: list[Task],
work_items: list[DelegationWorkItem],
active_work_item_tasks: dict[
asyncio.Task[TaskResult | None],
tuple[CompanyMemberSession, Task],
],
) -> list[tuple[CompanyMemberSession, Task]]:
"""Keep durable claim and coroutine ownership in one scope boundary."""
async def claim_and_create() -> list[tuple[CompanyMemberSession, Task]]:
claims = await self.runtime.claim_runnable_tasks(
tasks,
work_items=work_items,
)
for member_session, claimed_task in claims:
work_item_task = self._create_claimed_work_item_task(
member_session,
claimed_task,
{},
)
active_work_item_tasks[work_item_task] = (
member_session,
claimed_task,
)
return claims
registry = self.active_task_run_registry
project_id, runtime_session_id = self._runtime_scope_for_tasks(tasks)
if registry is None or not runtime_session_id:
return await claim_and_create()
async with registry.scope_lock(project_id, runtime_session_id):
return await claim_and_create()
def _create_claimed_work_item_task(
self,
member_session: CompanyMemberSession,
task: Task,
task_by_projection_id: dict[str, Task],
) -> asyncio.Task[TaskResult | None]:
"""Register ownership before scheduling the full claimed-item coroutine."""
registry = self.active_task_run_registry
project_id = str(task.project_id or "default").strip() or "default"
attempt_token = ""
if registry is not None:
try:
attempt_token = registry.register(project_id, task.id)
except ActiveTaskRunAdmissionClosed as exc:
raise asyncio.CancelledError(str(exc)) from exc
async def run_owned() -> TaskResult | None:
try:
return await self._run_claimed_work_item(
member_session,
task,
task_by_projection_id,
)
finally:
if registry is not None and attempt_token:
registry.unregister(project_id, task.id, attempt_token)
try:
return asyncio.create_task(run_owned())
except BaseException:
if registry is not None and attempt_token:
registry.unregister(project_id, task.id, attempt_token)
raise
async def _run_claimed_work_item(
self,
member_session: CompanyMemberSession,
@@ -5298,69 +5410,10 @@ class CompanyWorkItemExecutor:
timeout=self.work_item_timeout,
)
except asyncio.CancelledError:
task.metadata = dict(task.metadata or {})
task.metadata["company_runtime_suspended_at"] = datetime.now().isoformat()
task.metadata.setdefault("last_stop_reason", "runtime_cancelled")
task.metadata["company_runtime_stop_state"] = "suspended"
task.metadata["company_runtime_stop_marked_at"] = (
task.metadata.get("company_runtime_stop_marked_at") or datetime.now().isoformat()
)
task.metadata.setdefault(
"suspended_task_status",
task.status.value if isinstance(task.status, TaskStatus) else str(task.status or ""),
)
work_item_id = linked_work_item_id_for_task(task)
store_ready = self._store_is_ready(self.store)
if work_item_id and self.store and store_ready:
task.metadata.pop("progress_log", None)
await append_work_item_progress(
self.store,
work_item_id,
"Work item suspended by runtime cancellation.",
)
else:
progress = list(task.metadata.get("progress_log", []) or [])
progress.append("Work item suspended by runtime cancellation.")
task.metadata["progress_log"] = progress[-20:]
try:
work_item = (
await self.store.get_delegation_work_item(work_item_id)
if work_item_id and store_ready and hasattr(self.store, "get_delegation_work_item")
else None
)
if work_item is not None and getattr(work_item, "phase", None) not in {Phase.APPROVED, Phase.FAILED, Phase.CANCELLED}:
phase = getattr(work_item, "phase", Phase.RUNNING)
phase_value = phase.value if isinstance(phase, Phase) else str(phase or "")
original_claim = {
"claimed_by_role_runtime_session_id": str(getattr(work_item, "claimed_by_role_runtime_session_id", "") or ""),
"claimed_by_seat_id": str(getattr(work_item, "claimed_by_seat_id", "") or ""),
"claimed_by_role_session_id": str((getattr(work_item, "metadata", {}) or {}).get("claimed_by_role_session_id", "") or ""),
"claimed_task_id": str((getattr(work_item, "metadata", {}) or {}).get("claimed_task_id", "") or task.id),
}
await self.store.update_delegation_work_item(
work_item_id,
metadata_updates={
"dispatch_hold": "company_runtime_suspended",
"suspended_at": datetime.now().isoformat(),
"suspend_reason": task.metadata.get("last_stop_reason", "runtime_cancelled"),
"suspended_phase": phase_value,
"suspended_task_status": task.metadata.get("suspended_task_status", ""),
"suspended_claim": original_claim,
"claimed_by_role_session_id": "",
"claimed_task_id": "",
},
claimed_by_role_runtime_session_id="",
claimed_by_seat_id="",
)
task.metadata["dispatch_hold"] = "company_runtime_suspended"
task.metadata["suspended_phase"] = phase_value
task.status = task_status_for_phase(phase) if isinstance(phase, Phase) else task.status
except Exception:
logger.opt(exception=True).debug(
"company runtime cancellation: failed to apply suspend hold",
)
if self.save_task and self._store_is_ready(self.store):
await self.save_task(task)
# Suspension is a checkpoint transition owned by OPCEngine.
# This task object may be stale by the time cancellation is
# observed, so persisting it here can erase the canonical
# checkpoint type, stop intent, or WorkItem hold.
raise
except asyncio.TimeoutError:
logger.error(f"Company work item {projection_id} timed out after {self.work_item_timeout}s")
+73 -35
View File
@@ -1373,7 +1373,6 @@ class CompanyRuntime:
_skip("no task materialized for work_item this tick",
session=session_label, work_item_id=work_item_id)
continue
self._claimed_work_item_ids.add(work_item_id)
else:
task_id = queued_item_id
self._queued_task_ids.discard(task_id)
@@ -1392,6 +1391,30 @@ class CompanyRuntime:
status=getattr(task, "status", None))
continue
self._claimed_task_ids.add(task_id)
if work_item is not None:
claimed = await self._claim_role_session_work_item(
session,
work_item,
task,
)
if not claimed:
fresh_work_item = None
get_work_item = getattr(
self.store,
"get_delegation_work_item",
None,
)
if callable(get_work_item):
fresh_work_item = await get_work_item(work_item_id)
if fresh_work_item is not None:
work_item_map[work_item_id] = fresh_work_item
_skip(
"atomic WorkItem claim lost to a phase/hold/owner update",
session=session_label,
work_item_id=work_item_id,
)
continue
self._claimed_work_item_ids.add(work_item_id)
if can_soft_wake and (
bool((task.metadata or {}).get("review_task", False))
or bool((task.metadata or {}).get("review_execution_work_item", False))
@@ -1412,7 +1435,6 @@ class CompanyRuntime:
self.prepare_task_for_session(session, task)
await self._sync_current_turn_mode_to_work_item(task, session.current_turn_mode)
if work_item is not None:
await self._claim_role_session_work_item(session, work_item, task)
# #7: mirror the claim onto the in-memory work_item so
# subsequent iterations in the same claim pass see it
# and is_dispatchable returns False (race safety after
@@ -1908,21 +1930,65 @@ class CompanyRuntime:
self.role_sessions[role_session_id] = role_session
return role_session
async def _claim_role_session_work_item(self, session: CompanyMemberSession, work_item: Any, task: Task) -> None:
async def _claim_role_session_work_item(
self,
session: CompanyMemberSession,
work_item: Any,
task: Task,
) -> bool:
"""Atomically claim ``work_item`` for the role-instance behind
``session``.
In the role-instance model the claim identity is
``role_runtime_session_id``. Seat / manager-seat columns are
still written for org-chart lookups but they are NOT part of
the claim key — only the role session is.
the claim key — only the role session is. Pure in-memory runtimes have
no durable race to arbitrate and keep the same local claim semantics.
"""
work_item_id = str(getattr(work_item, "work_item_id", "") or "").strip()
if not work_item_id:
return
return False
role_session = self._ensure_role_session(task)
if role_session is None:
return
return False
work_item_revision = 0
try:
work_item_revision = int((getattr(work_item, "metadata", {}) or {}).get("manager_mutation_revision") or 0)
except (TypeError, ValueError):
work_item_revision = 0
store_ready = self.store is not None and bool(
getattr(self.store, "is_ready", False)
)
claim = (
getattr(self.store, "claim_delegation_work_item_if_dispatchable", None)
if store_ready
else None
)
if store_ready:
# A durable runtime must win the store CAS before it mutates any
# in-memory scheduling state. This is the Stop/shutdown race
# boundary: a missing CAS API is a failed claim, not permission to
# fall back to the former best-effort update path.
if not callable(claim):
return False
persisted = await claim(
work_item_id,
expected_phase=getattr(work_item, "phase", Phase.READY),
role_runtime_session_id=role_session.role_session_id,
seat_id=str(getattr(session, "seat_id", "") or "").strip(),
task_id=task.id,
work_item_revision=work_item_revision,
)
if persisted is None:
return False
work_item.phase = persisted.phase
work_item.role_runtime_session_id = persisted.role_runtime_session_id
work_item.claimed_by_role_runtime_session_id = (
persisted.claimed_by_role_runtime_session_id
)
work_item.claimed_by_seat_id = persisted.claimed_by_seat_id
work_item.metadata = dict(persisted.metadata or {})
ready_background_ids = [
item_id
for item_id in list(role_session.background_work_item_ids or [])
@@ -1937,39 +2003,11 @@ class CompanyRuntime:
session.background_work_item_ids = list(role_session.background_work_item_ids)
task.metadata = dict(task.metadata)
task.metadata["delegation_role_session_id"] = role_session.role_session_id
work_item_revision = 0
try:
work_item_revision = int((getattr(work_item, "metadata", {}) or {}).get("manager_mutation_revision") or 0)
except (TypeError, ValueError):
work_item_revision = 0
task.metadata["started_work_item_revision"] = work_item_revision
task.metadata["claimed_work_item_revision"] = work_item_revision
if self.store and bool(getattr(self.store, "is_ready", False)) and hasattr(self.store, "update_delegation_work_item"):
# Do not regress a work item that is already in a review
# phase (AWAITING_MANAGER_REVIEW / AWAITING_HUMAN) back to
# RUNNING: the DB phase validator rejects that transition
# and the error bubbles up to the session loop. This
# happens when the reactivation sweeper wakes a task whose
# work item has already been promoted to review — treat
# the claim as "refresh the task/role-session bindings
# only" and leave the phase alone.
current_phase = getattr(work_item, "phase", None)
phase_to_write: Phase | None = Phase.RUNNING
if current_phase in IN_REVIEW_PHASES:
phase_to_write = None
await self.store.update_delegation_work_item(
work_item_id,
phase=phase_to_write,
role_runtime_session_id=role_session.role_session_id,
claimed_by_role_runtime_session_id=role_session.role_session_id,
metadata_updates={
"claimed_by_role_session_id": role_session.role_session_id,
"claimed_task_id": task.id,
"claimed_work_item_revision": work_item_revision,
},
)
if self.store and bool(getattr(self.store, "is_ready", False)) and hasattr(self.store, "save_delegation_role_session"):
await self.store.save_delegation_role_session(role_session)
return True
def ensure_role_instance_session(
self, task: Task
@@ -0,0 +1,391 @@
"""Canonical company-runtime identity derived from durable records.
Company-mode Tasks are execution envelopes, not the identity of a run. A
runtime is owned by its root session and an active suspend checkpoint. This
module deliberately has no UI dependencies so every surface can resolve the
same scope without relying on process-local task maps.
"""
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any, Iterable
from opc.layer2_organization.work_item_links import linked_work_item_id_for_task
from opc.layer2_organization.work_item_runtime import is_work_item_runtime_metadata
COMPANY_RUNTIME_CHECKPOINT_TYPES: frozenset[str] = frozenset({
"company_runtime_suspended",
"company_runtime_interrupted",
})
ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES: frozenset[str] = frozenset({
"pending",
"resuming",
})
def _text(value: Any) -> str:
return str(value or "").strip()
def _metadata(task: Any) -> dict[str, Any]:
return dict(getattr(task, "metadata", {}) or {})
def _task_id(task: Any) -> str:
return _text(getattr(task, "id", ""))
def _task_session_id(task: Any) -> str:
return _text(getattr(task, "session_id", ""))
def _task_parent_session_id(task: Any) -> str:
metadata = _metadata(task)
return _text(
getattr(task, "parent_session_id", "")
or metadata.get("company_runtime_root_session_id")
or metadata.get("parent_session_id")
)
def _has_company_runtime_marker(task: Any) -> bool:
metadata = _metadata(task)
exec_mode = _text(metadata.get("exec_mode")).lower()
mode = _text(metadata.get("mode")).lower()
execution_mode = _text(metadata.get("execution_mode")).lower()
if exec_mode in {"company", "org", "custom"} or mode in {"company", "org", "custom"}:
return True
if execution_mode in {"company", "company_mode", "multi_team_org"}:
return True
if is_work_item_runtime_metadata(metadata):
return True
if linked_work_item_id_for_task(task):
return True
if any(
metadata.get(key) not in (None, "", [], {})
for key in (
"company_work_item_plan",
"company_runtime_root_session_id",
"delegation_run_id",
"work_item_projection_id",
"work_item_projection_ref",
"work_item_role_id",
"shared_role_session",
)
):
return True
# Old company records may predate exec_mode. An explicit task-mode marker
# wins over the legacy profile hint.
explicitly_task_mode = (
exec_mode in {"task", "project", "single"}
or mode == "task"
or execution_mode in {"task", "task_mode", "project"}
or _text(metadata.get("task_mode_contract")) == "single_full_capability_main_agent"
)
return not explicitly_task_mode and bool(_text(metadata.get("company_profile")))
def is_company_runtime_task(task: Any) -> bool:
"""Return whether durable Task metadata identifies company-owned work."""
return _has_company_runtime_marker(task)
def is_pure_company_ui_anchor(task: Any, runtime_session_id: str) -> bool:
"""Return whether *task* is the user-facing container for a runtime.
A shared final-decider Task can have the same ``session_id`` as the UI
anchor. Work-item, role, or parent links therefore disqualify a Task even
when its session id is an exact match.
"""
session_id = _text(runtime_session_id)
if not session_id or _task_session_id(task) != session_id:
return False
if _text(getattr(task, "parent_session_id", "")) or _text(getattr(task, "parent_id", "")):
return False
if linked_work_item_id_for_task(task):
return False
metadata = _metadata(task)
return not any(
metadata.get(key) not in (None, "", [], {}, False)
for key in (
"work_item_runtime",
"work_item_projection_id",
"work_item_projection_ref",
"work_item_id",
"work_item_role_id",
"delegation_role_session_id",
"shared_role_session",
"shared_role_id",
"company_runtime_root_session_id",
)
)
def _created_sort_key(value: Any) -> tuple[float, str]:
created_at = getattr(value, "created_at", None)
if isinstance(created_at, datetime):
timestamp = created_at.timestamp()
elif hasattr(created_at, "timestamp"):
try:
timestamp = float(created_at.timestamp())
except Exception:
timestamp = 0.0
else:
timestamp = 0.0
return timestamp, _task_id(value)
def _checkpoint_sort_key(checkpoint: Any) -> tuple[float, float, str]:
def _timestamp(value: Any) -> float:
if isinstance(value, datetime):
return value.timestamp()
if hasattr(value, "timestamp"):
try:
return float(value.timestamp())
except Exception:
return 0.0
return 0.0
return (
_timestamp(getattr(checkpoint, "updated_at", None)),
_timestamp(getattr(checkpoint, "created_at", None)),
_text(getattr(checkpoint, "checkpoint_id", "")),
)
def _checkpoint_runtime_session_id(checkpoint: Any) -> str:
payload = dict(getattr(checkpoint, "payload", {}) or {})
return _text(
getattr(checkpoint, "session_id", "")
or payload.get("parent_session_id")
or payload.get("session_id")
)
@dataclass(frozen=True)
class CompanyRuntimeIdentity:
"""Resolved identity for one company runtime scope."""
project_id: str
runtime_session_id: str
runtime_task_ids: tuple[str, ...]
ui_anchor_task_id: str = ""
config_source_task_id: str = ""
pending_checkpoint_id: str = ""
pending_checkpoint_type: str = ""
pending_checkpoint_status: str = ""
resumable: bool = False
checkpoint: Any | None = field(default=None, repr=False, compare=False)
class CompanyRuntimeIdentityIndex:
"""Session-first index over preloaded Tasks and checkpoints."""
def __init__(self, tasks: Iterable[Any], checkpoints: Iterable[Any] = ()) -> None:
self.tasks = tuple(tasks or ())
self.checkpoints = tuple(checkpoints or ())
self.tasks_by_id = {
_task_id(task): task
for task in self.tasks
if _task_id(task)
}
self.checkpoints_by_id = {
_text(getattr(checkpoint, "checkpoint_id", "")): checkpoint
for checkpoint in self.checkpoints
if _text(getattr(checkpoint, "checkpoint_id", ""))
}
self._identities_by_session = self._build_identities()
self._runtime_session_by_task_id: dict[str, str] = {}
runtime_sessions_by_task_session_id: dict[str, set[str]] = {}
for runtime_session_id, identity in self._identities_by_session.items():
for task_id in identity.runtime_task_ids:
self._runtime_session_by_task_id[task_id] = runtime_session_id
task_session_id = _task_session_id(self.tasks_by_id.get(task_id))
if task_session_id:
runtime_sessions_by_task_session_id.setdefault(
task_session_id,
set(),
).add(runtime_session_id)
self._runtime_session_by_task_session_id = {
task_session_id: next(iter(runtime_session_ids))
for task_session_id, runtime_session_ids in runtime_sessions_by_task_session_id.items()
if len(runtime_session_ids) == 1
}
@property
def identities(self) -> tuple[CompanyRuntimeIdentity, ...]:
return tuple(self._identities_by_session.values())
def task(self, task_id: str) -> Any | None:
return self.tasks_by_id.get(_text(task_id))
def resolve(
self,
*,
task_id: str = "",
task_session_id: str = "",
runtime_session_id: str = "",
checkpoint_id: str = "",
) -> CompanyRuntimeIdentity | None:
requested_task_id = _text(task_id)
requested_task_session_id = _text(task_session_id)
requested_session_id = _text(runtime_session_id)
requested_checkpoint_id = _text(checkpoint_id)
task_scope_id = self._runtime_session_by_task_id.get(requested_task_id, "")
session_scope_id = self._runtime_session_by_task_session_id.get(
requested_task_session_id,
"",
)
checkpoint = self.checkpoints_by_id.get(requested_checkpoint_id) if requested_checkpoint_id else None
checkpoint_session_id = _checkpoint_runtime_session_id(checkpoint) if checkpoint is not None else ""
candidates = {
value
for value in (
requested_session_id,
task_scope_id,
session_scope_id,
checkpoint_session_id,
)
if value
}
if requested_task_session_id and not session_scope_id:
return None
if len(candidates) != 1:
return None
resolved_session_id = next(iter(candidates))
identity = self._identities_by_session.get(resolved_session_id)
if identity is None:
return None
if requested_task_id and requested_task_id not in identity.runtime_task_ids:
return None
if requested_checkpoint_id and requested_checkpoint_id != identity.pending_checkpoint_id:
return None
return identity
def _build_identities(self) -> dict[str, CompanyRuntimeIdentity]:
active_checkpoints_by_session: dict[str, list[Any]] = {}
for checkpoint in self.checkpoints:
checkpoint_type = _text(getattr(checkpoint, "checkpoint_type", ""))
checkpoint_status = _text(getattr(checkpoint, "status", "")).lower()
if (
checkpoint_type not in COMPANY_RUNTIME_CHECKPOINT_TYPES
or checkpoint_status not in ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES
):
continue
runtime_session_id = _checkpoint_runtime_session_id(checkpoint)
if runtime_session_id:
active_checkpoints_by_session.setdefault(runtime_session_id, []).append(checkpoint)
known_sessions = set(active_checkpoints_by_session)
for task in self.tasks:
if not _has_company_runtime_marker(task):
continue
runtime_session_id = _task_parent_session_id(task) or _task_session_id(task)
if runtime_session_id:
known_sessions.add(runtime_session_id)
tasks_by_session: dict[str, list[Any]] = {session_id: [] for session_id in known_sessions}
for task in self.tasks:
task_id = _task_id(task)
if not task_id:
continue
parent_session_id = _task_parent_session_id(task)
own_session_id = _task_session_id(task)
runtime_session_id = parent_session_id or own_session_id
if runtime_session_id not in known_sessions:
continue
if not (
_has_company_runtime_marker(task)
or runtime_session_id in active_checkpoints_by_session
or is_pure_company_ui_anchor(task, runtime_session_id)
):
continue
tasks_by_session.setdefault(runtime_session_id, []).append(task)
identities: dict[str, CompanyRuntimeIdentity] = {}
for runtime_session_id in sorted(known_sessions):
group = sorted(tasks_by_session.get(runtime_session_id, []), key=_created_sort_key)
anchor = next(
(task for task in group if is_pure_company_ui_anchor(task, runtime_session_id)),
None,
)
def _has_runtime_config(task: Any) -> bool:
metadata = _metadata(task)
return any(
metadata.get(key) not in (None, "", [], {})
for key in (
"exec_mode",
"mode",
"company_profile",
"org_id",
"organization_id",
"preferred_agent",
"selected_execution_agent",
)
)
config_source = (
anchor if anchor is not None and _has_runtime_config(anchor) else None
) or next(
(
task for task in group
if _has_runtime_config(task)
),
anchor or (group[0] if group else None),
)
checkpoint_candidates = active_checkpoints_by_session.get(runtime_session_id, [])
checkpoint = max(checkpoint_candidates, key=_checkpoint_sort_key) if checkpoint_candidates else None
checkpoint_status = _text(getattr(checkpoint, "status", "")).lower() if checkpoint is not None else ""
project_id = _text(
getattr(checkpoint, "project_id", "") if checkpoint is not None else ""
) or _text(getattr(config_source, "project_id", "") if config_source is not None else "") or "default"
identities[runtime_session_id] = CompanyRuntimeIdentity(
project_id=project_id,
runtime_session_id=runtime_session_id,
runtime_task_ids=tuple(_task_id(task) for task in group if _task_id(task)),
ui_anchor_task_id=_task_id(anchor) if anchor is not None else "",
config_source_task_id=_task_id(config_source) if config_source is not None else "",
pending_checkpoint_id=_text(getattr(checkpoint, "checkpoint_id", "")) if checkpoint is not None else "",
pending_checkpoint_type=_text(getattr(checkpoint, "checkpoint_type", "")) if checkpoint is not None else "",
pending_checkpoint_status=checkpoint_status,
resumable=checkpoint_status == "pending",
checkpoint=checkpoint,
)
return identities
def build_company_runtime_identity_index(
tasks: Iterable[Any],
checkpoints: Iterable[Any] = (),
) -> CompanyRuntimeIdentityIndex:
return CompanyRuntimeIdentityIndex(tasks, checkpoints)
async def load_company_runtime_identity_index(
store: Any,
project_id: str,
) -> CompanyRuntimeIdentityIndex:
"""Load durable records once and build the canonical runtime index."""
tasks = await store.get_tasks(project_id=project_id)
checkpoint_getter = getattr(store, "get_execution_checkpoints", None)
if callable(checkpoint_getter):
checkpoints = await checkpoint_getter(
project_id=project_id,
checkpoint_types=sorted(COMPANY_RUNTIME_CHECKPOINT_TYPES),
statuses=sorted(ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES),
)
else:
checkpoint_getter = getattr(store, "get_pending_checkpoints", None)
checkpoints = await checkpoint_getter(
project_id=project_id,
checkpoint_types=sorted(COMPANY_RUNTIME_CHECKPOINT_TYPES),
) if callable(checkpoint_getter) else []
return build_company_runtime_identity_index(tasks, checkpoints)
@@ -75,6 +75,8 @@ class CustomRuntimeRunner:
store=shared_store,
owns_store=shared_store is None,
run_startup_reconcile=shared_store is None,
active_task_run_registry=getattr(self.parent, "_active_task_run_registry", None),
owns_active_task_run_registry=False,
on_progress=self.parent.on_progress,
on_runtime_event=self.parent.on_runtime_event,
on_escalation=self.parent.on_escalation,
+4 -2
View File
@@ -144,8 +144,10 @@ class EngineSeatExecutor:
member_session: CompanyMemberSession | None = None,
) -> None:
_ = member_session
if hasattr(self.host, "_active_task_runs"):
self.host._active_task_runs.discard(task.id)
# The task coroutine owns its registry attempt token and removes it in
# ``finally``. Interrupt requests must not make a still-running
# coroutine appear inactive.
return None
async def shutdown(
self,
-297
View File
@@ -1,297 +0,0 @@
"""CLI-board-specific company runtime recovery manager.
Independent from office_ui/recovery_manager.py — same Core Engine APIs,
different notification path (TUI event bridge instead of WebSocket broadcast).
"""
from __future__ import annotations
import asyncio
import logging
import time
from dataclasses import dataclass, field
from datetime import datetime
from typing import TYPE_CHECKING, Any
from opc.layer2_organization.work_item_identity import work_item_projection_id_from_metadata
from opc.layer2_organization.work_item_transition import apply_task_status_transition
if TYPE_CHECKING:
from .engine_facade import EngineFacade
logger = logging.getLogger(__name__)
@dataclass
class RecoverableWorkItem:
projection_id: str
title: str
task_id: str
status: str
interrupted: bool
previous_status: str = ""
@dataclass
class InterruptedCompanyRuntime:
parent_session_id: str
parent_task_id: str
project_id: str
title: str
profile: str
interrupted_at: str
work_items: list[RecoverableWorkItem] = field(default_factory=list)
@dataclass
class RecoveryStatus:
interrupted: list[InterruptedCompanyRuntime] = field(default_factory=list)
active_recoveries: list[str] = field(default_factory=list)
scanned_at: float = 0.0
def _is_interrupted(task: Any) -> bool:
from opc.core.models import TaskStatus
if task.status != TaskStatus.FAILED:
return False
meta = getattr(task, "metadata", {}) or {}
if meta.get("interrupted_recovery"):
return True
result = getattr(task, "result", {}) or {}
artifacts = result.get("artifacts", {}) or {}
return bool(artifacts.get("interrupted"))
class CliRecoveryManager:
"""Scan for interrupted company runtimes and provide resume/cancel."""
_CACHE_TTL = 10.0
def __init__(self, facade: EngineFacade) -> None:
self._facade = facade
self._lock = asyncio.Lock()
self._active: dict[str, asyncio.Task[Any]] = {}
self._cached: RecoveryStatus | None = None
self._cache_until: float = 0.0
@property
def _project_id(self) -> str:
return self._facade.project_id or "default"
async def get_status(self) -> RecoveryStatus:
now = time.time()
if self._cached is not None and now < self._cache_until:
self._cached.active_recoveries = list(self._active.keys())
return self._cached
status = await self.scan()
self._cached = status
self._cache_until = now + self._CACHE_TTL
return status
async def scan(self) -> RecoveryStatus:
engine = await self._facade.ensure_ready()
if not engine.store:
return RecoveryStatus()
try:
all_tasks = await engine.store.get_tasks(project_id=self._project_id)
except Exception as exc:
logger.warning("Recovery scan failed: %s", exc)
return RecoveryStatus()
groups: dict[str, list[Any]] = {}
tasks_by_session: dict[str, Any] = {}
for task in all_tasks:
sid = str(getattr(task, "session_id", "") or "").strip()
if sid:
tasks_by_session[sid] = task
parent_sid = str(getattr(task, "parent_session_id", "") or "").strip()
projection_id = work_item_projection_id_from_metadata(getattr(task, "metadata", {}) or {})
if parent_sid and projection_id:
groups.setdefault(parent_sid, []).append(task)
from opc.core.models import TaskStatus
interrupted: list[InterruptedCompanyRuntime] = []
for parent_sid, tasks in groups.items():
if not any(_is_interrupted(t) for t in tasks):
continue
non_terminal = [t for t in tasks if t.status not in (TaskStatus.DONE, TaskStatus.CANCELLED)]
if not non_terminal:
continue
parent_task = tasks_by_session.get(parent_sid)
parent_task_id = parent_task.id if parent_task else parent_sid
title = parent_task.title if parent_task else "Unknown company runtime"
work_items: list[RecoverableWorkItem] = []
earliest = ""
for t in sorted(tasks, key=lambda x: (x.created_at, x.id)):
meta = dict(getattr(t, "metadata", {}) or {})
rmeta = meta.get("interrupted_recovery", {})
is_int = _is_interrupted(t)
if is_int and rmeta.get("detected_at", ""):
det = rmeta["detected_at"]
if not earliest or det < earliest:
earliest = det
work_items.append(RecoverableWorkItem(
projection_id=work_item_projection_id_from_metadata(meta, fallback=t.id),
title=t.title,
task_id=t.id,
status=t.status.value if hasattr(t.status, "value") else str(t.status),
interrupted=is_int,
previous_status=rmeta.get("previous_status", ""),
))
profile = ""
for t in tasks:
p = (getattr(t, "metadata", {}) or {}).get("company_profile", "")
if p:
profile = p
break
interrupted.append(InterruptedCompanyRuntime(
parent_session_id=parent_sid,
parent_task_id=parent_task_id,
project_id=self._project_id,
title=title,
profile=profile,
interrupted_at=earliest or datetime.now().isoformat(),
work_items=work_items,
))
return RecoveryStatus(
interrupted=interrupted,
active_recoveries=list(self._active.keys()),
scanned_at=time.time(),
)
async def resume(self, parent_task_id: str) -> dict[str, Any]:
async with self._lock:
if parent_task_id in self._active:
return {"ok": False, "error": "already_in_progress"}
status = await self.scan()
wf = next((w for w in status.interrupted if w.parent_task_id == parent_task_id), None)
if not wf:
return {"ok": False, "error": "not_found"}
engine = await self._facade.ensure_ready()
snapshot = await engine._load_company_runtime_snapshot(wf.parent_session_id)
if not snapshot:
return {"ok": False, "error": "snapshot_unavailable"}
plan, tasks = snapshot
await self._clean_checkpoints(wf, tasks)
from opc.core.models import TaskStatus
resumed_ids: list[str] = []
for task in tasks:
if task.status == TaskStatus.DONE:
continue
if task.status in (TaskStatus.FAILED, TaskStatus.BLOCKED):
task.result = None
task.execution_lock = False
task.execution_locked_at = None
meta = dict(task.metadata)
meta.pop("interrupted_recovery", None)
progress = list(meta.get("progress_log", []))
progress.append(f"[Recovery] Resumed at {datetime.now().isoformat()}")
meta["progress_log"] = progress[-20:]
task.metadata = meta
try:
await apply_task_status_transition(
engine.store,
task,
target_status_or_phase=TaskStatus.PENDING,
reason="cli_recovery_resume",
release_claim=True,
)
except Exception as exc:
logger.warning("Recovery resume skipped %s: %s", task.id, exc)
continue
if task.status != TaskStatus.PENDING:
logger.warning("Recovery resume preserved non-runnable phase for %s", task.id)
continue
await engine.store.save_task(task)
resumed_ids.append(work_item_projection_id_from_metadata(meta, fallback=task.id))
if not resumed_ids:
return {"ok": False, "error": "no_work_items_to_resume"}
self._cache_until = 0.0
bg = asyncio.create_task(self._execute(parent_task_id, plan, tasks))
self._active[parent_task_id] = bg
return {"ok": True, "resumed_work_item_projection_ids": resumed_ids}
async def cancel(self, parent_task_id: str) -> dict[str, Any]:
async with self._lock:
bg = self._active.pop(parent_task_id, None)
if bg and not bg.done():
bg.cancel()
status = await self.scan()
wf = next((w for w in status.interrupted if w.parent_task_id == parent_task_id), None)
if not wf:
return {"ok": False, "error": "not_found"}
engine = await self._facade.ensure_ready()
snapshot = await engine._load_company_runtime_snapshot(wf.parent_session_id)
if not snapshot:
return {"ok": False, "error": "snapshot_unavailable"}
_, tasks = snapshot
from opc.core.models import TaskStatus
cancelled = 0
for task in tasks:
if task.status not in (TaskStatus.DONE, TaskStatus.CANCELLED):
try:
await apply_task_status_transition(
engine.store,
task,
target_status_or_phase=TaskStatus.CANCELLED,
reason="cli_recovery_cancel",
release_claim=True,
)
except Exception as exc:
logger.warning("Recovery cancel skipped %s: %s", task.id, exc)
continue
if task.status != TaskStatus.CANCELLED:
logger.warning("Recovery cancel preserved non-cancelled phase for %s", task.id)
continue
cancelled += 1
await self._clean_checkpoints(wf, tasks)
self._cache_until = 0.0
return {"ok": True, "cancelled_count": cancelled}
async def _execute(self, parent_task_id: str, plan: Any, tasks: list[Any]) -> None:
try:
engine = await self._facade.ensure_ready()
executor = engine.company_executor
if not executor:
raise RuntimeError("company_executor not available")
await executor.execute(plan, tasks)
except asyncio.CancelledError:
pass
except Exception as exc:
logger.warning("Recovery execution failed for %s: %s", parent_task_id, exc)
finally:
self._active.pop(parent_task_id, None)
self._cache_until = 0.0
async def _clean_checkpoints(self, wf: InterruptedCompanyRuntime, tasks: list[Any]) -> None:
engine = await self._facade.ensure_ready()
if not engine.store:
return
session_ids = {str(getattr(t, "session_id", "") or "").strip() for t in tasks}
session_ids.add(wf.parent_session_id)
session_ids.discard("")
try:
pending = await engine.store.get_pending_checkpoints(project_id=wf.project_id)
for cp in pending:
if str(cp.session_id or "").strip() in session_ids:
await engine.store.resolve_execution_checkpoint(cp.checkpoint_id, status="cancelled")
except Exception as exc:
logger.debug("Checkpoint cleanup error: %s", exc)
-39
View File
@@ -15,7 +15,6 @@ from opc.plugins.cli_board.state.store import BoardStateStore
from opc.plugins.cli_board.tui.screens.help import HelpScreen
from opc.plugins.cli_board.tui.screens.palette import CommandPaletteScreen, PaletteCommand
from opc.plugins.cli_board.tui.screens.prompt import PromptField, PromptScreen
from opc.plugins.cli_board.tui.screens.recovery import RecoveryAction, RecoveryScreen
from opc.plugins.cli_board.widgets.activity_pane import ActivityPaneWidget
from opc.plugins.cli_board.widgets.context_tabs import ContextTabsWidget
from opc.plugins.cli_board.widgets.detail_pane import DetailPaneWidget
@@ -36,7 +35,6 @@ if TYPE_CHECKING:
from opc.plugins.cli_board.services.engine_facade import EngineFacade
from opc.plugins.cli_board.services.event_bridge import CliBoardEventBridge
from opc.plugins.cli_board.services.reconcile import ReconcileLoop
from opc.plugins.cli_board.services.recovery import CliRecoveryManager
class CliBoardApp(App[None]):
@@ -74,7 +72,6 @@ class CliBoardApp(App[None]):
Binding("x", "cancel_task", "Cancel"),
Binding("t", "retry_selected", "Retry"),
Binding("e", "checkpoint_feedback", "Feedback"),
Binding("w", "recovery_scan", "Recovery"),
Binding("R", "rename_session", "Rename", show=False),
Binding("D", "delete_session", "Delete", show=False),
Binding("E", "switch_mode", "Mode", show=False),
@@ -115,7 +112,6 @@ class CliBoardApp(App[None]):
self.repository: BoardRepository | None = None
self.actions: BoardActions | None = None
self.event_bridge: CliBoardEventBridge | None = None
self.recovery_manager: CliRecoveryManager | None = None
self.reconcile_loop: ReconcileLoop | None = None
self.exec_mode = "task"
self.company_profile = "corporate"
@@ -143,13 +139,10 @@ class CliBoardApp(App[None]):
from opc.plugins.cli_board.services.engine_facade import EngineFacade
from opc.plugins.cli_board.services.event_bridge import CliBoardEventBridge
from opc.plugins.cli_board.services.recovery import CliRecoveryManager
self.facade = EngineFacade(project_id=self.project_id)
self.repository = BoardRepository(self.facade, project_id=self.project_id)
self.actions = BoardActions(self.facade, project_id=self.project_id)
self.event_bridge = CliBoardEventBridge(self._handle_board_event)
self.recovery_manager = CliRecoveryManager(self.facade)
def compose(self) -> ComposeResult:
yield Header(show_clock=True)
@@ -650,36 +643,6 @@ class CliBoardApp(App[None]):
success_message=f"{label} checkpoint for {task.title}{suffix}.",
)
def action_recovery_scan(self) -> None:
if self._readonly_guard():
return
self._action_recovery_scan()
@work(group="modal", exclusive=True)
async def _action_recovery_scan(self) -> None:
if self.recovery_manager is None:
self.status_widget.set_message("Recovery unavailable.")
return
status = await self.recovery_manager.get_status()
result = await self.push_screen_wait(RecoveryScreen(status))
if result is None:
return
if result.action == "resume":
self.status_widget.set_message(f"Resuming {result.parent_task_id}...")
outcome = await self.recovery_manager.resume(result.parent_task_id)
if outcome.get("ok"):
ids = outcome.get("resumed_work_item_projection_ids", [])
self.status_widget.set_message(f"Resumed {len(ids)} work item(s).")
else:
self.status_widget.set_message(f"Resume failed: {outcome.get('error', '?')}.")
elif result.action == "cancel":
outcome = await self.recovery_manager.cancel(result.parent_task_id)
if outcome.get("ok"):
self.status_widget.set_message(f"Cancelled {outcome.get('cancelled_count', 0)} task(s).")
else:
self.status_widget.set_message(f"Cancel failed: {outcome.get('error', '?')}.")
await self._refresh_snapshot(reason="recovery", silent=True)
def action_rename_session(self) -> None:
if self._readonly_guard():
return
@@ -1354,7 +1317,6 @@ class CliBoardApp(App[None]):
PaletteCommand("view_focus", "Switch to Focus", "Zoom into the selected task.", "3"),
PaletteCommand("view_pipeline", "Switch to Projection", "Show the read-only work-item projection for the selected company run.", "4"),
PaletteCommand("view_org", "Switch to Organisation", "Show read-only org structure.", "5"),
PaletteCommand("recovery_scan", "Runtime Recovery", "Scan and resume interrupted company runtimes.", "w"),
PaletteCommand("rename_session", "Rename Session", "Change the title of the selected task.", "R"),
PaletteCommand("delete_session", "Delete Session", "Cancel and remove the selected task.", "D"),
PaletteCommand("purge_cancelled", "Purge Cancelled Tasks", "Permanently delete all cancelled/failed tasks.", ""),
@@ -1382,7 +1344,6 @@ class CliBoardApp(App[None]):
"project_delete",
"session_config",
"org_add_role",
"recovery_scan",
"rename_session",
"delete_session",
"purge_cancelled",
+1 -1
View File
@@ -49,7 +49,7 @@ class HelpScreen(ModalScreen[None]):
" s: reply in session m: move between columns\n"
" a / d: approve / deny checkpoint\n"
" e: checkpoint feedback (approve/deny with message)\n"
" c: done x: cancel t: retry w: runtime recovery\n"
" c: done x: cancel t: retry\n"
"\n"
"Session Management\n"
" R: rename session D: delete session\n"
@@ -1,126 +0,0 @@
"""Recovery modal screen for the CLI board."""
from __future__ import annotations
from dataclasses import dataclass
from textual.app import ComposeResult
from textual.containers import Horizontal, Vertical, VerticalScroll
from textual.screen import ModalScreen
from textual.widgets import Button, Label, Static
from opc.plugins.cli_board.services.recovery import RecoveryStatus
@dataclass
class RecoveryAction:
action: str # "resume" | "cancel" | "dismiss"
parent_task_id: str = ""
class RecoveryScreen(ModalScreen[RecoveryAction | None]):
"""Show interrupted company runtimes with resume/cancel options."""
DEFAULT_CSS = """
RecoveryScreen {
align: center middle;
}
.recovery-dialog {
width: 88;
max-width: 90%;
height: auto;
max-height: 80%;
border: solid $primary;
background: $surface;
padding: 1 2;
}
.recovery-title {
text-style: bold;
margin-bottom: 1;
}
.recovery-runtime {
margin-bottom: 1;
padding: 1;
border: round $secondary;
}
.recovery-actions {
align-horizontal: right;
height: auto;
margin-top: 1;
}
.recovery-empty {
color: $text-muted;
margin: 1;
}
"""
BINDINGS = [("escape", "dismiss_screen", "Close")]
def __init__(self, status: RecoveryStatus) -> None:
super().__init__()
self.status = status
def compose(self) -> ComposeResult:
with Vertical(classes="recovery-dialog"):
yield Static("Interrupted Runtimes", classes="recovery-title")
if not self.status.interrupted:
yield Static("No interrupted runtimes found.", classes="recovery-empty")
else:
with VerticalScroll():
for wf in self.status.interrupted:
with Vertical(classes="recovery-runtime"):
# Runtime header
active = wf.parent_task_id in set(self.status.active_recoveries)
status_label = " (recovering...)" if active else ""
yield Label(f"{wf.title}{status_label}")
yield Static(
f" Profile: {wf.profile or 'unknown'} "
f"Interrupted: {wf.interrupted_at[:19] if wf.interrupted_at else '?'}"
)
# Work-item summary
done = sum(1 for s in wf.work_items if s.status == "done")
total = len(wf.work_items)
failed = sum(1 for s in wf.work_items if s.interrupted)
yield Static(
f" Work items: {done}/{total} done, {failed} interrupted"
)
if not active:
with Horizontal():
yield Button(
"Resume",
id=f"resume-{wf.parent_task_id}",
variant="primary",
)
yield Button(
"Cancel",
id=f"cancel-{wf.parent_task_id}",
variant="error",
)
with Horizontal(classes="recovery-actions"):
yield Button("Close", id="close-recovery")
def on_button_pressed(self, event: Button.Pressed) -> None:
btn_id = event.button.id or ""
if btn_id == "close-recovery":
self.dismiss(None)
return
if btn_id.startswith("resume-"):
task_id = btn_id[len("resume-"):]
self.dismiss(RecoveryAction(action="resume", parent_task_id=task_id))
return
if btn_id.startswith("cancel-"):
task_id = btn_id[len("cancel-"):]
self.dismiss(RecoveryAction(action="cancel", parent_task_id=task_id))
return
def action_dismiss_screen(self) -> None:
self.dismiss(None)
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -5,9 +5,9 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<link rel="icon" href="data:," />
<title>OpenOPC Pixel Office</title>
<script type="module" crossorigin src="./assets/index-02tfsorH.js"></script>
<script type="module" crossorigin src="./assets/index-Cson66Y4.js"></script>
<link rel="modulepreload" crossorigin href="./assets/phaser-DFK5Ua9d.js">
<link rel="stylesheet" crossorigin href="./assets/index-BCWwLlJm.css">
<link rel="stylesheet" crossorigin href="./assets/index-DEvLDWDw.css">
</head>
<body>
<div id="root"></div>
+7 -23
View File
@@ -476,7 +476,6 @@ export default function App() {
const [globalCompanyProfile, setGlobalCompanyProfile] = useState<'corporate' | 'custom'>('corporate')
const [globalTaskPreferredAgent, setGlobalTaskPreferredAgent] = useState<TaskPreferredAgent>('native')
const [orgInfoData, setOrgInfoData] = useState<OrgInfoPayload | null>(null)
const [recoveryStatus, setRecoveryStatus] = useState<any>(null)
const [commsState, setCommsState] = useState<import('./lib/wsClient').CommsStatePayload | null>(null)
const [commsMessage, setCommsMessage] = useState<import('./lib/wsClient').CommsMessagePayload | null>(null)
const [talentTemplates, setTalentTemplates] = useState<TalentTemplate[]>([])
@@ -1239,7 +1238,6 @@ export default function App() {
runtimeControlState: String(payload.runtime_control_state ?? payload.runtimeControlState ?? 'idle') as any,
canStop: Boolean(payload.can_stop ?? payload.canStop),
canResume: Boolean(payload.can_resume ?? payload.canResume),
resumeParentTaskId: String(payload.resume_parent_task_id ?? payload.resumeParentTaskId ?? ''),
resumeParentSessionId: String(payload.resume_parent_session_id ?? payload.resumeParentSessionId ?? ''),
pendingRuntimeCheckpointId: String(payload.pending_runtime_checkpoint_id ?? payload.pendingRuntimeCheckpointId ?? ''),
stopIntentId: String(payload.stop_intent_id ?? payload.stopIntentId ?? ''),
@@ -1749,10 +1747,6 @@ export default function App() {
clientRef.current?.collabSync(getActiveProjectId(), undefined, projectViewGenerationRef.current)
}
},
onRecoveryStatus: (payload) => {
if (!payloadMatchesActiveProject(payload as unknown as Record<string, unknown>, false)) return
setRecoveryStatus(payload)
},
onCommsState: (payload) => {
if (!payloadMatchesActiveProject(payload as unknown as Record<string, unknown>, false)) return
setCommsState(payload)
@@ -1761,13 +1755,6 @@ export default function App() {
if (!payloadMatchesActiveProject(payload as unknown as Record<string, unknown>, true)) return
setCommsMessage(payload)
},
onRecoveryResult: (payload) => {
if (!payloadMatchesActiveProject(payload as unknown as Record<string, unknown>, false)) return
if (payload?.status === 'completed' || payload?.status === 'cancelled') {
// Trigger a re-scan
clientRef.current?.recoveryAction(getActiveProjectId(), 'scan')
}
},
onTalentList: (payload) => {
setTalentTemplates(payload.templates ?? [])
if (payload.talent_dir) setDefaultTalentDir(payload.talent_dir)
@@ -2242,18 +2229,13 @@ export default function App() {
) => {
const session = sessionStore.sessions.find(s => s.taskId === taskId)
const parentSessionId = session?.resumeParentSessionId ?? session?.parentSessionId ?? session?.sessionId
const parentTaskId = session?.resumeParentTaskId
?? (parentSessionId ? sessionStore.sessions.find(s => s.sessionId === parentSessionId && !s.parentSessionId)?.taskId : undefined)
?? taskId
for (const candidate of sessionStore.sessions) {
if (
candidate.taskId === taskId
|| candidate.taskId === parentTaskId
|| (!!parentSessionId && (candidate.parentSessionId === parentSessionId || candidate.sessionId === parentSessionId))
) {
sessionStore.updateSession(candidate.taskId, {
...patch,
resumeParentTaskId: parentTaskId,
resumeParentSessionId: parentSessionId,
})
}
@@ -2278,7 +2260,7 @@ export default function App() {
clientRef.current?.sessionStop(getActiveProjectId(), taskId)
}, [sessionStore.sessions, markRuntimeControlForTask, getActiveProjectId])
const handleSessionResume = useCallback((taskId: string) => {
const handleSessionResume = useCallback((taskId: string, runtimeSessionId?: string, checkpointId?: string) => {
const session = sessionStore.sessions.find(s => s.taskId === taskId)
const isCompanyRuntime = session?.execMode === 'company'
|| session?.execMode === 'org'
@@ -2293,7 +2275,12 @@ export default function App() {
canResume: false,
})
}
clientRef.current?.sessionResume(getActiveProjectId(), taskId)
clientRef.current?.sessionResume(
getActiveProjectId(),
taskId,
runtimeSessionId ?? session?.resumeParentSessionId ?? session?.parentSessionId ?? session?.sessionId,
checkpointId ?? session?.pendingRuntimeCheckpointId,
)
}, [sessionStore.sessions, markRuntimeControlForTask, getActiveProjectId])
const handleGlobalModeChange = useCallback((mode: 'task' | 'company' | 'org' | 'custom', profile?: string, orgId?: string) => {
@@ -2426,9 +2413,6 @@ export default function App() {
activeSavedOrg={activeSavedOrg}
onSavedOrgsList={handleSavedOrgsList}
onSavedOrgLoad={handleSavedOrgLoad}
recoveryStatus={recoveryStatus}
onRecoveryResume={(id) => clientRef.current?.recoveryAction(getActiveProjectId(), 'resume', id)}
onRecoveryCancel={(id) => clientRef.current?.recoveryAction(getActiveProjectId(), 'cancel', id)}
commsState={commsState}
commsMessage={commsMessage}
onCommsRefresh={(opts) => {
@@ -676,7 +676,6 @@ export function mapBackendSession(raw: any): Session {
runtimeControlState: raw.runtime_control_state ?? raw.runtimeControlState,
canStop: raw.can_stop ?? raw.canStop,
canResume: raw.can_resume ?? raw.canResume,
resumeParentTaskId: raw.resume_parent_task_id ?? raw.resumeParentTaskId,
resumeParentSessionId: raw.resume_parent_session_id ?? raw.resumeParentSessionId,
pendingRuntimeCheckpointId: raw.pending_runtime_checkpoint_id ?? raw.pendingRuntimeCheckpointId,
stopIntentId: raw.stop_intent_id ?? raw.stopIntentId,
@@ -387,7 +387,6 @@ export function getConversationSessionView(
runtimeControlState: runtimeSource.runtimeControlState ?? normalizedActiveSession.runtimeControlState,
canStop: runtimeSource.canStop ?? normalizedActiveSession.canStop,
canResume: runtimeSource.canResume ?? normalizedActiveSession.canResume,
resumeParentTaskId: runtimeSource.resumeParentTaskId ?? normalizedActiveSession.resumeParentTaskId,
resumeParentSessionId: runtimeSource.resumeParentSessionId ?? normalizedActiveSession.resumeParentSessionId,
pendingRuntimeCheckpointId: runtimeSource.pendingRuntimeCheckpointId ?? normalizedActiveSession.pendingRuntimeCheckpointId,
stopIntentId: runtimeSource.stopIntentId ?? normalizedActiveSession.stopIntentId,
@@ -447,7 +446,6 @@ export function getConversationHeaderSession(
runtimeControlState: runtimeSource.runtimeControlState ?? normalizedActiveSession.runtimeControlState,
canStop: runtimeSource.canStop ?? normalizedActiveSession.canStop,
canResume: runtimeSource.canResume ?? normalizedActiveSession.canResume,
resumeParentTaskId: runtimeSource.resumeParentTaskId ?? normalizedActiveSession.resumeParentTaskId,
resumeParentSessionId: runtimeSource.resumeParentSessionId ?? normalizedActiveSession.resumeParentSessionId,
pendingRuntimeCheckpointId: runtimeSource.pendingRuntimeCheckpointId ?? normalizedActiveSession.pendingRuntimeCheckpointId,
stopIntentId: runtimeSource.stopIntentId ?? normalizedActiveSession.stopIntentId,
@@ -30,6 +30,17 @@ const flushPromises = async () => {
const client = new VisualSocketClient('ws://unit.test', {})
// Company Continue keeps the selected UI channel task separate from the
// durable runtime identity used by the checkpoint handoff.
client.sessionResume('project-a', 'ui-task', 'runtime-session', 'checkpoint-1')
const resumeEnvelope = JSON.parse(
(client as unknown as TestSocketClient).pendingQueue.pop() ?? '{}',
) as Record<string, unknown>
assert.equal(resumeEnvelope.type, 'session_resume')
assert.equal(resumeEnvelope.task_id, 'ui-task')
assert.equal(resumeEnvelope.runtime_session_id, 'runtime-session')
assert.equal(resumeEnvelope.checkpoint_id, 'checkpoint-1')
// A summary and a full request for the same task are distinct correlations.
// Neither Promise may settle merely because the request was queued locally.
const summaryPromise = client.sessionDetail('project-a', 'task-1', { detailLevel: 'summary' })
@@ -41,8 +41,6 @@ interface SocketHandlers {
onProjectSwitched?: (payload: { project_id: string; switch_seq?: string }) => void
onProjectDeleted?: (payload: { project_id: string }) => void
onOrgInfo?: (payload: OrgInfoPayload) => void
onRecoveryStatus?: (payload: any) => void
onRecoveryResult?: (payload: any) => void
onTalentList?: (payload: TalentListPayload) => void
onTalentScanLocal?: (payload: { templates: Array<{ template_id: string; name: string; description: string; category: string; domains: string[]; tags: string[] }> }) => void
onEmployeeDetail?: (payload: EmployeeDetailPayload) => void
@@ -169,7 +167,6 @@ const PROJECT_SCOPED_MESSAGE_TYPES = new Set([
'session_update_title',
'secretary_send',
'project_index',
'recovery_action',
'comms_state',
'comms_read_message',
])
@@ -420,9 +417,22 @@ export class VisualSocketClient {
this.send({ type: 'session_stop', project_id: pid, task_id: taskId })
}
sessionResume(projectId: string, taskId: string, content?: string): void {
sessionResume(
projectId: string,
taskId: string,
runtimeSessionId?: string,
checkpointId?: string,
content?: string,
): void {
const pid = this.requireProjectId(projectId, 'session_resume')
this.send({ type: 'session_resume', project_id: pid, task_id: taskId, content })
this.send({
type: 'session_resume',
project_id: pid,
task_id: taskId,
runtime_session_id: runtimeSessionId,
checkpoint_id: checkpointId,
content,
})
}
sessionComplete(projectId: string, taskId: string): void {
@@ -646,11 +656,6 @@ export class VisualSocketClient {
this.send({ type: 'org_saved_delete', name })
}
recoveryAction(projectId: string, action: 'resume' | 'cancel' | 'scan', parentTaskId?: string): void {
const pid = this.requireProjectId(projectId, 'recovery_action')
this.send({ type: 'recovery_action', project_id: pid, action, parent_task_id: parentTaskId })
}
commsState(projectId: string, opts?: { task_id?: string; session_id?: string }): void {
const pid = this.requireProjectId(projectId, 'comms_state')
this.send({ type: 'comms_state', project_id: pid, ...(opts || {}) })
@@ -801,12 +806,6 @@ export class VisualSocketClient {
if (projectId) this.commsState(projectId)
} catch { /* ignore */ }
break
case 'recovery_status':
this.handlers.onRecoveryStatus?.(parsed.payload)
break
case 'recovery_result':
this.handlers.onRecoveryResult?.(parsed.payload)
break
case 'talent_list':
this.handlers.onTalentList?.(parsed.payload)
break
@@ -244,7 +244,6 @@ function sessionReducer(state: Session[], action: SessionAction): Session[] {
runtimeControlState: guardedRuntimeControl.runtimeControlState ?? existing.runtimeControlState,
canStop: guardedRuntimeControl.canStop ?? existing.canStop,
canResume: guardedRuntimeControl.canResume ?? existing.canResume,
resumeParentTaskId: incoming.resumeParentTaskId ?? existing.resumeParentTaskId,
resumeParentSessionId: incoming.resumeParentSessionId ?? existing.resumeParentSessionId,
pendingRuntimeCheckpointId: guardedRuntimeControl.pendingRuntimeCheckpointId ?? existing.pendingRuntimeCheckpointId,
stopIntentId: guardedRuntimeControl.stopIntentId ?? existing.stopIntentId,
@@ -337,7 +336,6 @@ function sessionReducer(state: Session[], action: SessionAction): Session[] {
runtimeControlState: guarded.runtimeControlState ?? s.runtimeControlState,
canStop: guarded.canStop ?? s.canStop,
canResume: guarded.canResume ?? s.canResume,
resumeParentTaskId: guarded.resumeParentTaskId ?? s.resumeParentTaskId,
resumeParentSessionId: guarded.resumeParentSessionId ?? s.resumeParentSessionId,
pendingRuntimeCheckpointId: guarded.pendingRuntimeCheckpointId ?? s.pendingRuntimeCheckpointId,
stopIntentId: guarded.stopIntentId ?? s.stopIntentId,
@@ -316,10 +316,9 @@ export interface Session {
originChannel?: string
originTaskId?: string
runtimeControlState?: 'running' | 'suspending' | 'suspended' | 'resuming' | 'idle'
canStop?: boolean
canResume?: boolean
resumeParentTaskId?: string
resumeParentSessionId?: string
canStop?: boolean
canResume?: boolean
resumeParentSessionId?: string
pendingRuntimeCheckpointId?: string
stopIntentId?: string
// Handoff context from upstream work item (Company Mode)
@@ -100,8 +100,6 @@ export type SocketEnvelope =
| { type: 'work_item_batch_updated'; payload: { run_id?: string; work_items: RuntimeWorkItemInfo[]; frontier?: RuntimeFrontierSummary } }
| { type: 'project_recovery_updated'; payload: Record<string, unknown> }
| { type: 'project_revision_created'; payload: { run_id?: string; revision_links: SessionLinkInfo[] } }
| { type: 'recovery_status'; payload: Record<string, unknown> }
| { type: 'recovery_result'; payload: Record<string, unknown> }
| { type: 'talent_list'; payload: TalentListPayload }
| { type: 'talent_scan_local'; payload: { templates: Array<{ template_id: string; name: string; description: string; category: string; domains: string[]; tags: string[] }> } }
| { type: 'employee_detail'; payload: EmployeeDetailPayload }
@@ -77,7 +77,6 @@ interface ContextPanelProps {
onCommsRefresh?: () => void
onCommsReadMessage?: (path: string) => void
orgInfoData?: OrgInfoPayload | null
recoveryStatus?: Record<string, unknown> | null
canShowTeamTab?: boolean
onTeamStopRun?: () => void
@@ -497,7 +496,6 @@ export function ContextPanel({
onCommsRefresh,
onCommsReadMessage,
orgInfoData,
recoveryStatus,
canShowTeamTab = false,
onTeamStopRun,
onTitleChange,
@@ -1306,7 +1304,6 @@ export function ContextPanel({
<div style={{ flex: 1, overflow: 'auto', minHeight: 0 }}>
<ProjectCockpit
orgInfoData={orgInfoData ?? null}
recoveryStatus={recoveryStatus ?? null}
commsState={commsState ?? null}
onStopRun={onTeamStopRun}
embedded
@@ -48,7 +48,6 @@ interface TeamCardInfo {
interface ProjectCockpitProps {
orgInfoData?: OrgInfoPayload | null
recoveryStatus?: Record<string, unknown> | null
commsState?: CommsStatePayload | null
onStopRun?: () => void
embedded?: boolean
@@ -56,7 +55,6 @@ interface ProjectCockpitProps {
export function ProjectCockpit({
orgInfoData,
recoveryStatus,
commsState,
onStopRun,
embedded = false,
@@ -78,7 +76,6 @@ export function ProjectCockpit({
count + asRecordList(asRecord(digest.manager_digest).notification_backlog).length
), 0)
const unreadCount = actionableCount + protocolCount + notificationCount
const interrupted = Array.isArray(recoveryStatus?.interrupted) ? recoveryStatus.interrupted.length : 0
const communicationItems = [
{ label: 'Actionable', value: actionableCount },
@@ -198,7 +195,7 @@ export function ProjectCockpit({
<span>Seats {runtimeView.runtimeSeats.length}</span>
<span>Approvals {pendingDecisionCount}</span>
<span>Unread {unreadCount}</span>
<span>Recovery {interrupted > 0 ? `${interrupted} interrupted` : summarizeText(asRecord(projectRun?.recovery_pointer).status, 'clean')}</span>
<span>Run state {summarizeText(asRecord(projectRun?.recovery_pointer).status, 'clean')}</span>
</div>
</div>
@@ -1,116 +0,0 @@
import { useState } from 'react'
export interface RecoverableWorkItem {
work_item_projection_id: string
title: string
task_id: string
status: string
interrupted: boolean
previous_status: string
}
export interface InterruptedWorkItemRuntime {
parent_session_id: string
parent_task_id: string
project_id: string
title: string
profile: string
interrupted_at: string
work_items: RecoverableWorkItem[]
}
export interface RecoveryStatusPayload {
interrupted: InterruptedWorkItemRuntime[]
active_recoveries: string[]
scanned_at: number
}
interface WorkItemRecoveryPanelProps {
data: RecoveryStatusPayload
onResume: (parentTaskId: string) => void
onCancel: (parentTaskId: string) => void
}
const STATUS_ICON: Record<string, string> = {
done: '\u2713',
failed: '\u2717',
pending: '\u25CB',
blocked: '\u25A0',
cancelled: '\u2014',
running: '\u25B6',
}
const STATUS_COLOR: Record<string, string> = {
done: 'var(--green, #27ae60)',
failed: 'var(--red, #e74c3c)',
pending: 'var(--text-secondary, #888)',
blocked: 'var(--yellow, #f39c12)',
cancelled: 'var(--text-dim, #555)',
running: 'var(--accent, #3498db)',
}
export function WorkItemRecoveryPanel({ data, onResume, onCancel }: WorkItemRecoveryPanelProps) {
const [dismissed, setDismissed] = useState<Set<string>>(new Set())
if (!data.interrupted.length && !data.active_recoveries.length) return null
const visible = data.interrupted.filter(w => !dismissed.has(w.parent_task_id))
if (!visible.length && !data.active_recoveries.length) return null
return (
<div className="wfr-panel">
{visible.map(wf => {
const isRecovering = data.active_recoveries.includes(wf.parent_task_id)
const doneCount = wf.work_items.filter(item => item.status === 'done').length
const failedCount = wf.work_items.filter(item => item.interrupted || item.status === 'failed').length
return (
<div key={wf.parent_task_id} className="wfr-card">
<div className="wfr-header">
<span className="wfr-icon">&#x26A0;</span>
<div className="wfr-header-text">
<span className="wfr-title">Interrupted: {wf.title}</span>
<span className="wfr-subtitle">
{doneCount}/{wf.work_items.length} work items done, {failedCount} interrupted
{wf.profile && <> &middot; {wf.profile}</>}
</span>
</div>
</div>
<div className="wfr-work-items">
{wf.work_items.map(item => (
<div key={item.work_item_projection_id} className={`wfr-work-item wfr-work-item--${item.status}`}>
<span className="wfr-work-item-icon" style={{ color: STATUS_COLOR[item.status] || STATUS_COLOR.pending }}>
{STATUS_ICON[item.status] || STATUS_ICON.pending}
</span>
<span className="wfr-work-item-title">{item.title}</span>
{item.interrupted && <span className="wfr-work-item-badge">interrupted</span>}
</div>
))}
</div>
<div className="wfr-actions">
{isRecovering ? (
<span className="wfr-recovering">
<span className="spinner-inline" /> Recovering...
</span>
) : (
<>
<button className="wfr-btn wfr-btn--resume" onClick={() => onResume(wf.parent_task_id)}>
Resume
</button>
<button className="wfr-btn wfr-btn--cancel" onClick={() => onCancel(wf.parent_task_id)}>
Cancel
</button>
<button className="wfr-btn wfr-btn--dismiss" onClick={() => setDismissed(prev => new Set(prev).add(wf.parent_task_id))}>
Dismiss
</button>
</>
)}
</div>
</div>
)
})}
</div>
)
}
@@ -1,6 +1,5 @@
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react'
import type { AgentInfo, OrgInfoPayload, SavedOrgSummary } from '../types/visual'
import { WorkItemRecoveryPanel } from './WorkItemRecoveryPanel'
import type { ChatMessage, CheckpointReplyMetadata, OutgoingAttachmentPayload } from '../types/chat'
import type { KanbanTask, Session, TaskPreferredAgent } from '../types/kanban'
import type { BoardStoreState } from '../kanban/BoardStore'
@@ -253,7 +252,7 @@ interface WorkspacePageProps {
*/
onContinueInNewChat?: (mode: 'task' | 'company' | 'org' | 'custom', companyProfile?: 'corporate' | 'custom', orgId?: string) => void
onSessionStop?: (taskId: string) => void
onSessionResume?: (taskId: string) => void
onSessionResume?: (taskId: string, runtimeSessionId?: string, checkpointId?: string) => void
onSessionComplete?: (taskId: string) => void
onLoadSessionDetail?: (
taskId: string,
@@ -263,9 +262,6 @@ interface WorkspacePageProps {
onCollabSync?: () => void
orgInfoData?: OrgInfoPayload | null
onNavigateToOrg?: () => void
recoveryStatus?: any
onRecoveryResume?: (parentTaskId: string) => void
onRecoveryCancel?: (parentTaskId: string) => void
commsState?: import('../lib/wsClient').CommsStatePayload | null
commsMessage?: import('../lib/wsClient').CommsMessagePayload | null
onCommsRefresh?: (opts?: { task_id?: string; session_id?: string; project_id?: string }) => void
@@ -305,9 +301,6 @@ export function WorkspacePage({
onCollabSync,
orgInfoData,
onNavigateToOrg,
recoveryStatus,
onRecoveryResume,
onRecoveryCancel,
commsState,
commsMessage,
onCommsRefresh,
@@ -988,13 +981,21 @@ export function WorkspacePage({
const handleResume = useCallback(() => {
const targetSession = activeConversation.runtimeSession ?? activeConversation.displaySession ?? activeSession
const targetTaskId = targetSession?.resumeParentTaskId ?? targetSession?.taskId ?? activeSessionId
if (targetTaskId) onSessionResume?.(targetTaskId)
const uiTaskId = activeSessionId ?? targetSession?.taskId
const runtimeSessionId = targetSession?.resumeParentSessionId
?? targetSession?.parentSessionId
?? targetSession?.sessionId
if (uiTaskId) {
onSessionResume?.(uiTaskId, runtimeSessionId, targetSession?.pendingRuntimeCheckpointId)
}
}, [activeConversation.runtimeSession, activeConversation.displaySession, activeSession, activeSessionId, onSessionResume])
const handleResumeTask = useCallback((taskId: string) => {
const session = sessions.find(s => s.taskId === taskId)
onSessionResume?.(session?.resumeParentTaskId ?? taskId)
const runtimeSessionId = session?.resumeParentSessionId
?? session?.parentSessionId
?? session?.sessionId
onSessionResume?.(taskId, runtimeSessionId, session?.pendingRuntimeCheckpointId)
}, [sessions, onSessionResume])
const handleCompleteTask = useCallback((taskId: string) => {
@@ -1034,8 +1035,11 @@ export function WorkspacePage({
}
const targetTaskId = activeSessionId
if (!targetTaskId) return
const checkpointReplyId = String(latestPendingCheckpointReply?.response_to_checkpoint_id ?? '').trim()
const runtimeSession = activeConversation.runtimeSession ?? activeConversation.displaySession ?? activeSession
const runtimeCheckpointId = String(runtimeSession?.pendingRuntimeCheckpointId ?? '').trim()
let outgoingMetadata = latestPendingCheckpointReply
?? (runtimeCheckpointId ? { response_to_checkpoint_id: runtimeCheckpointId } : undefined)
const checkpointReplyId = String(outgoingMetadata?.response_to_checkpoint_id ?? '').trim()
if (!checkpointReplyId) {
const uiMessageId = makeOptimisticUserMessageId()
outgoingMetadata = { ...(latestPendingCheckpointReply ?? {}), ui_message_id: uiMessageId }
@@ -1050,7 +1054,7 @@ export function WorkspacePage({
}
dispatchSessionSend(targetTaskId, content, attachments, outgoingMetadata)
},
[effectiveView.kind, activeSessionId, activeConversation.displaySession, activeSession, latestPendingCheckpointReply, chatStore, dispatchSessionSend, onSecretarySend],
[effectiveView.kind, activeSessionId, activeConversation.runtimeSession, activeConversation.displaySession, activeSession, latestPendingCheckpointReply, chatStore, dispatchSessionSend, onSecretarySend],
)
// ── MessageList send (checkpoint replies) ──
@@ -1114,13 +1118,6 @@ export function WorkspacePage({
{/* Middle column: Kanban Board (hidden when panel maximized) */}
{panelState !== 'maximized' && (
<div className="workspace-board">
{recoveryStatus && onRecoveryResume && onRecoveryCancel && (
<WorkItemRecoveryPanel
data={recoveryStatus}
onResume={onRecoveryResume}
onCancel={onRecoveryCancel}
/>
)}
{agents.length > 0 && <AgentStatusBar agents={agents} tasks={boardStore.tasks} />}
{isCompanyMode ? (
boardStore.activeBoard && activeSession && (
@@ -1206,7 +1203,6 @@ export function WorkspacePage({
onCommsRefresh={onCommsRefresh ? () => onCommsRefresh({ session_id: activeSession?.sessionId || undefined, project_id: projectId || undefined }) : undefined}
onCommsReadMessage={onCommsReadMessage}
orgInfoData={orgInfoData ?? null}
recoveryStatus={recoveryStatus ?? null}
canShowTeamTab={canShowTeamTab}
onTeamStopRun={activeSessionId ? () => onSessionStop?.(activeSessionId) : undefined}
onTitleChange={onTitleChange}
@@ -1605,132 +1605,6 @@
}
}
/* ── Work Item Recovery Panel ────────────────────────────────────────── */
.wfr-panel {
padding: 0 8px;
}
.wfr-card {
background: color-mix(in srgb, var(--yellow, #f39c12) 8%, var(--bg-secondary));
border: 1px solid color-mix(in srgb, var(--yellow, #f39c12) 25%, transparent);
border-radius: 8px;
padding: 12px 14px;
margin-bottom: 8px;
}
.wfr-header {
display: flex;
align-items: flex-start;
gap: 8px;
margin-bottom: 8px;
}
.wfr-icon {
font-size: 16px;
line-height: 1;
flex-shrink: 0;
}
.wfr-header-text {
display: flex;
flex-direction: column;
gap: 2px;
min-width: 0;
}
.wfr-title {
font-size: 13px;
font-weight: 600;
color: var(--text);
}
.wfr-subtitle {
font-size: 11px;
color: var(--text-secondary);
}
.wfr-work-items {
display: flex;
flex-direction: column;
gap: 4px;
margin-bottom: 10px;
}
.wfr-work-item {
display: flex;
align-items: center;
gap: 6px;
font-size: 12px;
color: var(--text-secondary);
}
.wfr-work-item-icon {
font-size: 12px;
width: 14px;
text-align: center;
flex-shrink: 0;
font-weight: 700;
}
.wfr-work-item-title {
flex: 1;
min-width: 0;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.wfr-work-item-badge {
font-size: 9px;
padding: 1px 5px;
border-radius: 3px;
background: color-mix(in srgb, var(--red, #e74c3c) 15%, transparent);
color: var(--red, #e74c3c);
flex-shrink: 0;
}
.wfr-actions {
display: flex;
gap: 8px;
align-items: center;
}
.wfr-btn {
padding: 5px 14px;
font-size: 12px;
font-weight: 500;
border: none;
border-radius: 6px;
cursor: pointer;
}
.wfr-btn--resume {
background: var(--green, #27ae60);
color: #fff;
}
.wfr-btn--resume:hover { opacity: 0.85; }
.wfr-btn--cancel {
background: var(--red, #e74c3c);
color: #fff;
}
.wfr-btn--cancel:hover { opacity: 0.85; }
.wfr-btn--dismiss {
background: transparent;
color: var(--text-secondary);
}
.wfr-btn--dismiss:hover { color: var(--text); }
.wfr-recovering {
display: flex;
align-items: center;
gap: 6px;
font-size: 12px;
color: var(--accent);
}
/*
Kanban empty-state
*/
-552
View File
@@ -1,552 +0,0 @@
"""Pluggable work-item crash-recovery manager.
Detects interrupted company-mode work items after server restart and provides
deterministic Resume / Cancel operations no LLM needed.
Hooks into existing engine capabilities without modifying core code:
- engine.store task queries + persistence
- engine._load_company_runtime_snapshot() reconstruct work-item plan + tasks
- engine.company_executor.execute() resume execution
- ws_handler.broadcast() push status to all clients
"""
from __future__ import annotations
import asyncio
import dataclasses
import logging
import time
from dataclasses import dataclass, field
from datetime import datetime
from typing import Any, Awaitable, Callable
from opc.layer2_organization.work_item_identity import work_item_projection_id_from_metadata
from opc.layer2_organization.work_item_transition import apply_task_status_transition
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Data types
# ---------------------------------------------------------------------------
@dataclass
class RecoverableWorkItem:
work_item_projection_id: str
title: str
task_id: str
status: str # "done" | "failed" | "pending" | "blocked" | "cancelled"
interrupted: bool # has interrupted_recovery metadata
previous_status: str # what it was before reconciliation
@dataclass
class InterruptedWorkItemRun:
parent_session_id: str
parent_task_id: str
project_id: str
title: str
profile: str
interrupted_at: str
work_items: list[RecoverableWorkItem] = field(default_factory=list)
@dataclass
class RecoveryStatus:
interrupted: list[InterruptedWorkItemRun] = field(default_factory=list)
active_recoveries: list[str] = field(default_factory=list)
scanned_at: float = 0.0
# ---------------------------------------------------------------------------
# Manager
# ---------------------------------------------------------------------------
class RuntimeRecoveryManager:
"""Scans for interrupted work-item runs and provides deterministic recovery."""
_CACHE_TTL = 10.0 # seconds
def __init__(
self,
engine: Any,
broadcast_fn: Callable[[dict[str, Any]], Awaitable[None]],
) -> None:
self._engine = engine
self._broadcast = broadcast_fn
self._lock = asyncio.Lock()
self._active_recoveries: dict[str, asyncio.Task[Any]] = {}
self._cached: RecoveryStatus | None = None
self._cache_until: float = 0.0
# ------------------------------------------------------------------
# Public API
# ------------------------------------------------------------------
async def get_recovery_status(self) -> RecoveryStatus:
"""Return cached scan results, re-scanning if stale."""
now = time.time()
if self._cached is not None and now < self._cache_until:
# Keep active_recoveries up to date even from cache
self._cached.active_recoveries = list(self._active_recoveries.keys())
return self._cached
status = await self.scan()
self._cached = status
self._cache_until = now + self._CACHE_TTL
return status
async def scan(self) -> RecoveryStatus:
"""Scan the database for recoverable interrupted work-item runs."""
store = self._engine.store
if not store:
return RecoveryStatus()
project_id = self._engine.project_id or "default"
try:
all_tasks = await store.get_tasks(project_id=project_id)
except Exception as exc:
logger.warning(f"Recovery scan failed: {exc}")
return RecoveryStatus()
# Group projected work-item tasks by parent_session_id.
groups: dict[str, list[Any]] = {}
all_tasks_by_session: dict[str, Any] = {}
for task in all_tasks:
session_id = str(getattr(task, "session_id", "") or "").strip()
if session_id:
all_tasks_by_session[session_id] = task
parent_sid = str(getattr(task, "parent_session_id", "") or "").strip()
projection_id = work_item_projection_id_from_metadata(getattr(task, "metadata", {}) or {})
if parent_sid and projection_id:
groups.setdefault(parent_sid, []).append(task)
interrupted: list[InterruptedWorkItemRun] = []
for parent_sid, tasks in groups.items():
# Check if any task has interrupted_recovery metadata
has_interrupted = any(
_is_interrupted(t) for t in tasks
)
if not has_interrupted:
continue
# Skip if all tasks are terminal (DONE or CANCELLED)
from opc.core.models import TaskStatus
non_terminal = [
t for t in tasks
if t.status not in (TaskStatus.DONE, TaskStatus.CANCELLED)
]
if not non_terminal:
continue
# Find the parent (primary) task
parent_task = all_tasks_by_session.get(parent_sid)
parent_task_id = parent_task.id if parent_task else parent_sid
title = parent_task.title if parent_task else "Unknown work-item run"
# Build work-item list.
work_items: list[RecoverableWorkItem] = []
earliest_interrupt = ""
for t in sorted(tasks, key=lambda x: (x.created_at, x.id)):
meta = dict(getattr(t, "metadata", {}) or {})
recovery_meta = meta.get("interrupted_recovery", {})
is_int = _is_interrupted(t)
if is_int and recovery_meta.get("detected_at", ""):
detected = recovery_meta["detected_at"]
if not earliest_interrupt or detected < earliest_interrupt:
earliest_interrupt = detected
work_items.append(RecoverableWorkItem(
work_item_projection_id=work_item_projection_id_from_metadata(meta, fallback=t.id),
title=t.title,
task_id=t.id,
status=t.status.value if hasattr(t.status, "value") else str(t.status),
interrupted=is_int,
previous_status=recovery_meta.get("previous_status", ""),
))
profile = ""
for t in tasks:
p = (getattr(t, "metadata", {}) or {}).get("company_profile", "")
if p:
profile = p
break
interrupted.append(InterruptedWorkItemRun(
parent_session_id=parent_sid,
parent_task_id=parent_task_id,
project_id=project_id,
title=title,
profile=profile,
interrupted_at=earliest_interrupt or datetime.now().isoformat(),
work_items=work_items,
))
return RecoveryStatus(
interrupted=interrupted,
active_recoveries=list(self._active_recoveries.keys()),
scanned_at=time.time(),
)
async def resume(self, parent_task_id: str) -> dict[str, Any]:
"""Deterministically resume an interrupted work-item run."""
async with self._lock:
if parent_task_id in self._active_recoveries:
return {"ok": False, "error": "already_in_progress"}
# Find the interrupted work-item run.
status = await self.scan()
wf = next((w for w in status.interrupted if w.parent_task_id == parent_task_id), None)
if not wf:
return {"ok": False, "error": "not_found"}
# Load snapshot
snapshot = await self._engine._load_company_runtime_snapshot(wf.parent_session_id)
if not snapshot:
return {"ok": False, "error": "snapshot_unavailable"}
plan, tasks = snapshot
# Clean orphaned checkpoints
await self._clean_orphaned_checkpoints(wf, tasks)
# Reset interrupted/failed/blocked tasks → PENDING
from opc.core.models import TaskStatus
resumed_ids: list[str] = []
failed_ids: list[str] = []
for task in tasks:
if task.status == TaskStatus.DONE:
continue
if task.status in (TaskStatus.FAILED, TaskStatus.BLOCKED):
task.result = None
task.execution_lock = False
task.execution_locked_at = None
meta = dict(task.metadata)
meta.pop("interrupted_recovery", None)
progress = list(meta.get("progress_log", []))
progress.append(f"[Recovery] Resumed at {datetime.now().isoformat()}")
meta["progress_log"] = progress[-20:]
task.metadata = meta
projection_id = work_item_projection_id_from_metadata(meta, fallback=task.id)
try:
await apply_task_status_transition(
self._engine.store,
task,
target_status_or_phase=TaskStatus.PENDING,
reason="office_recovery_resume",
release_claim=True,
)
except Exception as exc:
logger.warning("Recovery resume skipped %s: %s", task.id, exc)
failed_ids.append(projection_id)
continue
if task.status != TaskStatus.PENDING:
logger.warning("Recovery resume preserved non-runnable phase for %s", task.id)
failed_ids.append(projection_id)
continue
await self._engine.store.save_task(task)
resumed_ids.append(projection_id)
if not resumed_ids:
return {
"ok": False,
"error": "no_work_items_to_resume",
"failed_work_item_projection_ids": failed_ids,
}
# Invalidate cache
self._cache_until = 0.0
# Launch execution in background
await self._set_run_recovery_state(
wf.parent_session_id,
status="resuming",
lifecycle_status="active",
)
bg_task = asyncio.create_task(
self._execute_recovery(parent_task_id, wf.parent_session_id, plan, tasks)
)
self._active_recoveries[parent_task_id] = bg_task
return {
"ok": True,
"resumed_work_item_projection_ids": resumed_ids,
"failed_work_item_projection_ids": failed_ids,
}
async def cancel(self, parent_task_id: str) -> dict[str, Any]:
"""Cancel an interrupted work-item run and clean up."""
async with self._lock:
# Cancel active recovery if running
bg = self._active_recoveries.pop(parent_task_id, None)
if bg and not bg.done():
bg.cancel()
# Find the interrupted work-item run.
status = await self.scan()
wf = next((w for w in status.interrupted if w.parent_task_id == parent_task_id), None)
if not wf:
return {"ok": False, "error": "not_found"}
# Load tasks and cancel non-terminal ones
snapshot = await self._engine._load_company_runtime_snapshot(wf.parent_session_id)
if not snapshot:
return {"ok": False, "error": "snapshot_unavailable"}
_, tasks = snapshot
from opc.core.models import TaskStatus
cancelled_count = 0
failed_ids: list[str] = []
for task in tasks:
if task.status not in (TaskStatus.DONE, TaskStatus.CANCELLED):
projection_id = work_item_projection_id_from_metadata(
getattr(task, "metadata", {}) or {},
fallback=task.id,
)
try:
await apply_task_status_transition(
self._engine.store,
task,
target_status_or_phase=TaskStatus.CANCELLED,
reason="office_recovery_cancel",
release_claim=True,
)
except Exception as exc:
logger.warning("Recovery cancel skipped %s: %s", task.id, exc)
failed_ids.append(projection_id)
continue
if task.status != TaskStatus.CANCELLED:
logger.warning("Recovery cancel preserved non-cancelled phase for %s", task.id)
failed_ids.append(projection_id)
continue
cancelled_count += 1
# Clean orphaned checkpoints
await self._clean_orphaned_checkpoints(wf, tasks)
# Invalidate cache
self._cache_until = 0.0
await self._set_run_recovery_state(
wf.parent_session_id,
status="cancelled",
lifecycle_status="cancelled",
)
await self._broadcast_status()
return {
"ok": True,
"cancelled_count": cancelled_count,
"failed_work_item_projection_ids": failed_ids,
}
# ------------------------------------------------------------------
# Internal helpers
# ------------------------------------------------------------------
async def _execute_recovery(
self,
parent_task_id: str,
parent_session_id: str,
plan: Any,
tasks: list[Any],
) -> None:
"""Run the work-item executor in the background."""
project_id = self._engine.project_id or "default"
try:
await self._set_run_recovery_state(
parent_session_id,
status="started",
lifecycle_status="active",
)
await self._broadcast({"type": "recovery_result", "payload": {
"project_id": project_id,
"parent_task_id": parent_task_id, "status": "started",
}})
executor = self._engine.company_executor
if not executor:
raise RuntimeError("company_executor not available")
result = await executor.execute(plan, tasks)
await self._broadcast({"type": "recovery_result", "payload": {
"project_id": project_id,
"parent_task_id": parent_task_id, "status": "completed",
"summary": result[:500] if result else "",
}})
await self._set_run_recovery_state(
parent_session_id,
status="completed",
lifecycle_status="active",
)
except asyncio.CancelledError:
await self._broadcast({"type": "recovery_result", "payload": {
"project_id": project_id,
"parent_task_id": parent_task_id, "status": "cancelled",
}})
await self._set_run_recovery_state(
parent_session_id,
status="cancelled",
lifecycle_status="cancelled",
)
except Exception as exc:
logger.warning(f"Recovery execution failed for {parent_task_id}: {exc}")
await self._broadcast({"type": "recovery_result", "payload": {
"project_id": project_id,
"parent_task_id": parent_task_id, "status": "failed",
"error": str(exc),
}})
await self._set_run_recovery_state(
parent_session_id,
status="failed",
lifecycle_status="blocked",
extra={"error": str(exc)},
)
finally:
self._active_recoveries.pop(parent_task_id, None)
self._cache_until = 0.0
await self._broadcast_status()
async def _clean_orphaned_checkpoints(
self,
wf: InterruptedWorkItemRun,
tasks: list[Any],
) -> int:
"""Resolve pending checkpoints whose tasks are no longer active."""
store = self._engine.store
if not store:
return 0
session_ids = {
str(getattr(t, "session_id", "") or "").strip()
for t in tasks
}
session_ids.add(wf.parent_session_id)
session_ids.discard("")
cleaned = 0
try:
pending = await store.get_pending_checkpoints(
project_id=wf.project_id,
)
for cp in pending:
cp_session = str(cp.session_id or "").strip()
if cp_session in session_ids:
await store.resolve_execution_checkpoint(
cp.checkpoint_id, status="cancelled"
)
cleaned += 1
except Exception as exc:
logger.debug(f"Checkpoint cleanup error: {exc}")
return cleaned
async def _broadcast_status(self) -> None:
"""Push updated recovery status to all connected clients."""
try:
status = await self.get_recovery_status()
await self._broadcast({"type": "recovery_status", "payload":
_serialize_status(status, project_id=self._engine.project_id or "default")
})
except Exception as exc:
logger.debug(f"Recovery status broadcast failed: {exc}")
def _invalidate_cache(self) -> None:
"""Force next get_recovery_status to re-scan."""
self._cache_until = 0.0
self._cached = None
async def _set_run_recovery_state(
self,
key: str,
*,
status: str,
lifecycle_status: str | None = None,
match_task_id: bool = False,
extra: dict[str, Any] | None = None,
) -> None:
store = getattr(self._engine, "store", None)
if not store or not hasattr(store, "list_delegation_runs") or not hasattr(store, "save_delegation_run"):
return
runs = await store.list_delegation_runs(project_id=self._engine.project_id or "default")
target = None
if match_task_id:
for run in runs:
metadata = dict(getattr(run, "metadata", {}) or {})
if str(metadata.get("origin_task_id", "") or "").strip() == key:
target = run
break
if target is None:
for run in runs:
if str(run.session_id or "").strip() == key:
target = run
break
if target is None:
return
target.recovery_pointer = {
**dict(getattr(target, "recovery_pointer", {}) or {}),
"status": status,
"updated_at": datetime.now().isoformat(),
**dict(extra or {}),
}
if lifecycle_status:
target.lifecycle_status = lifecycle_status
target.updated_at = datetime.now()
await store.save_delegation_run(target)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _is_interrupted(task: Any) -> bool:
"""Check if a task was interrupted by crash."""
from opc.core.models import TaskStatus
if task.status != TaskStatus.FAILED:
return False
meta = getattr(task, "metadata", {}) or {}
if meta.get("interrupted_recovery"):
return True
result = getattr(task, "result", {}) or {}
artifacts = result.get("artifacts", {}) or {}
return bool(artifacts.get("interrupted"))
def _serialize_status(status: RecoveryStatus, *, project_id: str | None = None) -> dict[str, Any]:
"""Convert RecoveryStatus to JSON-safe dict."""
resolved_project_id = str(project_id or "").strip()
if not resolved_project_id:
for item in status.interrupted:
if item.project_id:
resolved_project_id = item.project_id
break
payload = {
"interrupted": [
{
"parent_session_id": w.parent_session_id,
"parent_task_id": w.parent_task_id,
"project_id": w.project_id,
"title": w.title,
"profile": w.profile,
"interrupted_at": w.interrupted_at,
"work_items": [
{
"work_item_projection_id": s.work_item_projection_id,
"title": s.title,
"task_id": s.task_id,
"status": s.status,
"interrupted": s.interrupted,
"previous_status": s.previous_status,
}
for s in w.work_items
],
}
for w in status.interrupted
],
"active_recoveries": status.active_recoveries,
"scanned_at": status.scanned_at,
}
if resolved_project_id:
payload["project_id"] = resolved_project_id
return payload
-11
View File
@@ -144,17 +144,6 @@ async def create_app(
engine.event_bus.subscribe_all(_root_engine_event)
# ── Runtime crash recovery (pluggable, no engine modifications) ──
from opc.plugins.office_ui.recovery_manager import RuntimeRecoveryManager
recovery_manager = RuntimeRecoveryManager(engine, ws_handler.broadcast)
ws_handler.recovery_manager = recovery_manager
# ── Startup self-heal for tasks abandoned by a prior process ─────
# Must run before restoring persisted mode and before any WS client can
# connect, so orphaned running/locked rows do not block new Continue /
# session_send acquisitions.
await ws_handler.heal_orphan_tasks_on_boot()
# ── Restore persisted mode and load matching agents on startup ───
await ws_handler.restore_persisted_mode()
startup_preset = ws_handler._resolve_preset_name()
+10 -1
View File
@@ -32,4 +32,13 @@ class ServiceError(Exception):
self.payload = dict(payload or {})
def to_payload(self) -> dict[str, Any]:
return {"error": self.message, "code": self.code, **self.payload}
# Transport envelope fields are authoritative. Business details may
# add context, but must never turn an error acknowledgement into a
# success or replace the exception's code/message.
payload = {
key: value
for key, value in self.payload.items()
if key not in {"ok", "error", "code"}
}
payload.update({"error": self.message, "code": self.code})
return payload
-45
View File
@@ -200,48 +200,3 @@ class RuntimeService:
payload["display_text"] = " | ".join(display_parts)
payload["event_type"] = event_type
return payload
async def recovery_scan(self, *, project_id: str) -> ServiceResult:
manager = await self._recovery_manager(project_id)
from opc.plugins.office_ui.recovery_manager import _serialize_status
status = await manager.get_recovery_status()
return ServiceResult(_serialize_status(status, project_id=project_id))
async def recovery_action(self, *, project_id: str, action: str, parent_task_id: str) -> ServiceResult:
manager = await self._recovery_manager(project_id)
normalized = str(action or "").strip().lower()
if normalized == "scan":
return await self.recovery_scan(project_id=project_id)
if not str(parent_task_id or "").strip():
raise ServiceError("parent_task_id_required", "parent_task_id required")
if normalized in {"resume", "retry"}:
payload = await manager.resume(parent_task_id)
elif normalized == "cancel":
payload = await manager.cancel(parent_task_id)
else:
raise ServiceError("unknown_recovery_action", f"unknown action: {action}", {"action": action})
payload = {**dict(payload), "project_id": project_id, "parent_task_id": parent_task_id, "action": normalized}
if not payload.get("ok", False):
raise ServiceError(str(payload.get("error") or "recovery_failed"), str(payload.get("error") or "recovery_failed"), payload)
return ServiceResult(payload)
async def _recovery_manager(self, project_id: str) -> Any:
engine = await self.context.engine_for_project(project_id)
async def _noop_broadcast(_event: dict[str, Any]) -> None:
return None
managers = getattr(self.context, "recovery_managers", None)
if managers is None:
managers = {}
setattr(self.context, "recovery_managers", managers)
key = self.context.normalize_project_id(project_id)
existing = managers.get(key)
if existing is not None and getattr(existing, "_engine", None) is engine:
return existing
from opc.plugins.office_ui.recovery_manager import RuntimeRecoveryManager
manager = RuntimeRecoveryManager(engine, _noop_broadcast)
managers[key] = manager
return manager
+305 -182
View File
@@ -12,6 +12,12 @@ from typing import Any
from loguru import logger
from opc.core.models import Task, TaskStatus
from opc.layer2_organization.company_runtime_identity import (
ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES,
COMPANY_RUNTIME_CHECKPOINT_TYPES,
is_company_runtime_task,
load_company_runtime_identity_index,
)
from opc.plugins.office_ui.execution_identity import (
ExecutionIdentity,
canonicalize_execution_identity,
@@ -156,6 +162,32 @@ class SessionService:
task_project = self.context.normalize_project_id(getattr(task, "project_id", None))
if task_project != pid:
raise ServiceError("target_wrong_project", "Target belongs to a different project", {"project_id": task_project})
# Company control is resolved from durable session/checkpoint scope
# before any generic Task/session fallback. This prevents a shared
# final-decider row from winning merely because it was loaded first.
identity_index = await load_company_runtime_identity_index(store, pid)
if task is not None:
identity = identity_index.resolve(task_id=raw_target)
else:
identity = (
identity_index.resolve(runtime_session_id=raw_target)
or identity_index.resolve(task_session_id=raw_target)
)
if identity is not None:
# A concrete Task id is the caller's UI/chat channel, never the
# company execution identity. Keep that channel stable while the
# shared identity supplies the durable runtime session/checkpoint.
if task is not None:
return task, identity.runtime_session_id
resolved_task = (
identity_index.task(identity.ui_anchor_task_id)
or identity_index.task(identity.config_source_task_id)
)
if resolved_task is not None:
return resolved_task, identity.runtime_session_id
if task is not None:
return task, str(getattr(task, "session_id", "") or getattr(task, "parent_session_id", "") or "")
session = await store.get_session(raw_target) if hasattr(store, "get_session") else None
@@ -165,139 +197,64 @@ class SessionService:
if session_project != pid:
raise ServiceError("target_wrong_project", "Target belongs to a different project", {"project_id": session_project})
tasks = await store.get_tasks(project_id=pid) if hasattr(store, "get_tasks") else []
tasks = list(identity_index.tasks)
session_tasks = [
candidate for candidate in tasks
if str(getattr(candidate, "session_id", "") or "") == raw_target
]
if not session_tasks:
raise ServiceError("session_not_task_backed", "Session is not linked to a task-backed runtime", {"session_id": raw_target})
session_tasks.sort(key=lambda item: bool(str(getattr(item, "parent_session_id", "") or "")))
return session_tasks[0], raw_target
task_mode_anchor = min(
session_tasks,
key=lambda item: (
bool(str(getattr(item, "parent_session_id", "") or "")),
str(getattr(item, "created_at", "") or ""),
str(getattr(item, "id", "") or ""),
),
)
return task_mode_anchor, raw_target
async def _resolve_company_runtime_target(self, engine: Any, task: Any) -> dict[str, Any]:
store = getattr(engine, "store", None)
parent_session_id = str(
getattr(task, "parent_session_id", "")
or getattr(task, "session_id", "")
or ""
).strip()
parent_task_id = str(self.context.session_to_task.get(parent_session_id) or "").strip()
project_id = self.context.normalize_project_id(getattr(task, "project_id", None) or getattr(engine, "project_id", None))
try:
project_tasks = await store.get_tasks(project_id=project_id) if hasattr(store, "get_tasks") else [task]
except Exception:
project_tasks = [task]
for candidate in project_tasks:
candidate_id = str(getattr(candidate, "id", "") or "").strip()
candidate_session_id = str(getattr(candidate, "session_id", "") or "").strip()
candidate_parent_session_id = str(getattr(candidate, "parent_session_id", "") or "").strip()
if candidate_session_id == parent_session_id and not candidate_parent_session_id:
parent_task_id = candidate_id
break
if not parent_task_id:
parent_task_id = str(
self.context.active_runtime_children.get(str(getattr(task, "id", "") or ""))
or getattr(task, "id", "")
or ""
).strip()
affected_task_ids: list[str] = []
for candidate in project_tasks:
candidate_id = str(getattr(candidate, "id", "") or "").strip()
if not candidate_id:
continue
candidate_session_id = str(getattr(candidate, "session_id", "") or "").strip()
candidate_parent_session_id = str(getattr(candidate, "parent_session_id", "") or "").strip()
if (
candidate_id == str(getattr(task, "id", "") or "")
or candidate_id == parent_task_id
or candidate_session_id == parent_session_id
or candidate_parent_session_id == parent_session_id
):
if candidate_id not in affected_task_ids:
affected_task_ids.append(candidate_id)
for child_id, origin_id in list(self.context.active_runtime_children.items()):
if origin_id == parent_task_id or child_id == str(getattr(task, "id", "") or ""):
if child_id not in affected_task_ids:
affected_task_ids.append(child_id)
if parent_task_id and parent_task_id not in affected_task_ids:
affected_task_ids.insert(0, parent_task_id)
return {
"parent_session_id": parent_session_id,
"parent_task_id": parent_task_id or str(getattr(task, "id", "") or ""),
"origin_task_id": parent_task_id or str(getattr(task, "id", "") or ""),
"affected_task_ids": affected_task_ids or [str(getattr(task, "id", "") or "")],
}
async def _mark_company_runtime_stop_state(
async def _resolve_company_runtime_target(
self,
*,
engine: Any,
task_ids: list[str],
state: str,
stop_intent_id: str,
checkpoint_type: str = "",
) -> None:
task: Any,
*,
runtime_session_id: str = "",
checkpoint_id: str = "",
) -> dict[str, Any]:
store = getattr(engine, "store", None)
project_id = self.context.normalize_project_id(getattr(task, "project_id", None) or getattr(engine, "project_id", None))
if not self.context.store_is_ready(store):
return
for task_id in task_ids:
try:
task = await store.get_task(str(task_id))
except Exception:
task = None
if not task or self._is_terminal_status(task):
continue
metadata = dict(getattr(task, "metadata", {}) or {})
metadata["company_runtime_stop_state"] = state
metadata["company_runtime_stop_intent_id"] = stop_intent_id
metadata["company_runtime_stop_marked_at"] = datetime.now().isoformat()
metadata["dispatch_hold"] = "company_runtime_suspended"
metadata["company_runtime_suspended_at"] = datetime.now().isoformat()
if checkpoint_type:
metadata["company_runtime_suspend_checkpoint_type"] = checkpoint_type
metadata.setdefault("suspended_task_status", self._task_status_value(task))
task.metadata = metadata
task.status = TaskStatus.BLOCKED
if hasattr(task, "execution_lock"):
task.execution_lock = False
if hasattr(task, "execution_locked_at"):
task.execution_locked_at = None
try:
await store.save_task(task)
except Exception:
logger.opt(exception=True).debug("failed to mark company runtime stop state")
async def _clear_company_runtime_stop_state(self, *, engine: Any, task_ids: list[str]) -> None:
store = getattr(engine, "store", None)
if not self.context.store_is_ready(store):
return
for task_id in task_ids:
try:
task = await store.get_task(str(task_id))
except Exception:
task = None
if not task:
continue
metadata = dict(getattr(task, "metadata", {}) or {})
for key in (
"dispatch_hold",
"company_runtime_stop_state",
"company_runtime_stop_intent_id",
"company_runtime_stop_marked_at",
"company_runtime_suspend_checkpoint_type",
"company_runtime_suspended_at",
"suspended_task_status",
):
metadata.pop(key, None)
task.metadata = metadata
if self._task_status_value(task) == "blocked":
task.status = TaskStatus.IDLE
try:
await store.save_task(task)
except Exception:
logger.opt(exception=True).debug("failed to clear company runtime stop state")
raise ServiceError("store_not_ready", "store_not_ready", {"project_id": project_id})
index = await load_company_runtime_identity_index(store, project_id)
identity = index.resolve(
task_id=str(getattr(task, "id", "") or ""),
runtime_session_id=runtime_session_id,
checkpoint_id=checkpoint_id,
)
if identity is None:
raise ServiceError(
"company_runtime_identity_mismatch",
"Company runtime identity does not match the requested task/session/checkpoint",
{
"task_id": str(getattr(task, "id", "") or ""),
"runtime_session_id": str(runtime_session_id or ""),
"checkpoint_id": str(checkpoint_id or ""),
},
)
config_task = index.task(identity.config_source_task_id) or task
ui_anchor_task_id = identity.ui_anchor_task_id
return {
"identity": identity,
"runtime_session_id": identity.runtime_session_id,
"ui_channel_task_id": str(getattr(task, "id", "") or ""),
"ui_anchor_task_id": ui_anchor_task_id,
"config_source_task_id": identity.config_source_task_id,
"config_task": config_task,
"origin_task_id": ui_anchor_task_id,
"affected_task_ids": list(identity.runtime_task_ids),
"checkpoint": identity.checkpoint,
}
def _normalize_requested_config(
self,
@@ -743,8 +700,54 @@ class SessionService:
if getattr(engine, "memory", None):
await engine.memory.ensure_session(task.session_id, project_id=project_id, title=task.title, mode="primary", metadata={"source": "service"})
await store.save_task(task)
# A Task selected by the UI/CLI is only the chat channel for company
# mode. Resolve the durable runtime scope before choosing execution
# configuration, session, origin, or checkpoint. In particular, a
# role/work-item Task must never become the parent execution identity.
company_target: dict[str, Any] | None = None
task_is_company_runtime = is_company_runtime_task(task)
try:
company_target = await self._resolve_company_runtime_target(engine, task)
except ServiceError as exc:
if exc.code != "company_runtime_identity_mismatch" or task_is_company_runtime:
raise
if task.status == TaskStatus.CANCELLED:
company_identity = (
company_target.get("identity")
if company_target is not None
else None
)
checkpoint = (
company_target.get("checkpoint")
if company_target is not None
else None
)
active_cancelled_anchor = bool(
company_identity is not None
and str(getattr(company_identity, "ui_anchor_task_id", "") or "").strip()
== str(getattr(task, "id", "") or "").strip()
and checkpoint is not None
and str(getattr(checkpoint, "checkpoint_type", "") or "").strip()
in COMPANY_RUNTIME_CHECKPOINT_TYPES
and str(getattr(checkpoint, "status", "") or "").strip().lower()
in ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES
)
if not active_cancelled_anchor:
raise ServiceError(
"session_ended",
"session_ended",
{"project_id": project_id, "task_id": task.id},
)
config_task = (
company_target.get("config_task")
if company_target is not None
else task
) or task
identity = self.resolve_task_identity(
task,
config_task,
default_exec_mode=mode,
default_company_profile=company_profile if company_profile is not None else "corporate",
default_preferred_agent=preferred_agent if preferred_agent is not None else "native",
@@ -752,26 +755,85 @@ class SessionService:
)
if identity.is_custom_org and not identity.org_id:
raise ServiceError("org_id_required", "org_id_required", {"project_id": project_id, "task_id": task.id})
await self.persist_session_config(
task,
exec_mode=identity.exec_mode,
company_profile=identity.company_profile,
preferred_agent=identity.preferred_agent,
org_id=identity.org_id,
engine=engine,
)
# Existing company scopes read configuration from the resolver's
# config source. Only persist when that source is the selected Task;
# this preserves normal session configuration while avoiding writes to
# an internal work item merely because it was used as the UI channel.
if (
company_target is None
or str(getattr(config_task, "id", "") or "").strip()
== str(getattr(task, "id", "") or "").strip()
):
await self.persist_session_config(
task,
exec_mode=identity.exec_mode,
company_profile=identity.company_profile,
preferred_agent=identity.preferred_agent,
org_id=identity.org_id,
engine=engine,
)
execution_session_id = str(getattr(task, "session_id", "") or "").strip()
origin_task_id = str(getattr(task, "id", "") or "").strip() or None
message_metadata: dict[str, Any] | None = None
if company_target is not None:
execution_session_id = str(
company_target.get("runtime_session_id", "") or ""
).strip()
origin_task_id = str(
company_target.get("ui_anchor_task_id", "") or ""
).strip() or None
if not execution_session_id:
raise ServiceError(
"company_runtime_identity_mismatch",
"Company runtime has no canonical runtime session",
{"project_id": project_id, "task_id": task.id},
)
checkpoint = company_target.get("checkpoint")
if checkpoint is not None:
checkpoint_id = str(
getattr(checkpoint, "checkpoint_id", "") or ""
).strip()
checkpoint_status = str(
getattr(checkpoint, "status", "") or ""
).strip().lower()
if checkpoint_status != "pending":
raise ServiceError(
"company_runtime_checkpoint_not_pending",
"Company runtime checkpoint is not pending",
{
"project_id": project_id,
"task_id": task.id,
"checkpoint_id": checkpoint_id,
"checkpoint_status": checkpoint_status,
},
)
message_metadata = {
"response_to_checkpoint_id": checkpoint_id,
"response_to_checkpoint_type": str(
getattr(checkpoint, "checkpoint_type", "") or ""
).strip(),
}
response = await engine.process_message(
str(content or "").strip(),
project_id=project_id,
session_id=task.session_id,
session_id=execution_session_id,
mode=identity.exec_mode,
org_id=identity.org_id or None,
company_profile=identity.company_profile if identity.is_company_runtime else None,
preferred_agent=identity.preferred_agent if identity.is_task else None,
domains=list(domains or []),
origin_task_id=task.id,
origin_task_id=origin_task_id,
message_metadata=message_metadata,
)
return ServiceResult({"project_id": project_id, "task_id": task.id, "session_id": task.session_id, "response": response})
return ServiceResult({
"project_id": project_id,
"task_id": task.id,
"session_id": execution_session_id,
"response": response,
})
async def rename(self, *, project_id: str, task_id: str = "", session_id: str = "", title: str) -> ServiceResult:
pid = self.context.normalize_project_id(project_id)
@@ -935,50 +997,44 @@ class SessionService:
default_payload=default_payload,
)
engine = await self.context.engine_for_project(project_id)
exec_mode, _company_profile = self.resolve_task_session_config(task)
if exec_mode in {"company", "org", "custom"}:
try:
target_info = await self._resolve_company_runtime_target(engine, task)
except ServiceError as exc:
if exc.code != "company_runtime_identity_mismatch":
raise
target_info = None
if target_info is None and is_company_runtime_task(task):
raise ServiceError(
"company_runtime_identity_mismatch",
"Company runtime identity could not be resolved; refusing task-mode cancellation",
{"task_id": resolved_task_id, "session_id": resolved_session_id},
)
if target_info is not None:
stop_intent_id = str(uuid.uuid4())
affected_task_ids = list(target_info.get("affected_task_ids", []) or [resolved_task_id])
suspended: dict[str, Any] | None = None
suspend = getattr(engine, "suspend_company_runtime", None)
await self._mark_company_runtime_stop_state(
engine=engine,
task_ids=affected_task_ids,
state="suspending",
stop_intent_id=stop_intent_id,
)
if callable(suspend):
try:
suspended = await suspend(
origin_task_id=str(target_info.get("origin_task_id", "") or resolved_task_id),
session_id=(str(target_info.get("parent_session_id", "") or resolved_session_id).strip() or None),
session_id=(str(target_info.get("runtime_session_id", "") or resolved_session_id).strip() or None),
reason="user_stop",
checkpoint_type="company_runtime_suspended",
stop_intent_id=stop_intent_id,
)
except Exception:
logger.opt(exception=True).warning("suspend_company_runtime failed during service stop")
if suspended is not None:
for candidate in list(suspended.get("task_ids", []) or []):
candidate_id = str(candidate or "").strip()
if candidate_id and candidate_id not in affected_task_ids:
affected_task_ids.append(candidate_id)
await self._mark_company_runtime_stop_state(
engine=engine,
task_ids=affected_task_ids,
state="suspended",
stop_intent_id=stop_intent_id,
checkpoint_type=str(suspended.get("checkpoint_type", "") or "company_runtime_suspended"),
)
else:
await self._mark_company_runtime_stop_state(
engine=engine,
task_ids=affected_task_ids,
state="suspended",
stop_intent_id=stop_intent_id,
checkpoint_type="company_runtime_suspended",
if suspended is None:
raise ServiceError(
"company_runtime_suspend_failed",
"Company runtime could not be suspended",
{"runtime_session_id": target_info.get("runtime_session_id", "")},
)
for candidate in list(suspended.get("task_ids", []) or []):
candidate_id = str(candidate or "").strip()
if candidate_id and candidate_id not in affected_task_ids:
affected_task_ids.append(candidate_id)
self.context.stop_requested_task_ids.update(affected_task_ids)
if self.context.cancel_session_tasks is not None:
for tid in affected_task_ids:
@@ -1009,8 +1065,7 @@ class SessionService:
"stop_intent_id": stop_intent_id,
"checkpoint_id": str((suspended or {}).get("checkpoint_id", "") or ""),
"task_ids": affected_task_ids,
"resume_parent_task_id": str(target_info.get("parent_task_id", "") or resolved_task_id),
"resume_parent_session_id": str(target_info.get("parent_session_id", "") or resolved_session_id),
"resume_parent_session_id": str(target_info.get("runtime_session_id", "") or resolved_session_id),
}
return ServiceResult(payload, [ServiceEvent("session_runtime_control", payload), ServiceEvent("session_updated", payload)])
@@ -1039,6 +1094,8 @@ class SessionService:
project_id: str,
task_id: str = "",
session_id: str = "",
runtime_session_id: str = "",
checkpoint_id: str = "",
target: str = "",
content: str = "",
) -> ServiceResult:
@@ -1066,45 +1123,111 @@ class SessionService:
default_payload=default_payload,
)
engine = await self.context.engine_for_project(project_id)
exec_mode, company_profile = self.resolve_task_session_config(task)
target_info = await self._resolve_company_runtime_target(engine, task) if exec_mode in {"company", "org", "custom"} else {
try:
company_target = await self._resolve_company_runtime_target(
engine,
task,
runtime_session_id=runtime_session_id,
checkpoint_id=checkpoint_id,
)
except ServiceError as exc:
if exc.code != "company_runtime_identity_mismatch" or runtime_session_id or checkpoint_id:
raise
if is_company_runtime_task(task):
raise
company_target = None
target_info = company_target or {
"affected_task_ids": [resolved_task_id],
"parent_task_id": resolved_task_id,
"parent_session_id": resolved_session_id,
"ui_anchor_task_id": resolved_task_id,
"runtime_session_id": resolved_session_id,
"config_task": task,
"checkpoint": None,
}
affected_task_ids = list(target_info.get("affected_task_ids", []) or [resolved_task_id])
await self._clear_company_runtime_stop_state(engine=engine, task_ids=affected_task_ids)
message = str(content or "").strip() or "Resume the existing runtime."
engine_mode = "company" if exec_mode == "company" else ("org" if exec_mode in {"org", "custom"} else "task")
org_id = self.resolve_task_org_id(task) if engine_mode == "org" else ""
config_task = target_info.get("config_task") or task
config_exec_mode, config_company_profile = self.resolve_task_session_config(config_task)
engine_mode = "company" if config_exec_mode == "company" else (
"org" if config_exec_mode in {"org", "custom"} else "task"
)
if engine_mode in {"company", "org"}:
checkpoint = target_info.get("checkpoint")
if checkpoint is None:
raise ServiceError("company_runtime_checkpoint_not_found", "No active company runtime checkpoint")
if str(getattr(checkpoint, "status", "") or "").strip().lower() != "pending":
raise ServiceError(
"company_runtime_checkpoint_not_pending",
"Company runtime checkpoint is not pending",
{"checkpoint_id": str(getattr(checkpoint, "checkpoint_id", "") or "")},
)
else:
checkpoint = None
org_id = self.resolve_task_org_id(config_task) if engine_mode == "org" else ""
message_metadata: dict[str, Any] = {"ui_force_resume": True}
if checkpoint is not None:
message_metadata.update({
"response_to_checkpoint_id": str(getattr(checkpoint, "checkpoint_id", "") or ""),
"response_to_checkpoint_type": str(getattr(checkpoint, "checkpoint_type", "") or ""),
})
response = await engine.process_message(
message,
project_id=self.context.normalize_project_id(project_id),
session_id=str(target_info.get("parent_session_id", "") or resolved_session_id),
session_id=str(target_info.get("runtime_session_id", "") or resolved_session_id),
mode=engine_mode,
org_id=org_id or None,
company_profile=company_profile if engine_mode == "company" else None,
preferred_agent=self.resolve_task_preferred_agent(task) if engine_mode == "task" else None,
origin_task_id=str(target_info.get("parent_task_id", "") or resolved_task_id),
message_metadata={"ui_force_resume": True},
company_profile=config_company_profile if engine_mode == "company" else None,
preferred_agent=self.resolve_task_preferred_agent(config_task) if engine_mode == "task" else None,
origin_task_id=(str(target_info.get("ui_anchor_task_id", "") or "").strip() or None),
message_metadata=message_metadata,
)
runtime_control_state = "idle"
pending_runtime_checkpoint_id = ""
if engine_mode in {"company", "org"}:
refreshed_index = await load_company_runtime_identity_index(
engine.store,
self.context.normalize_project_id(project_id),
)
refreshed_identity = refreshed_index.resolve(
runtime_session_id=str(
target_info.get("runtime_session_id", "") or resolved_session_id
),
)
if refreshed_identity is not None and refreshed_identity.pending_checkpoint_id:
pending_runtime_checkpoint_id = refreshed_identity.pending_checkpoint_id
runtime_control_state = (
"suspended"
if refreshed_identity.pending_checkpoint_status == "pending"
else "resuming"
)
payload = {
**default_payload,
"status": "resuming",
"runtime_control_state": "resuming",
"can_resume": False,
"status": runtime_control_state,
"runtime_control_state": runtime_control_state,
"can_resume": runtime_control_state == "suspended",
"response": response,
"task_ids": affected_task_ids,
"resume_parent_task_id": str(target_info.get("parent_task_id", "") or resolved_task_id),
"resume_parent_session_id": str(target_info.get("parent_session_id", "") or resolved_session_id),
"resume_parent_session_id": str(target_info.get("runtime_session_id", "") or resolved_session_id),
"pending_runtime_checkpoint_id": pending_runtime_checkpoint_id,
}
return ServiceResult(payload, [ServiceEvent("session_runtime_control", payload), ServiceEvent("session_updated", payload)])
async def resume(self, *, project_id: str, task_id: str = "", session_id: str = "", target: str = "", content: str = "") -> ServiceResult:
async def resume(
self,
*,
project_id: str,
task_id: str = "",
session_id: str = "",
runtime_session_id: str = "",
checkpoint_id: str = "",
target: str = "",
content: str = "",
) -> ServiceResult:
return await self.continue_run(
project_id=project_id,
task_id=task_id,
session_id=session_id,
runtime_session_id=runtime_session_id,
checkpoint_id=checkpoint_id,
target=target,
content=content,
)
+71 -80
View File
@@ -39,6 +39,11 @@ from opc.layer2_organization.phase import (
should_hide_work_item_from_company_kanban,
verdict,
)
from opc.layer2_organization.company_runtime_identity import (
ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES,
COMPANY_RUNTIME_CHECKPOINT_TYPES,
build_company_runtime_identity_index,
)
from opc.layer2_organization.work_item_context_view import WorkItemContextView
from opc.layer2_organization.work_item_identity import (
WORK_ITEM_PROJECTION_ID_KEY,
@@ -1503,6 +1508,11 @@ def _primary_session_tasks_by_session_id(
*,
task_meta_map: dict[str, dict[str, Any]] | None = None,
) -> tuple[dict[str, Any], list[str]]:
identity_index = build_company_runtime_identity_index(tasks)
company_identities = {
identity.runtime_session_id: identity
for identity in identity_index.identities
}
primary_tasks_by_session_id: dict[str, Any] = {}
ordered_session_ids: list[str] = []
for task in tasks:
@@ -1513,6 +1523,19 @@ def _primary_session_tasks_by_session_id(
if bool(task_meta.get("review_task", False)):
continue
session_id = str(getattr(task, "session_id", "") or "").strip()
company_identity = company_identities.get(session_id)
if company_identity is not None:
anchor = identity_index.task(company_identity.ui_anchor_task_id)
if anchor is not None and session_id not in primary_tasks_by_session_id:
primary_tasks_by_session_id[session_id] = anchor
ordered_session_ids.append(session_id)
if anchor is not None:
# A shared final-decider/work-item Task must never replace a
# pure UI anchor that owns the same session id.
continue
# Without a pure anchor this scope has no primary chat container.
# Never synthesize one from a role/work-item Task.
continue
if not session_id or _task_parent_session_link(task, task_meta):
continue
current = primary_tasks_by_session_id.get(session_id)
@@ -1530,31 +1553,16 @@ def _primary_session_tasks_by_session_id(
return primary_tasks_by_session_id, ordered_session_ids
def _shared_role_identity_tasks_by_session_id(
def _company_config_source_tasks_by_session_id(
tasks: list[Any],
*,
task_meta_map: dict[str, dict[str, Any]] | None = None,
) -> dict[str, Any]:
identity_tasks_by_session_id: dict[str, Any] = {}
for task in tasks:
task_id = str(getattr(task, "id", "") or "").strip()
task_meta = (
task_meta_map.get(task_id, {}) if task_meta_map is not None and task_id else _task_metadata(task)
)
session_id = _shared_role_session_key(task, task_meta)
if not session_id:
continue
current = identity_tasks_by_session_id.get(session_id)
if current is None:
identity_tasks_by_session_id[session_id] = task
continue
current_id = str(getattr(current, "id", "") or "").strip()
current_meta = (
task_meta_map.get(current_id, {}) if task_meta_map is not None and current_id else _task_metadata(current)
)
if _session_representative_rank(task, task_meta) > _session_representative_rank(current, current_meta):
identity_tasks_by_session_id[session_id] = task
return identity_tasks_by_session_id
identity_index = build_company_runtime_identity_index(tasks)
return {
identity.runtime_session_id: task
for identity in identity_index.identities
if identity.config_source_task_id
and (task := identity_index.task(identity.config_source_task_id)) is not None
}
async def build_company_kanban_projection(
@@ -2594,31 +2602,7 @@ async def _build_company_runtime_control_by_task(
if not store:
return {}
parent_task_by_session: dict[str, str] = {}
tasks_by_parent_session: dict[str, list[Any]] = {}
for task in tasks:
metadata = dict(getattr(task, "metadata", {}) or {})
mode = str(metadata.get("mode", "") or metadata.get("exec_mode", "") or "").strip().lower()
is_company_runtime_task = bool(
mode in {"company", "org", "custom"}
or str(getattr(task, "parent_session_id", "") or "").strip()
or metadata.get("company_profile")
or metadata.get("company_work_item_plan")
or metadata.get("work_item_runtime")
or metadata.get("work_item_projection_id")
)
if not is_company_runtime_task:
continue
session_id = str(getattr(task, "session_id", "") or "").strip()
parent_session_id = str(getattr(task, "parent_session_id", "") or "").strip()
task_id = str(getattr(task, "id", "") or "").strip()
if session_id and not parent_session_id:
parent_task_by_session[session_id] = task_id
runtime_parent_session_id = parent_session_id or session_id
if runtime_parent_session_id:
tasks_by_parent_session.setdefault(runtime_parent_session_id, []).append(task)
checkpoints_by_session: dict[str, Any] = {}
checkpoints: list[Any] = []
getter = getattr(store, "get_execution_checkpoints", None)
if not callable(getter):
getter = getattr(store, "get_pending_checkpoints", None)
@@ -2626,29 +2610,25 @@ async def _build_company_runtime_control_by_task(
try:
kwargs = {
"project_id": project_id,
"checkpoint_types": ["company_runtime_suspended", "company_runtime_interrupted"],
"checkpoint_types": sorted(COMPANY_RUNTIME_CHECKPOINT_TYPES),
}
if getattr(getter, "__name__", "") == "get_execution_checkpoints":
kwargs["statuses"] = ["pending", "resuming"]
kwargs["statuses"] = sorted(ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES)
checkpoints = await getter(**kwargs)
for checkpoint in checkpoints:
sid = str(getattr(checkpoint, "session_id", "") or "").strip()
if sid and sid not in checkpoints_by_session:
checkpoints_by_session[sid] = checkpoint
except Exception:
logger.opt(exception=True).debug("snapshot: failed to load company runtime checkpoints")
checkpoints = []
identity_index = build_company_runtime_identity_index(tasks, checkpoints)
result: dict[str, dict[str, Any]] = {}
for parent_session_id, group in tasks_by_parent_session.items():
checkpoint = checkpoints_by_session.get(parent_session_id)
parent_task_id = parent_task_by_session.get(parent_session_id, "")
if not parent_task_id:
for task in group:
if not str(getattr(task, "parent_session_id", "") or "").strip():
parent_task_id = str(getattr(task, "id", "") or "").strip()
break
if not parent_task_id and group:
parent_task_id = str(getattr(group[0], "id", "") or "").strip()
for identity in identity_index.identities:
group = [
task
for task_id in identity.runtime_task_ids
if (task := identity_index.task(task_id)) is not None
]
checkpoint = identity.checkpoint
def _task_status_value(task: Any) -> str:
status = getattr(task, "status", "")
@@ -2660,10 +2640,19 @@ async def _build_company_runtime_control_by_task(
task for task in group
if _task_status_value(task) not in {"done", "failed", "cancelled"}
]
has_running_task = any(
_task_status_value(task) == "running"
for task in non_terminal_group
)
# Persisted RUNNING is only a projection. The controller-local
# execution registry is the sole proof that this process still owns a
# coroutine capable of monitoring and persisting the run.
runtime_is_live = getattr(engine, "_task_runtime_is_live", None)
has_running_task = False
if callable(runtime_is_live):
for task in non_terminal_group:
live_result = runtime_is_live(task)
if inspect.isawaitable(live_result):
live_result = await live_result
if live_result is True:
has_running_task = True
break
any_stop_in_progress = any(
str((getattr(task, "metadata", {}) or {}).get("company_runtime_stop_state", "") or "").strip()
in {"suspending", "suspended", "resuming_after_suspending"}
@@ -2708,8 +2697,7 @@ async def _build_company_runtime_control_by_task(
"runtime_control_state": state,
"can_stop": state == "running",
"can_resume": state == "suspended",
"resume_parent_task_id": parent_task_id,
"resume_parent_session_id": parent_session_id,
"resume_parent_session_id": identity.runtime_session_id,
"pending_runtime_checkpoint_id": pending_checkpoint_id,
"stop_intent_id": str(checkpoint_payload.get("stop_intent_id", "") or ""),
}
@@ -3014,9 +3002,8 @@ async def build_project_index_sync(
session_tasks,
task_meta_map=task_meta_map,
)
shared_identity_tasks_by_session_id = _shared_role_identity_tasks_by_session_id(
company_config_tasks_by_session_id = _company_config_source_tasks_by_session_id(
session_tasks,
task_meta_map=task_meta_map,
)
child_tasks_by_parent: dict[str, list[Any]] = {}
for task in session_tasks:
@@ -3087,11 +3074,12 @@ async def build_project_index_sync(
representative_task = primary_tasks_by_session_id.get(session_id)
representative_task_id = str(getattr(representative_task, "id", "") or "").strip()
shared_session_id = _shared_role_session_key(t, t_meta)
if shared_session_id and representative_task_id and representative_task_id != task_id:
continue
if shared_session_id:
if not representative_task_id or representative_task_id != task_id:
continue
identity_task = t
identity_meta = t_meta
shared_identity_task = shared_identity_tasks_by_session_id.get(session_id)
shared_identity_task = company_config_tasks_by_session_id.get(session_id)
shared_identity_task_id = str(getattr(shared_identity_task, "id", "") or "").strip()
if shared_identity_task_id and shared_identity_task_id != task_id:
identity_task = shared_identity_task
@@ -3569,9 +3557,8 @@ async def build_collab_sync(
session_tasks,
task_meta_map=task_meta_map,
)
shared_identity_tasks_by_session_id = _shared_role_identity_tasks_by_session_id(
company_config_tasks_by_session_id = _company_config_source_tasks_by_session_id(
session_tasks,
task_meta_map=task_meta_map,
)
child_tasks_by_parent: dict[str, list[Any]] = {}
for task in session_tasks:
@@ -3616,11 +3603,15 @@ async def build_collab_sync(
representative_task = primary_tasks_by_session_id.get(session_id)
representative_task_id = str(getattr(representative_task, "id", "") or "").strip()
shared_session_id = _shared_role_session_key(t, t_meta)
if shared_session_id and representative_task_id and representative_task_id != str(getattr(t, "id", "") or "").strip():
continue
if shared_session_id:
if (
not representative_task_id
or representative_task_id != str(getattr(t, "id", "") or "").strip()
):
continue
identity_task = t
identity_meta = t_meta
shared_identity_task = shared_identity_tasks_by_session_id.get(session_id)
shared_identity_task = company_config_tasks_by_session_id.get(session_id)
shared_identity_task_id = str(getattr(shared_identity_task, "id", "") or "").strip()
if shared_identity_task_id and shared_identity_task_id != str(getattr(t, "id", "") or "").strip():
identity_task = shared_identity_task
@@ -51,8 +51,11 @@ class CompanyKanbanProjectionTests(unittest.IsolatedAsyncioTestCase):
},
)
store = MagicMock()
store.get_pending_checkpoints = AsyncMock(return_value=[])
engine = SimpleNamespace(store=store)
store.get_execution_checkpoints = AsyncMock(return_value=[])
engine = SimpleNamespace(
store=store,
_task_runtime_is_live=AsyncMock(return_value=True),
)
control = await _build_company_runtime_control_by_task(
engine,
@@ -95,8 +98,11 @@ class CompanyKanbanProjectionTests(unittest.IsolatedAsyncioTestCase):
},
)
store = MagicMock()
store.get_pending_checkpoints = AsyncMock(return_value=[])
engine = SimpleNamespace(store=store)
store.get_execution_checkpoints = AsyncMock(return_value=[])
engine = SimpleNamespace(
store=store,
_task_runtime_is_live=AsyncMock(return_value=True),
)
control = await _build_company_runtime_control_by_task(
engine,
@@ -106,8 +112,9 @@ class CompanyKanbanProjectionTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(control["child-task"]["runtime_control_state"], "running")
self.assertTrue(control["child-task"]["can_stop"])
engine._task_runtime_is_live.assert_awaited()
async def test_runtime_control_running_status_does_not_require_live_heartbeat(self) -> None:
async def test_runtime_control_running_status_requires_controller_registry_ownership(self) -> None:
created_at = datetime.now(timezone.utc)
parent_task = SimpleNamespace(
id="parent-task",
@@ -122,7 +129,7 @@ class CompanyKanbanProjectionTests(unittest.IsolatedAsyncioTestCase):
},
)
store = MagicMock()
store.get_pending_checkpoints = AsyncMock(return_value=[])
store.get_execution_checkpoints = AsyncMock(return_value=[])
engine = SimpleNamespace(
store=store,
_task_runtime_is_live=AsyncMock(return_value=False),
@@ -134,9 +141,9 @@ class CompanyKanbanProjectionTests(unittest.IsolatedAsyncioTestCase):
"proj1",
)
self.assertEqual(control["parent-task"]["runtime_control_state"], "running")
self.assertTrue(control["parent-task"]["can_stop"])
engine._task_runtime_is_live.assert_not_awaited()
self.assertEqual(control["parent-task"]["runtime_control_state"], "idle")
self.assertFalse(control["parent-task"]["can_stop"])
engine._task_runtime_is_live.assert_awaited_once_with(parent_task)
async def test_runtime_control_treats_dispatch_hold_as_suspending_without_checkpoint(self) -> None:
created_at = datetime.now(timezone.utc)
@@ -1482,7 +1489,7 @@ class CollabSyncCompanyModeTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(session["company_profile"], "custom")
self.assertEqual(session["preferred_agent"], "codex")
async def test_build_collab_sync_deduplicates_shared_role_sessions(self) -> None:
async def test_build_collab_sync_does_not_promote_shared_role_session_without_ui_anchor(self) -> None:
created_at = datetime.now(timezone.utc)
shared_session_id = "root-session:role:cto"
pending_task = SimpleNamespace(
@@ -1519,6 +1526,7 @@ class CollabSyncCompanyModeTests(unittest.IsolatedAsyncioTestCase):
engine = MagicMock()
engine.store = MagicMock()
engine.store.get_tasks = AsyncMock(return_value=[pending_task, running_task])
engine.store.get_execution_checkpoints = AsyncMock(return_value=[])
engine.project_id = "proj-shared"
engine.llm = None
@@ -1586,10 +1594,7 @@ class CollabSyncCompanyModeTests(unittest.IsolatedAsyncioTestCase):
)
sessions = result.get("sessions", [])
self.assertEqual(len(sessions), 1)
self.assertEqual(sessions[0]["project_id"], "proj-shared")
self.assertEqual(sessions[0]["task_id"], "task-running")
self.assertEqual(sessions[0]["session_id"], shared_session_id)
self.assertEqual(sessions, [])
async def test_build_collab_sync_keeps_root_session_visible_when_final_decider_shares_session(self) -> None:
created_at = datetime.now(timezone.utc)
File diff suppressed because it is too large Load Diff
+549
View File
@@ -0,0 +1,549 @@
from __future__ import annotations
import asyncio
from functools import wraps
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from opc.core.active_task_runs import (
ActiveTaskRunAdmissionClosed,
ActiveTaskRunRegistry,
)
from opc.core.models import CompanyMemberSession, Task, TaskResult, TaskStatus
from opc.engine import OPCEngine
from opc.layer2_organization.company_mode import CompanyWorkItemExecutor
from opc.layer2_organization.company_runtime_identity import is_company_runtime_task
from opc.layer2_organization.org_work_item_planner import CompanyWorkItemRuntimePlan
def _async_test(func):
@wraps(func)
def runner(*args, **kwargs):
return asyncio.run(func(*args, **kwargs))
return runner
def test_overlapping_attempts_remain_active_until_last_attempt_exits() -> None:
registry = ActiveTaskRunRegistry()
first = registry.register("project-a", "task-1")
second = registry.register("project-a", "task-1")
assert first != second
assert registry.attempt_count("project-a", "task-1") == 2
assert registry.is_active("project-a", "task-1")
assert registry.unregister("project-a", "task-1", first)
assert registry.is_active("project-a", "task-1")
assert registry.unregister("project-a", "task-1", second)
assert not registry.is_active("project-a", "task-1")
def test_registry_isolates_projects_with_equal_task_ids() -> None:
registry = ActiveTaskRunRegistry()
token = registry.register("project-a", "task-1")
assert registry.active_task_ids("project-a") == {"task-1"}
assert registry.active_task_ids("project-b") == set()
assert not registry.is_active("project-b", "task-1")
assert registry.unregister("project-a", "task-1", token)
def test_plain_child_task_is_not_classified_as_company_runtime_scope() -> None:
task = Task(
id="plain-task",
title="Plain task",
project_id="project-a",
parent_session_id="parent-session",
metadata={"mode": "task", "parent_session_id": "parent-session"},
)
assert not is_company_runtime_task(task)
def test_closing_admission_preserves_existing_attempts_and_rejects_new_ones() -> None:
registry = ActiveTaskRunRegistry()
token = registry.register("project-a", "task-1")
registry.close_admission()
assert registry.admission_closed
assert registry.active_task_ids("project-a") == {"task-1"}
assert registry.is_active("project-a", "task-1")
with pytest.raises(ActiveTaskRunAdmissionClosed):
registry.register("project-a", "task-2")
assert registry.unregister("project-a", "task-1", token)
def test_closed_admission_allows_only_nested_live_driver_attempts() -> None:
registry = ActiveTaskRunRegistry()
driver_token = registry.register("project-a", "driver-task")
with registry.bind_driver_attempt(driver_token):
registry.close_admission()
nested_token = registry.register("project-a", "claimed-child")
assert registry.is_active("project-a", "claimed-child")
registry.unregister("project-a", "claimed-child", nested_token)
with pytest.raises(ActiveTaskRunAdmissionClosed):
registry.register("project-a", "new-ingress")
registry.unregister("project-a", "driver-task", driver_token)
@_async_test
async def test_shutdown_barrier_allows_only_reserved_handoff_to_register() -> None:
registry = ActiveTaskRunRegistry()
handoff_token = registry.reserve_handoff()
with registry.bind_handoff(handoff_token):
barrier = asyncio.create_task(
registry.close_admission_and_wait_for_handoffs()
)
await asyncio.sleep(0)
assert not barrier.done()
attempt_token = registry.register("project-a", "task-1")
registry.release_handoff(handoff_token)
await asyncio.wait_for(barrier, timeout=0.1)
# The handoff wait ends at real coroutine registration, not at the end of
# that execution attempt.
assert registry.is_active("project-a", "task-1")
assert registry.pending_handoff_count == 0
with pytest.raises(ActiveTaskRunAdmissionClosed):
registry.register("project-a", "late-task")
assert registry.unregister("project-a", "task-1", attempt_token)
@_async_test
async def test_shutdown_barrier_drains_request_that_exits_before_registration() -> None:
registry = ActiveTaskRunRegistry()
handoff_token = registry.reserve_handoff()
barrier = asyncio.create_task(registry.close_admission_and_wait_for_handoffs())
await asyncio.sleep(0)
assert not barrier.done()
assert registry.release_handoff(handoff_token)
await asyncio.wait_for(barrier, timeout=0.1)
assert registry.pending_handoff_count == 0
assert registry.active_task_ids("project-a") == set()
@_async_test
async def test_revoked_handoff_cannot_block_shutdown_or_register_late() -> None:
registry = ActiveTaskRunRegistry()
handoff_token = registry.reserve_handoff()
with registry.bind_handoff(handoff_token):
assert registry.retain_current_handoff() == handoff_token
registry.close_admission()
assert registry.revoke_handoff(handoff_token)
await asyncio.wait_for(
registry.close_admission_and_wait_for_handoffs(),
timeout=0.1,
)
with pytest.raises(ActiveTaskRunAdmissionClosed):
registry.register("project-a", "late-task")
assert registry.pending_handoff_count == 0
assert not registry.release_handoff(handoff_token)
@_async_test
async def test_engine_turns_closed_admission_into_infrastructure_cancellation() -> None:
registry = ActiveTaskRunRegistry()
registry.close_admission()
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
engine._run_task_once = AsyncMock()
task = Task(
id="late-task",
title="Late task",
project_id="project-a",
status=TaskStatus.PENDING,
)
with pytest.raises(asyncio.CancelledError):
await engine._execute_task(task)
engine._run_task_once.assert_not_awaited()
assert registry.active_task_ids("project-a") == set()
@_async_test
async def test_task_liveness_uses_registry_only() -> None:
registry = ActiveTaskRunRegistry()
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
engine.store = SimpleNamespace(get_latest_external_session_for_task=AsyncMock())
task = Task(
id="task-1",
title="Live task",
project_id="project-a",
status=TaskStatus.RUNNING,
)
assert not await engine._task_runtime_is_live(task)
engine.store.get_latest_external_session_for_task.assert_not_awaited()
token = registry.register("project-a", task.id)
assert await engine._task_runtime_is_live(task)
registry.unregister("project-a", task.id, token)
@_async_test
async def test_project_delegate_receives_controller_registry() -> None:
registry = ActiveTaskRunRegistry()
root = OPCEngine(project_id="project-a", active_task_run_registry=registry)
root._initialized = True
captured: dict[str, object] = {}
class FakeDelegate:
def __init__(self, **kwargs: object) -> None:
captured.update(kwargs)
self.store = None
async def initialize(self) -> None:
return None
with patch("opc.engine.OPCEngine", FakeDelegate):
delegate = await root._get_project_delegate("project-b")
assert delegate is root._project_engine_delegates["project-b"]
assert captured["active_task_run_registry"] is registry
assert captured["owns_active_task_run_registry"] is False
@_async_test
async def test_shutdown_cancellation_does_not_write_business_cancelled() -> None:
registry = ActiveTaskRunRegistry()
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
engine._shutting_down = True
engine.store = SimpleNamespace(
is_ready=True,
get_task=AsyncMock(),
save_task=AsyncMock(),
)
engine._run_task_once = AsyncMock(side_effect=asyncio.CancelledError)
task = Task(
id="task-1",
title="Interrupted task",
project_id="project-a",
status=TaskStatus.RUNNING,
)
with pytest.raises(asyncio.CancelledError):
await engine._execute_task(task)
engine.store.get_task.assert_not_awaited()
engine.store.save_task.assert_not_awaited()
assert not registry.is_active("project-a", task.id)
@_async_test
async def test_company_cancellation_never_synthesizes_hold_without_checkpoint() -> None:
engine = OPCEngine(project_id="project-a")
engine.store = SimpleNamespace(
is_ready=True,
get_task=AsyncMock(),
save_task=AsyncMock(),
)
engine._run_task_once = AsyncMock(side_effect=asyncio.CancelledError)
task = Task(
id="company-task",
title="Company task",
project_id="project-a",
status=TaskStatus.RUNNING,
metadata={"work_item_runtime": True},
)
with pytest.raises(asyncio.CancelledError):
await engine._execute_task(task)
engine.store.get_task.assert_not_awaited()
engine.store.save_task.assert_not_awaited()
assert "company_runtime_suspended_at" not in task.metadata
assert "last_stop_reason" not in task.metadata
@_async_test
async def test_suspended_checkpoint_discards_racing_task_completion() -> None:
engine = OPCEngine(project_id="project-a")
engine._shutting_down = False
task = Task(
id="company-task",
title="Company task",
project_id="project-a",
status=TaskStatus.RUNNING,
metadata={"work_item_runtime": True},
)
suspended = Task(
id=task.id,
title=task.title,
project_id=task.project_id,
status=TaskStatus.BLOCKED,
metadata={
"work_item_runtime": True,
"dispatch_hold": "company_runtime_suspended",
"company_runtime_stop_state": "suspended",
},
)
engine.store = SimpleNamespace(
get_task=AsyncMock(return_value=suspended),
save_task=AsyncMock(),
)
engine._run_task_once = AsyncMock(
return_value=TaskResult(status=TaskStatus.DONE, content="done", artifacts={})
)
engine._apply_runtime_state_to_task = MagicMock()
with pytest.raises(asyncio.CancelledError):
await engine._execute_task(task)
engine.store.save_task.assert_not_awaited()
engine._apply_runtime_state_to_task.assert_not_called()
@_async_test
async def test_attempt_stays_active_until_result_persistence_finishes() -> None:
registry = ActiveTaskRunRegistry()
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
save_started = asyncio.Event()
allow_save = asyncio.Event()
async def blocked_save(_task: Task) -> None:
save_started.set()
await allow_save.wait()
engine.store = SimpleNamespace(
get_task=AsyncMock(return_value=None),
save_task=blocked_save,
)
engine._run_task_once = AsyncMock(
return_value=TaskResult(status=TaskStatus.IDLE, content="done", artifacts={})
)
engine._apply_runtime_state_to_task = MagicMock()
task = Task(
id="persisting-task",
title="Persisting task",
project_id="project-a",
status=TaskStatus.RUNNING,
)
execution = asyncio.create_task(engine._execute_task(task))
await save_started.wait()
assert registry.is_active("project-a", task.id)
allow_save.set()
await execution
assert not registry.is_active("project-a", task.id)
@_async_test
async def test_claimed_work_item_ownership_covers_post_execution_finalize_gap() -> None:
registry = ActiveTaskRunRegistry()
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
engine.store = SimpleNamespace(
get_task=AsyncMock(return_value=None),
save_task=AsyncMock(),
)
engine._run_task_once = AsyncMock(
return_value=TaskResult(status=TaskStatus.IDLE, content="done", artifacts={})
)
engine._apply_runtime_state_to_task = MagicMock()
task = Task(
id="finalizing-work-item",
title="Finalizing work item",
project_id="project-a",
parent_session_id="runtime-session",
status=TaskStatus.RUNNING,
metadata={"work_item_runtime": True},
)
inner_finished = asyncio.Event()
allow_finalize = asyncio.Event()
executor = object.__new__(CompanyWorkItemExecutor)
executor.active_task_run_registry = registry
async def run_claimed(*_args: object, **_kwargs: object) -> TaskResult:
result = await engine._execute_task(task)
inner_finished.set()
await allow_finalize.wait()
return result
executor._run_claimed_work_item = run_claimed
owned = executor._create_claimed_work_item_task(
CompanyMemberSession(
role_id="executor",
seat_id="seat::executor",
member_session_id="role-session",
),
task,
{},
)
await inner_finished.wait()
assert registry.attempt_count("project-a", task.id) == 1
allow_finalize.set()
await owned
assert not registry.is_active("project-a", task.id)
@_async_test
async def test_work_item_claim_and_spawn_share_stop_scope_lock() -> None:
registry = ActiveTaskRunRegistry()
executor = object.__new__(CompanyWorkItemExecutor)
executor.active_task_run_registry = registry
claim_entered = asyncio.Event()
allow_claim = asyncio.Event()
allow_child_exit = asyncio.Event()
stop_acquired = asyncio.Event()
task = Task(
id="claimed-task",
title="Claimed task",
project_id="project-a",
session_id="role-session",
parent_session_id="runtime-session",
metadata={"work_item_runtime": True},
)
member_session = CompanyMemberSession(
role_id="executor",
seat_id="seat::executor",
member_session_id="role-session",
)
async def claim_runnable_tasks(
_tasks: list[Task],
*,
work_items: list[object],
) -> list[tuple[CompanyMemberSession, Task]]:
del work_items
claim_entered.set()
await allow_claim.wait()
return [(member_session, task)]
async def run_claimed(*_args: object, **_kwargs: object) -> None:
await allow_child_exit.wait()
executor.runtime = SimpleNamespace(
claim_runnable_tasks=claim_runnable_tasks,
)
executor._run_claimed_work_item = run_claimed
active: dict[asyncio.Task, tuple[CompanyMemberSession, Task]] = {}
scheduled = asyncio.create_task(
executor._claim_and_create_work_item_tasks([task], [], active)
)
await claim_entered.wait()
async def stop_scope() -> None:
async with registry.scope_lock("project-a", "runtime-session"):
assert registry.is_active("project-a", task.id)
stop_acquired.set()
stopping = asyncio.create_task(stop_scope())
await asyncio.sleep(0)
assert not stop_acquired.is_set()
allow_claim.set()
await scheduled
await stopping
assert len(active) == 1
allow_child_exit.set()
await asyncio.gather(*active)
assert not registry.is_active("project-a", task.id)
@_async_test
async def test_company_executor_driver_ownership_covers_idle_scheduler_window() -> None:
registry = ActiveTaskRunRegistry()
entered = asyncio.Event()
allow_exit = asyncio.Event()
executor = object.__new__(CompanyWorkItemExecutor)
executor.active_task_run_registry = registry
async def idle_scheduler(
_plan: CompanyWorkItemRuntimePlan,
_tasks: list[Task],
) -> str:
entered.set()
await allow_exit.wait()
return "done"
executor._execute_multi_team_org = idle_scheduler
task = Task(
id="driver-task",
title="Driver task",
project_id="project-a",
parent_session_id="runtime-session",
metadata={"work_item_runtime": True},
)
execution = asyncio.create_task(
executor.execute(CompanyWorkItemRuntimePlan(), [task])
)
await entered.wait()
assert registry.is_active("project-a", task.id)
allow_exit.set()
assert await execution == "done"
assert not registry.is_active("project-a", task.id)
@_async_test
async def test_borrowed_engine_shutdown_keeps_controller_registry_open() -> None:
registry = ActiveTaskRunRegistry()
root_token = registry.register("project-a", "root-attempt")
borrowed = OPCEngine(
project_id="project-a",
active_task_run_registry=registry,
owns_active_task_run_registry=False,
)
await borrowed.shutdown()
assert not registry.admission_closed
assert registry.is_active("project-a", "root-attempt")
next_token = registry.register("project-a", "next-attempt")
registry.unregister("project-a", "next-attempt", next_token)
registry.unregister("project-a", "root-attempt", root_token)
@_async_test
async def test_shutdown_preparation_includes_project_delegates() -> None:
engine = OPCEngine(project_id="project-a")
delegate_prepare = AsyncMock(
return_value=[{"session_id": "delegate-session", "checkpoint_id": "checkpoint-1"}]
)
engine._project_engine_delegates["project-b"] = SimpleNamespace(
prepare_active_company_runtimes_for_shutdown=delegate_prepare,
)
prepared = await engine.prepare_active_company_runtimes_for_shutdown()
assert prepared == [{"session_id": "delegate-session", "checkpoint_id": "checkpoint-1"}]
delegate_prepare.assert_awaited_once()
@_async_test
async def test_engine_shutdown_prepares_before_closing_subsystems() -> None:
engine = OPCEngine(project_id="project-a")
engine.prepare_active_company_runtimes_for_shutdown = AsyncMock(return_value=[])
await engine.shutdown()
engine.prepare_active_company_runtimes_for_shutdown.assert_awaited_once()
@_async_test
async def test_engine_shutdown_does_not_close_store_when_durable_prepare_fails() -> None:
engine = OPCEngine(project_id="project-a")
engine.prepare_active_company_runtimes_for_shutdown = AsyncMock(
side_effect=RuntimeError("checkpoint failed")
)
engine.store = SimpleNamespace(close=AsyncMock())
engine.message_bus.stop = MagicMock()
with pytest.raises(RuntimeError, match="checkpoint failed"):
await engine.shutdown()
engine.message_bus.stop.assert_not_called()
engine.store.close.assert_not_awaited()
+264 -50
View File
@@ -1116,6 +1116,34 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(calls, ["first", "second"])
self.assertIn("Queued #1", console.export_text())
def test_chat_turn_controller_prepares_checkpoint_before_cancelling_active_turn(self) -> None:
state, _engine = self._make_state()
async def _run() -> list[str]:
order: list[str] = []
started = asyncio.Event()
async def active_turn() -> None:
started.set()
try:
await asyncio.Event().wait()
finally:
order.append("cancelled")
async def prepare() -> list[dict[str, Any]]:
order.append("prepared")
return []
state.engine.prepare_active_company_runtimes_for_shutdown = prepare
controller = ChatTurnController(state)
controller.active_task = asyncio.create_task(active_turn())
await started.wait()
await controller.shutdown()
return order
self.assertEqual(asyncio.run(_run()), ["prepared", "cancelled"])
def test_busy_slash_policy_allows_readonly_and_blocks_mutating_commands(self) -> None:
self.assertEqual(_busy_slash_policy("kanban", []), BusyCommandPolicy.IMMEDIATE_READONLY)
self.assertEqual(_busy_slash_policy("logs", ["task-1"]), BusyCommandPolicy.IMMEDIATE_READONLY)
@@ -1168,6 +1196,17 @@ class CliSlashCommandTests(unittest.TestCase):
"org_id": "quantum_harbor",
"preferred_agent": "codex",
})
store.checkpoints = [
SimpleNamespace(
checkpoint_id="cp-org-interrupted",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id="task-1",
session_id="sess-1",
updated_at=datetime(2026, 5, 17, 12, 0),
payload={},
)
]
state, engine = self._make_state(store=store)
state.mode = "company"
state.company_profile = "corporate"
@@ -1183,7 +1222,59 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(engine.calls[-1]["org_id"], "quantum_harbor")
self.assertIsNone(engine.calls[-1]["company_profile"])
self.assertEqual(engine.calls[-1]["preferred_agent"], "codex")
self.assertEqual(engine.calls[-1]["message_metadata"], {"ui_force_resume": True})
self.assertEqual(engine.calls[-1]["message_metadata"], {
"ui_force_resume": True,
"response_to_checkpoint_id": "cp-org-interrupted",
"response_to_checkpoint_type": "company_runtime_interrupted",
})
def test_company_continue_requires_a_durable_runtime_checkpoint(self) -> None:
console = Console(record=True, force_terminal=False, width=120)
store = self._Store()
store.tasks[0].metadata.update({
"exec_mode": "company",
"company_profile": "corporate",
})
state, engine = self._make_state(store=store)
async def _run() -> None:
await _handle_chat_slash_command(state, "/continue")
with patch("opc.cli.app.console", console):
asyncio.run(_run())
self.assertEqual(engine.calls, [])
self.assertIn("No suspended or interrupted company runtime", console.export_text())
def test_continue_slash_routes_to_durable_runtime_checkpoint(self) -> None:
store = self._Store()
store.checkpoints = [
SimpleNamespace(
checkpoint_id="cp-interrupted",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id="task-1",
session_id="sess-1",
updated_at=datetime(2026, 5, 17, 12, 0),
payload={},
)
]
state, engine = self._make_state(store=store)
state.mode = "company"
state.runtime_control_state = "suspended"
async def _run() -> None:
await _handle_chat_slash_command(state, "/continue")
asyncio.run(_run())
self.assertEqual(engine.calls[-1]["session_id"], "sess-1")
self.assertEqual(engine.calls[-1]["message_metadata"], {
"ui_force_resume": True,
"response_to_checkpoint_id": "cp-interrupted",
"response_to_checkpoint_type": "company_runtime_interrupted",
})
self.assertEqual(state.runtime_control_checkpoint_id, "cp-interrupted")
def test_plain_message_after_stop_routes_to_suspend_checkpoint(self) -> None:
store = self._Store()
@@ -1209,7 +1300,127 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(engine.calls[-1]["message_metadata"]["response_to_checkpoint_id"], "cp-suspend")
self.assertEqual(engine.calls[-1]["message_metadata"]["response_to_checkpoint_type"], "company_runtime_suspended")
self.assertEqual(state.runtime_control_state, "running")
self.assertEqual(state.runtime_control_state, "suspended")
def test_plain_message_from_company_child_uses_root_runtime_checkpoint(self) -> None:
store = self._Store()
now = datetime(2026, 5, 17, 12, 0)
store.tasks[0].metadata.update({
"exec_mode": "company",
"mode": "company",
"company_profile": "corporate",
})
store.tasks[0].parent_session_id = ""
store.tasks.append(SimpleNamespace(
id="worker-child",
title="Worker",
description="Worker turn",
status=TaskStatus.BLOCKED,
priority=3,
assigned_to="worker",
session_id="sess-1:role:worker",
parent_session_id="sess-1",
project_id="demo",
created_at=now,
tags=[],
result={},
linked_work_item_id="worker-item",
metadata={
"exec_mode": "company",
"work_item_runtime": True,
"work_item_projection_id": "worker",
"company_runtime_root_session_id": "sess-1",
},
context_snapshot={},
))
store.checkpoints = [SimpleNamespace(
checkpoint_id="cp-child-interrupted",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id="task-1",
session_id="sess-1",
project_id="demo",
updated_at=now,
payload={"parent_session_id": "sess-1"},
)]
state, engine = self._make_state(store=store)
state.session_id = "sess-1:role:worker"
# The selected child channel may have left the CLI's ambient mode in
# task mode. Runtime config, not ambient state, owns resumed execution.
state.mode = "task"
asyncio.run(_process_interactive_chat_message(state, "revise and continue"))
call = engine.calls[-1]
self.assertEqual(call["session_id"], "sess-1")
self.assertEqual(call["origin_task_id"], "task-1")
self.assertEqual(call["mode"], "company")
self.assertEqual(call["company_profile"], "corporate")
self.assertEqual(call["message_metadata"], {
"response_to_checkpoint_id": "cp-child-interrupted",
"response_to_checkpoint_type": "company_runtime_interrupted",
})
def test_plain_message_during_resuming_checkpoint_fails_closed(self) -> None:
console = Console(record=True, force_terminal=False, width=140)
store = self._Store()
store.tasks[0].metadata.update({
"exec_mode": "company",
"mode": "company",
"company_profile": "corporate",
})
store.checkpoints = [SimpleNamespace(
checkpoint_id="cp-resuming",
checkpoint_type="company_runtime_suspended",
status="resuming",
task_id="task-1",
session_id="sess-1",
project_id="demo",
updated_at=datetime(2026, 5, 17, 12, 0),
payload={"parent_session_id": "sess-1"},
)]
state, engine = self._make_state(store=store)
state.mode = "company"
with patch("opc.cli.app.console", console):
asyncio.run(_process_interactive_chat_message(state, "continue again"))
self.assertEqual(engine.calls, [])
self.assertIn("checkpoint is resuming", console.export_text())
def test_explicit_cli_runtime_checkpoint_mismatch_fails_closed(self) -> None:
console = Console(record=True, force_terminal=False, width=140)
store = self._Store()
store.tasks[0].metadata.update({
"exec_mode": "company",
"mode": "company",
"company_profile": "corporate",
})
store.checkpoints = [SimpleNamespace(
checkpoint_id="cp-current",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id="task-1",
session_id="sess-1",
project_id="demo",
updated_at=datetime(2026, 5, 17, 12, 0),
payload={"parent_session_id": "sess-1"},
)]
state, engine = self._make_state(store=store)
state.mode = "company"
with patch("opc.cli.app.console", console):
asyncio.run(_process_interactive_chat_message(
state,
"continue stale",
message_metadata={
"response_to_checkpoint_id": "cp-stale",
"response_to_checkpoint_type": "company_runtime_interrupted",
},
))
self.assertEqual(engine.calls, [])
self.assertIn("checkpoint identity mismatch", console.export_text())
def test_session_resume_still_switches_session(self) -> None:
state, engine = self._make_state(store=self._Store())
@@ -1287,8 +1498,6 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(output["payload"]["checkpoint_id"], "cp-stop")
self.assertEqual(output["cancelled"], ["task-company"])
self.assertEqual(output["suspend_calls"][0]["session_id"], "sess-company")
self.assertEqual(output["task"].metadata["dispatch_hold"], "company_runtime_suspended")
self.assertEqual(output["task"].metadata["company_runtime_stop_state"], "suspended")
def test_session_service_continue_uses_force_resume_metadata(self) -> None:
class Store:
@@ -1305,10 +1514,17 @@ class CliSlashCommandTests(unittest.TestCase):
metadata={
"exec_mode": "company",
"company_profile": "corporate",
"dispatch_hold": "company_runtime_suspended",
"company_runtime_stop_state": "suspended",
},
)
self.checkpoint = SimpleNamespace(
checkpoint_id="cp-company",
checkpoint_type="company_runtime_suspended",
status="pending",
project_id="demo",
session_id="sess-company",
task_id="task-company",
payload={},
)
async def get_task(self, task_id: str):
return self.task if task_id == self.task.id else None
@@ -1319,6 +1535,9 @@ class CliSlashCommandTests(unittest.TestCase):
async def get_session(self, session_id: str):
return SimpleNamespace(session_id=session_id, project_id="demo")
async def get_execution_checkpoints(self, **_kwargs):
return [self.checkpoint]
async def save_task(self, task):
self.task = task
@@ -1331,7 +1550,12 @@ class CliSlashCommandTests(unittest.TestCase):
engine = SimpleNamespace(project_id="demo", store=Store(), process_message=process_message)
context = OfficeServiceContext(engine=engine, agent_store=None, chat_store=None, event_adapter=None)
result = await SessionService(context).continue_run(project_id="demo", target="sess-company")
result = await SessionService(context).continue_run(
project_id="demo",
target="sess-company",
runtime_session_id="sess-company",
checkpoint_id="cp-company",
)
self.assertEqual(result.payload["response"], "resumed")
return calls
@@ -1340,7 +1564,11 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(calls[-1]["content"], "Resume the existing runtime.")
self.assertEqual(calls[-1]["session_id"], "sess-company")
self.assertEqual(calls[-1]["mode"], "company")
self.assertEqual(calls[-1]["message_metadata"], {"ui_force_resume": True})
self.assertEqual(calls[-1]["message_metadata"], {
"ui_force_resume": True,
"response_to_checkpoint_id": "cp-company",
"response_to_checkpoint_type": "company_runtime_suspended",
})
def test_session_service_continue_preserves_custom_org_id(self) -> None:
class Store:
@@ -1359,10 +1587,17 @@ class CliSlashCommandTests(unittest.TestCase):
"company_profile": "custom",
"org_id": "quantum_harbor",
"preferred_agent": "codex",
"dispatch_hold": "company_runtime_suspended",
"company_runtime_stop_state": "suspended",
},
)
self.checkpoint = SimpleNamespace(
checkpoint_id="cp-org",
checkpoint_type="company_runtime_interrupted",
status="pending",
project_id="demo",
session_id="sess-org",
task_id="task-org",
payload={},
)
async def get_task(self, task_id: str):
return self.task if task_id == self.task.id else None
@@ -1373,6 +1608,9 @@ class CliSlashCommandTests(unittest.TestCase):
async def get_session(self, session_id: str):
return SimpleNamespace(session_id=session_id, project_id="demo")
async def get_execution_checkpoints(self, **_kwargs):
return [self.checkpoint]
async def save_task(self, task):
self.task = task
@@ -1385,7 +1623,12 @@ class CliSlashCommandTests(unittest.TestCase):
engine = SimpleNamespace(project_id="demo", store=Store(), process_message=process_message)
context = OfficeServiceContext(engine=engine, agent_store=None, chat_store=None, event_adapter=None)
result = await SessionService(context).continue_run(project_id="demo", target="sess-org")
result = await SessionService(context).continue_run(
project_id="demo",
target="sess-org",
runtime_session_id="sess-org",
checkpoint_id="cp-org",
)
self.assertEqual(result.payload["response"], "resumed")
return calls
@@ -1394,7 +1637,11 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertEqual(calls[-1]["mode"], "org")
self.assertEqual(calls[-1]["org_id"], "quantum_harbor")
self.assertIsNone(calls[-1]["company_profile"])
self.assertEqual(calls[-1]["message_metadata"], {"ui_force_resume": True})
self.assertEqual(calls[-1]["message_metadata"], {
"ui_force_resume": True,
"response_to_checkpoint_id": "cp-org",
"response_to_checkpoint_type": "company_runtime_interrupted",
})
def test_queue_slash_lists_and_drops_queued_prompts(self) -> None:
console = Console(record=True, force_terminal=False, width=160)
@@ -2648,49 +2895,16 @@ class CliSlashCommandTests(unittest.TestCase):
self.assertIn("Pending Checkpoints", rendered)
self.assertIn("cp-1", rendered)
def test_recover_slash_lists_and_resumes_interrupted_runtime(self) -> None:
def test_legacy_recover_slash_is_not_registered(self) -> None:
console = Console(record=True, force_terminal=False, width=200)
state, _engine = self._make_state(store=self._Store())
resume_calls: list[str] = []
class _FakeRecoveryManager:
async def get_status(self):
return SimpleNamespace(
interrupted=[
SimpleNamespace(
parent_task_id="parent-task",
parent_session_id="parent-session",
title="Interrupted Runtime",
profile="corporate",
interrupted_at="2026-05-03T12:00:00",
work_items=[
SimpleNamespace(projection_id="wi-1", interrupted=True),
SimpleNamespace(projection_id="wi-2", interrupted=False),
],
)
],
active_recoveries=[],
)
async def resume(self, parent_task_id: str):
resume_calls.append(parent_task_id)
if parent_task_id == "parent-task":
return {"ok": True, "resumed_work_item_projection_ids": ["wi-1"]}
return {"ok": False, "error": "not_found"}
async def _run() -> None:
await _handle_chat_slash_command(state, "/recover")
await _handle_chat_slash_command(state, "/recover resume parent-task")
await _handle_chat_slash_command(state, "/recover resume cp-1")
with patch("opc.cli.app.console", console), patch("opc.cli.app._get_chat_recovery_manager", return_value=_FakeRecoveryManager()):
asyncio.run(_run())
with patch("opc.cli.app.console", console):
asyncio.run(_handle_chat_slash_command(state, "/recover"))
rendered = console.export_text()
self.assertIn("Interrupted Runtime", rendered)
self.assertIn("Recovery started for parent-task", rendered)
self.assertIn("Checkpoint cp-1 is not resumed directly", rendered)
self.assertEqual(resume_calls, ["parent-task", "cp-1"])
self.assertIn("Unknown command: /recover", rendered)
self.assertNotIn("Interrupted Company Runtimes", rendered)
def test_logs_slash_renders_task_and_session_runtime_details(self) -> None:
console = Console(record=True, force_terminal=False, width=220)
+76
View File
@@ -527,6 +527,25 @@ class CompanyCollaborationTests(unittest.IsolatedAsyncioTestCase):
await runtime.bootstrap([task])
runtime.enqueue_runnable_work_items([work_item], task_by_work_item_id={"work-item-1": task})
async def claim_work_item(*_args: object, **kwargs: object) -> DelegationWorkItem:
role_session_id = str(kwargs["role_runtime_session_id"])
work_item.phase = Phase.RUNNING
work_item.role_runtime_session_id = role_session_id
work_item.claimed_by_role_runtime_session_id = role_session_id
work_item.claimed_by_seat_id = str(kwargs.get("seat_id", ""))
work_item.metadata = {
**dict(work_item.metadata or {}),
"claimed_by_role_session_id": role_session_id,
"claimed_task_id": str(kwargs["task_id"]),
}
return work_item
runtime.store = SimpleNamespace(
is_ready=True,
claim_delegation_work_item_if_dispatchable=claim_work_item,
save_delegation_role_session=AsyncMock(),
)
claims = await runtime.claim_runnable_tasks([task], work_items=[work_item])
self.assertEqual(len(claims), 1)
@@ -534,6 +553,61 @@ class CompanyCollaborationTests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(claimed_task.id, task.id)
self.assertEqual(claimed_session.member_session_id, "role-session::proj1::root-a::executor::backend-architect")
async def test_company_runtime_does_not_spawn_after_atomic_claim_loses_to_hold(self) -> None:
runtime = CompanyRuntime(
org_engine=DummyOrgEngine(),
communication=DummyRuntimeCommunication(),
)
task = Task(
id="held-work-item-task",
title="Held Work Item",
session_id="root-a",
parent_session_id="root-a",
assigned_to="executor",
status=TaskStatus.PENDING,
project_id="proj1",
metadata={
"work_item_projection_id": "held_execution",
"work_item_role_id": "executor",
"employee_assignment": {
"employee_id": "backend-architect",
"role_id": "executor",
},
},
)
set_linked_work_item_id(task, "held-work-item")
work_item = DelegationWorkItem(
work_item_id="held-work-item",
run_id="run-1",
cell_id="cell-1",
role_id="executor",
projection_id="held_execution",
phase=Phase.READY,
)
held_after_race = DelegationWorkItem(
**{
**work_item.__dict__,
"metadata": {"dispatch_hold": "company_runtime_suspended"},
}
)
claim = AsyncMock(return_value=None)
runtime.store = SimpleNamespace(
is_ready=True,
claim_delegation_work_item_if_dispatchable=claim,
get_delegation_work_item=AsyncMock(return_value=held_after_race),
)
await runtime.bootstrap([task])
runtime.enqueue_runnable_work_items(
[work_item],
task_by_work_item_id={work_item.work_item_id: task},
)
claims = await runtime.claim_runnable_tasks([task], work_items=[work_item])
self.assertEqual(claims, [])
self.assertNotIn(work_item.work_item_id, runtime._claimed_work_item_ids)
claim.assert_awaited_once()
async def test_company_runtime_does_not_double_claim_same_work_item(self) -> None:
runtime = CompanyRuntime(
org_engine=DummyOrgEngine(),
@@ -2722,9 +2796,11 @@ class CompanyCollaborationTests(unittest.IsolatedAsyncioTestCase):
engine.project_id = "proj1"
engine.store = store
engine.company_executor = DummyExecutor()
attempt_token = engine._active_task_run_registry.register("proj1", running_task.id)
response = await engine._maybe_resume_existing_company_runtime("缁х画", "sess-parent-live")
refreshed = await store.get_task(running_task.id)
engine._active_task_run_registry.unregister("proj1", running_task.id, attempt_token)
self.assertIn("already in progress", response)
self.assertEqual(refreshed.status, TaskStatus.RUNNING)
@@ -80,7 +80,6 @@ class SharedRoleSessionExecutionTests(unittest.IsolatedAsyncioTestCase):
record_child_session_result=AsyncMock(),
record_task_completion_async=AsyncMock(),
)
engine._active_task_runs = set()
engine._run_task_once = AsyncMock(
return_value=TaskResult(status=TaskStatus.DONE, content="done", artifacts={})
)
@@ -106,4 +105,3 @@ class SharedRoleSessionExecutionTests(unittest.IsolatedAsyncioTestCase):
engine.memory.record_assistant_turn.assert_awaited_once()
engine.memory.record_child_session_result.assert_not_awaited()
+410
View File
@@ -0,0 +1,410 @@
from __future__ import annotations
import asyncio
import json
from datetime import datetime, timedelta
from types import SimpleNamespace
from unittest.mock import AsyncMock, MagicMock
from opc.core.models import ExecutionCheckpoint, Task, TaskStatus
from opc.layer2_organization.company_runtime_identity import (
build_company_runtime_identity_index,
)
from opc.plugins.office_ui.services.models import ServiceError
from opc.plugins.office_ui.snapshot_builder import (
_build_company_runtime_control_by_task,
_primary_session_tasks_by_session_id,
)
from opc.plugins.office_ui.ws_handler import WSHandler
def _runtime_records() -> tuple[list[Task], ExecutionCheckpoint]:
runtime_session_id = "runtime-session"
anchor = Task(
id="ui-anchor",
project_id="project-a",
session_id=runtime_session_id,
status=TaskStatus.CANCELLED,
metadata={
"exec_mode": "company",
"mode": "company",
"company_profile": "corporate",
},
created_at=datetime.now() - timedelta(minutes=3),
)
final_decider = Task(
id="final-decider",
project_id="project-a",
session_id=runtime_session_id,
parent_session_id=runtime_session_id,
status=TaskStatus.CANCELLED,
linked_work_item_id="work-item-root",
metadata={
"mode": "company",
"work_item_runtime": True,
"work_item_projection_id": "root",
"shared_role_session": True,
"shared_role_id": "ceo",
"company_runtime_root_session_id": runtime_session_id,
},
created_at=datetime.now() - timedelta(minutes=2),
)
child = Task(
id="worker",
project_id="project-a",
session_id=f"{runtime_session_id}:role:worker",
parent_session_id=runtime_session_id,
status=TaskStatus.BLOCKED,
linked_work_item_id="work-item-worker",
metadata={
"mode": "company",
"work_item_runtime": True,
"work_item_projection_id": "worker",
"shared_role_session": True,
"company_runtime_root_session_id": runtime_session_id,
},
created_at=datetime.now() - timedelta(minutes=1),
)
checkpoint = ExecutionCheckpoint(
checkpoint_id="checkpoint-1",
project_id="project-a",
session_id=runtime_session_id,
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id=final_decider.id,
payload={"parent_session_id": runtime_session_id},
)
return [anchor, final_decider, child], checkpoint
def test_identity_is_session_first_and_never_selects_shared_final_decider_as_ui_anchor() -> None:
tasks, checkpoint = _runtime_records()
index = build_company_runtime_identity_index(tasks, [checkpoint])
identity = index.resolve(
task_id="final-decider",
runtime_session_id="runtime-session",
checkpoint_id="checkpoint-1",
)
assert identity is not None
assert identity.runtime_session_id == "runtime-session"
assert identity.ui_anchor_task_id == "ui-anchor"
assert identity.config_source_task_id == "ui-anchor"
assert identity.runtime_task_ids == ("ui-anchor", "final-decider", "worker")
assert identity.pending_checkpoint_id == "checkpoint-1"
assert identity.resumable is True
assert index.resolve(task_session_id="runtime-session:role:worker") == identity
assert index.resolve(task_id="worker", runtime_session_id="other-session") is None
assert index.resolve(task_id="ui-anchor", checkpoint_id="other-checkpoint") is None
def test_config_source_uses_configured_scope_task_when_ui_anchor_has_no_config() -> None:
tasks, _checkpoint = _runtime_records()
tasks[0].metadata = {}
tasks[1].metadata.update({"exec_mode": "org", "company_profile": "custom", "org_id": "studio"})
identity = build_company_runtime_identity_index(tasks).resolve(
task_id="ui-anchor",
)
assert identity is not None
assert identity.ui_anchor_task_id == "ui-anchor"
assert identity.config_source_task_id == "final-decider"
def test_snapshot_session_representative_uses_canonical_ui_anchor() -> None:
tasks, _checkpoint = _runtime_records()
primary, ordered = _primary_session_tasks_by_session_id(
[tasks[1], tasks[0], tasks[2]],
)
assert ordered[0] == "runtime-session"
assert primary["runtime-session"].id == "ui-anchor"
def test_runtime_without_ui_anchor_never_promotes_shared_work_item() -> None:
tasks, checkpoint = _runtime_records()
shared_final = tasks[1]
shared_final.parent_session_id = None
index = build_company_runtime_identity_index([shared_final, tasks[2]], [checkpoint])
identity = index.resolve(runtime_session_id="runtime-session")
assert identity is not None
assert identity.ui_anchor_task_id == ""
assert identity.config_source_task_id == "final-decider"
primary, _ordered = _primary_session_tasks_by_session_id(
[shared_final, tasks[2]],
)
assert "runtime-session" not in primary
def test_snapshot_projects_checkpoint_control_to_cancelled_anchor_without_task_resume_identity() -> None:
tasks, checkpoint = _runtime_records()
class Store:
async def get_execution_checkpoints(self, **_kwargs):
return [checkpoint]
engine = SimpleNamespace(store=Store())
control = asyncio.run(_build_company_runtime_control_by_task(engine, tasks, "project-a"))
assert control["ui-anchor"]["runtime_control_state"] == "suspended"
assert control["ui-anchor"]["can_resume"] is True
assert control["ui-anchor"]["resume_parent_session_id"] == "runtime-session"
assert control["ui-anchor"]["pending_runtime_checkpoint_id"] == "checkpoint-1"
assert "resume_parent_task_id" not in control["ui-anchor"]
def test_service_error_transport_fields_cannot_be_overridden() -> None:
handler = WSHandler.__new__(WSHandler)
sent: list[dict] = []
async def send_ack(_ws, ok=True, **payload):
sent.append({"ok": ok, **payload})
handler._send_ack = send_ack
error = ServiceError(
"actual_code",
"actual message",
{"ok": True, "code": "wrong", "error": "wrong", "detail": "kept"},
)
asyncio.run(handler._send_service_error(object(), error, action="test_action"))
assert sent == [{
"ok": False,
"detail": "kept",
"error": "actual message",
"code": "actual_code",
"action": "test_action",
}]
def test_removed_recovery_action_receives_normal_unknown_message_ack() -> None:
handler = WSHandler.__new__(WSHandler)
handler._shutting_down = False
handler._active_message_tasks = set()
handler._send_ack = AsyncMock()
ws = object()
asyncio.run(handler._route_message(ws, json.dumps({"type": "recovery_action"})))
handler._send_ack.assert_awaited_once_with(
ws,
ok=False,
error="unknown_message_type",
action="recovery_action",
)
def test_work_item_chat_resume_uses_canonical_ui_anchor_as_engine_origin() -> None:
async def scenario() -> None:
tasks, checkpoint = _runtime_records()
handler = WSHandler.__new__(WSHandler)
handler._exec_mode = "task"
handler._company_profile = "corporate"
handler._shutting_down = False
handler._active_runtime_children = {}
handler._session_to_task = {}
handler._task_bg_context = {}
handler._company_suspend_reply_locks = {"runtime-session": asyncio.Lock()}
handler.chat_store = None
handler._set_company_runtime_control = AsyncMock()
handler._normalize_session_exec_mode = MagicMock(return_value="task")
handler._normalize_session_company_profile = MagicMock(return_value="corporate")
handler._resolve_task_session_config = MagicMock(
return_value=("company", "corporate")
)
handler._resolve_task_org_id = MagicMock(return_value="")
handler._extract_checkpoint_metadata = AsyncMock(return_value=None)
handler._sync_task_transcript_messages = AsyncMock()
handler.on_kanban_changed = AsyncMock()
handler._flush_progress = AsyncMock()
run_engine = SimpleNamespace(
project_id="project-a",
process_message=AsyncMock(return_value="resumed"),
)
target = {
"ui_anchor_task_id": "ui-anchor",
"config_task": tasks[1],
}
await handler._process_company_suspend_reply(
ui_task_id="final-decider",
runtime_session_id="runtime-session",
content="continue",
attachment_refs=None,
message_metadata={"ui_force_resume": True},
user_message_id=None,
user_message_created_at=None,
run_engine=run_engine,
run_project_id="project-a",
target=target,
checkpoint=checkpoint,
lock=handler._company_suspend_reply_locks["runtime-session"],
)
call = run_engine.process_message.await_args
assert call.kwargs["session_id"] == "runtime-session"
assert call.kwargs["origin_task_id"] == "ui-anchor"
assert handler._session_to_task["runtime-session"] == "ui-anchor"
handler.on_kanban_changed.assert_awaited_once_with(engine=run_engine)
asyncio.run(scenario())
def test_delivery_feedback_rejects_missing_canonical_identity_without_first_task_fallback() -> None:
async def scenario() -> None:
tasks, checkpoint = _runtime_records()
class Store:
is_ready = True
async def get_tasks(self, **_kwargs):
# A shared final-decider deliberately precedes the UI anchor;
# legacy first-match routing selected the wrong Task here.
return [tasks[1], tasks[0], tasks[2]]
handler = WSHandler.__new__(WSHandler)
handler._resolve_company_runtime_target = AsyncMock(return_value=None)
engine = SimpleNamespace(store=Store())
target = await handler._company_delivery_feedback_parent_target(
task_id="final-decider",
waiting_task_id="worker",
waiting_task=tasks[2],
checkpoint=checkpoint,
payload={"parent_session_id": "runtime-session"},
engine=engine,
)
assert target == {"parent_task_id": "", "parent_session_id": ""}
asyncio.run(scenario())
def test_delivery_feedback_route_consumes_missing_identity_instead_of_running_work_item() -> None:
async def scenario() -> None:
handler = WSHandler.__new__(WSHandler)
handler.chat_store = None
handler._company_delivery_feedback_reply_locks = {}
handler._load_execution_checkpoint_for_reply = AsyncMock(return_value=SimpleNamespace(
checkpoint_id="feedback-1",
checkpoint_type="company_delivery_feedback",
status="pending",
task_id="worker",
session_id="runtime-session",
payload={"waiting_task_id": "worker", "parent_session_id": "runtime-session"},
))
handler._company_delivery_feedback_parent_target = AsyncMock(return_value={
"parent_task_id": "",
"parent_session_id": "",
})
handler._track_session = MagicMock()
engine = SimpleNamespace(store=SimpleNamespace(is_ready=True))
handled = await handler._route_company_delivery_feedback_reply_if_pending(
task_id="worker",
content="looks good",
session_id="runtime-session:worker",
task=SimpleNamespace(id="worker"),
attachment_refs=None,
message_metadata={
"response_to_checkpoint_id": "feedback-1",
"response_to_checkpoint_type": "company_delivery_feedback",
},
user_message_id="message-1",
user_message_created_at=None,
run_engine=engine,
run_project_id="project-a",
reply_channel_id="session:worker",
)
assert handled is True
handler._track_session.assert_not_called()
asyncio.run(scenario())
def test_suspend_reply_identity_mismatch_and_resuming_checkpoint_fail_closed() -> None:
async def scenario() -> None:
tasks, checkpoint = _runtime_records()
handler = WSHandler.__new__(WSHandler)
handler.chat_store = None
handler._company_stop_finalize_tasks = {}
handler._company_suspend_reply_locks = {}
handler._track = MagicMock()
engine = SimpleNamespace(project_id="project-a")
target = {
"runtime_session_id": "runtime-session",
"checkpoint": checkpoint,
}
handler._resolve_company_runtime_target = AsyncMock(
side_effect=[target, None],
)
mismatched = await handler._route_company_suspend_reply_if_pending(
task_id="worker",
content="continue",
session_id="runtime-session:role:worker",
task=tasks[2],
attachment_refs=None,
message_metadata={
"response_to_checkpoint_id": "wrong-checkpoint",
"response_to_checkpoint_type": "company_runtime_interrupted",
},
user_message_id=None,
user_message_created_at=None,
run_engine=engine,
run_project_id="project-a",
)
assert mismatched is True
handler._track.assert_not_called()
checkpoint.status = "resuming"
handler._resolve_company_runtime_target = AsyncMock(return_value=target)
resuming = await handler._route_company_suspend_reply_if_pending(
task_id="worker",
content="continue again",
session_id="runtime-session:role:worker",
task=tasks[2],
attachment_refs=None,
message_metadata=None,
user_message_id=None,
user_message_created_at=None,
run_engine=engine,
run_project_id="project-a",
)
assert resuming is True
handler._track.assert_not_called()
asyncio.run(scenario())
def test_escalation_control_uses_durable_anchor_not_transient_progress_maps() -> None:
async def scenario() -> None:
tasks, checkpoint = _runtime_records()
class Store:
async def get_task(self, task_id: str):
return next((task for task in tasks if task.id == task_id), None)
async def get_tasks(self, **_kwargs):
return tasks
async def get_execution_checkpoints(self, **_kwargs):
return [checkpoint]
handler = WSHandler.__new__(WSHandler)
handler.engine = SimpleNamespace(project_id="project-a", store=Store())
handler._active_runtime_children = {"worker": "wrong-parent"}
handler._session_to_task = {"runtime-session": "wrong-parent"}
handler._ui_task_aliases = {}
resolved = await handler._resolve_escalation_session_task_id("worker")
assert resolved == "ui-anchor"
asyncio.run(scenario())
+42 -21
View File
@@ -74,6 +74,19 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
external_provider_session_id: str = "provider-session-1",
) -> tuple[CompanyWorkItemRuntimePlan, Task]:
plan = self._plan(profile)
await store.save_task(
Task(
id=f"ui-anchor-{parent_session_id}",
title="Company chat",
session_id=parent_session_id,
project_id="proj1",
status=TaskStatus.IDLE,
metadata={
"exec_mode": "company",
"company_profile": profile,
},
)
)
await store.save_delegation_work_item(
DelegationWorkItem(
work_item_id=work_item_id,
@@ -1445,7 +1458,7 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
self.assertNotIn("external_resume_session_id", resumed_task.metadata)
self.assertEqual(resumed_task.metadata["external_resume_fallback"], "context_replay")
async def test_company_runtime_checkpoint_resolves_before_long_execute(self) -> None:
async def test_company_runtime_checkpoint_stays_resuming_during_long_execute(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
engine = self._engine(store)
@@ -1472,7 +1485,7 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
)
captured["resuming_count_during_execute"] = len(resuming)
captured["resolved_count_during_execute"] = len(resolved)
captured["resume_state_during_execute"] = resolved[0].payload.get("resume_state") if resolved else ""
captured["resume_state_during_execute"] = resuming[0].payload.get("resume_state") if resuming else ""
return "runtime resumed"
engine.company_executor = DummyCompanyExecutor()
@@ -1489,9 +1502,9 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
statuses=["resolved"],
)
self.assertEqual(captured["resuming_count_during_execute"], 0)
self.assertEqual(captured["resolved_count_during_execute"], 1)
self.assertEqual(captured["resume_state_during_execute"], "handoff_complete")
self.assertEqual(captured["resuming_count_during_execute"], 1)
self.assertEqual(captured["resolved_count_during_execute"], 0)
self.assertEqual(captured["resume_state_during_execute"], "resuming")
self.assertEqual(len(resolved), 1)
self.assertEqual(resolved[0].payload.get("resume_state"), "handoff_complete")
@@ -1537,10 +1550,24 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
checkpoint_types=["company_runtime_suspended"],
statuses=["resuming"],
)
refreshed_task = await store.get_task(task.id)
refreshed_item = await store.get_delegation_work_item("work-item-1")
self.assertEqual(len(pending), 1)
self.assertEqual(resuming, [])
self.assertEqual(pending[0].payload.get("resume_state"), "failed_before_handoff")
assert refreshed_task is not None
assert refreshed_item is not None
self.assertEqual(
refreshed_task.metadata.get("dispatch_hold"),
"company_runtime_suspended",
)
self.assertEqual(
refreshed_item.metadata.get("dispatch_hold"),
"company_runtime_suspended",
)
self.assertEqual(refreshed_item.claimed_by_role_runtime_session_id, "")
self.assertEqual(refreshed_item.claimed_by_seat_id, "")
async def test_suspend_is_parent_session_idempotent(self) -> None:
store = await self._store()
@@ -1831,22 +1858,16 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
async def test_continue_clears_parent_runtime_stop_marker(self) -> None:
store = await self._store()
_, task = await self._seed_runtime(store)
parent = Task(
id="parent-task",
title="Parent company runtime",
session_id="sess-parent",
parent_session_id="",
status=TaskStatus.RUNNING,
project_id="proj1",
metadata={
"exec_mode": "company",
"company_profile": "corporate",
"company_runtime_stop_state": "suspended",
"company_runtime_stop_intent_id": "intent-1",
"company_runtime_stop_marked_at": "2026-04-29T11:02:40",
"company_runtime_suspended_at": "2026-04-29T11:02:40",
},
)
parent = await store.get_task("ui-anchor-sess-parent")
assert parent is not None
parent.status = TaskStatus.RUNNING
parent.metadata = {
**dict(parent.metadata or {}),
"company_runtime_stop_state": "suspended",
"company_runtime_stop_intent_id": "intent-1",
"company_runtime_stop_marked_at": "2026-04-29T11:02:40",
"company_runtime_suspended_at": "2026-04-29T11:02:40",
}
await store.save_task(parent)
engine = self._engine(store)
await engine.suspend_company_runtime(
-22
View File
@@ -1,22 +0,0 @@
from __future__ import annotations
import os
import unittest
from unittest.mock import patch
from opc import engine as engine_module
from opc.engine import OPCEngine
class EnginePidProbeTests(unittest.TestCase):
def test_current_process_is_running(self) -> None:
self.assertTrue(OPCEngine._pid_is_running(os.getpid()))
def test_posix_probe_treats_unexpected_oserror_as_not_running(self) -> None:
with patch.object(engine_module.os, "name", "posix"), patch.object(
engine_module.os,
"kill",
side_effect=OSError("platform probe failed"),
):
self.assertFalse(OPCEngine._pid_is_running(12345))
File diff suppressed because it is too large Load Diff
+295
View File
@@ -0,0 +1,295 @@
from __future__ import annotations
import asyncio
from datetime import timedelta
from opc.core.models import ExecutionCheckpoint, Task, TaskStatus
from opc.database.store import OPCStore
def test_checkpoint_compare_and_set_has_one_winner_across_store_connections(
tmp_path,
) -> None:
async def scenario() -> None:
db_path = tmp_path / "tasks.db"
first_store = OPCStore(db_path)
second_store = OPCStore(db_path)
await first_store.initialize()
await second_store.initialize()
try:
checkpoint = ExecutionCheckpoint(
checkpoint_id="checkpoint-1",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id="runtime-task",
payload={"reason": "service_restart"},
)
await first_store.save_execution_checkpoint(checkpoint)
start = asyncio.Event()
payloads = [
{"reason": "service_restart", "claimed_by": "office"},
{"reason": "service_restart", "claimed_by": "cli"},
]
async def claim(store: OPCStore, payload: dict[str, str]) -> bool:
await start.wait()
return await store.compare_and_set_execution_checkpoint(
checkpoint.checkpoint_id,
expected_statuses={"pending"},
status="resuming",
payload=payload,
)
claims = [
asyncio.create_task(claim(first_store, payloads[0])),
asyncio.create_task(claim(second_store, payloads[1])),
]
start.set()
results = await asyncio.gather(*claims)
assert results.count(True) == 1
assert results.count(False) == 1
winner = results.index(True)
rows = await first_store.get_execution_checkpoints(
project_id="project-a",
session_id="runtime-session",
)
assert len(rows) == 1
assert rows[0].status == "resuming"
assert rows[0].payload == payloads[winner]
finally:
await second_store.close()
await first_store.close()
asyncio.run(scenario())
def test_checkpoint_get_or_create_has_one_active_row_across_store_connections(
tmp_path,
) -> None:
async def scenario() -> None:
db_path = tmp_path / "tasks.db"
first_store = OPCStore(db_path)
second_store = OPCStore(db_path)
await first_store.initialize()
await second_store.initialize()
try:
checkpoint_types = {
"company_runtime_suspended",
"company_runtime_interrupted",
}
candidates = [
ExecutionCheckpoint(
checkpoint_id="checkpoint-office",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_interrupted",
task_id="runtime-task",
payload={"creator": "office"},
),
ExecutionCheckpoint(
checkpoint_id="checkpoint-cli",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_suspended",
task_id="runtime-task",
payload={"creator": "cli"},
),
]
start = asyncio.Event()
async def create(
store: OPCStore,
candidate: ExecutionCheckpoint,
) -> tuple[ExecutionCheckpoint, bool]:
await start.wait()
return await store.get_or_create_active_execution_checkpoint(
candidate,
checkpoint_types=checkpoint_types,
)
attempts = [
asyncio.create_task(create(first_store, candidates[0])),
asyncio.create_task(create(second_store, candidates[1])),
]
start.set()
results = await asyncio.gather(*attempts)
assert [created for _, created in results].count(True) == 1
assert [created for _, created in results].count(False) == 1
assert len({row.checkpoint_id for row, _ in results}) == 1
active = await first_store.get_execution_checkpoints(
project_id="project-a",
session_id="runtime-session",
checkpoint_types=list(checkpoint_types),
statuses=["pending", "resuming"],
)
assert len(active) == 1
assert active[0].checkpoint_id == results[0][0].checkpoint_id
finally:
await second_store.close()
await first_store.close()
asyncio.run(scenario())
def test_checkpoint_get_or_create_normalizes_historical_active_duplicates(
tmp_path,
) -> None:
async def scenario() -> None:
store = OPCStore(tmp_path / "tasks.db")
await store.initialize()
try:
older = ExecutionCheckpoint(
checkpoint_id="checkpoint-older",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_interrupted",
status="resuming",
payload={"created": "older"},
)
newer = ExecutionCheckpoint(
checkpoint_id="checkpoint-newer",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_suspended",
payload={"created": "newer"},
)
newer.updated_at = older.updated_at + timedelta(microseconds=1)
await store.save_execution_checkpoint(older)
await store.save_execution_checkpoint(newer)
winner, created = await store.get_or_create_active_execution_checkpoint(
ExecutionCheckpoint(
checkpoint_id="checkpoint-unused",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_interrupted",
),
checkpoint_types={
"company_runtime_suspended",
"company_runtime_interrupted",
},
)
assert created is False
assert winner.checkpoint_id == "checkpoint-newer"
active = await store.get_execution_checkpoints(
project_id="project-a",
session_id="runtime-session",
statuses=["pending", "resuming"],
)
assert [row.checkpoint_id for row in active] == ["checkpoint-newer"]
all_rows = await store.get_execution_checkpoints(
project_id="project-a",
session_id="runtime-session",
)
by_id = {row.checkpoint_id: row for row in all_rows}
assert by_id["checkpoint-older"].status == "superseded"
assert (
by_id["checkpoint-older"].payload["superseded_by_checkpoint_id"]
== "checkpoint-newer"
)
finally:
await store.close()
asyncio.run(scenario())
def test_checkpoint_completion_and_cancelled_anchor_reopen_are_atomic(tmp_path) -> None:
async def scenario() -> None:
store = OPCStore(tmp_path / "tasks.db")
await store.initialize()
try:
anchor = Task(
id="ui-anchor",
project_id="project-a",
session_id="runtime-session",
title="Company chat",
status=TaskStatus.CANCELLED,
execution_lock=True,
)
checkpoint = ExecutionCheckpoint(
checkpoint_id="checkpoint-1",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_interrupted",
status="resuming",
payload={"ui_anchor_task_id": anchor.id},
)
await store.save_task(anchor)
await store.save_execution_checkpoint(checkpoint)
completed = await store.complete_execution_checkpoint_and_reopen_ui_anchor(
checkpoint.checkpoint_id,
project_id="project-a",
session_id="runtime-session",
expected_status="resuming",
status="resolved",
payload={"resume_state": "handoff_complete"},
ui_anchor_task_id=anchor.id,
)
assert completed is True
assert (await store.get_task(anchor.id)).status == TaskStatus.IDLE
rows = await store.get_execution_checkpoints(
project_id="project-a",
session_id="runtime-session",
)
assert rows[0].status == "resolved"
assert rows[0].payload == {"resume_state": "handoff_complete"}
finally:
await store.close()
asyncio.run(scenario())
def test_checkpoint_completion_does_not_reopen_anchor_after_stop_wins(tmp_path) -> None:
async def scenario() -> None:
store = OPCStore(tmp_path / "tasks.db")
await store.initialize()
try:
anchor = Task(
id="ui-anchor",
project_id="project-a",
session_id="runtime-session",
title="Company chat",
status=TaskStatus.CANCELLED,
)
checkpoint = ExecutionCheckpoint(
checkpoint_id="checkpoint-1",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_interrupted",
status="pending",
payload={"resume_state": "interrupted"},
)
await store.save_task(anchor)
await store.save_execution_checkpoint(checkpoint)
completed = await store.complete_execution_checkpoint_and_reopen_ui_anchor(
checkpoint.checkpoint_id,
project_id="project-a",
session_id="runtime-session",
expected_status="resuming",
status="resolved",
payload={"resume_state": "handoff_complete"},
ui_anchor_task_id=anchor.id,
)
assert completed is False
assert (await store.get_task(anchor.id)).status == TaskStatus.CANCELLED
rows = await store.get_execution_checkpoints(
project_id="project-a",
session_id="runtime-session",
)
assert rows[0].status == "pending"
assert rows[0].payload == {"resume_state": "interrupted"}
finally:
await store.close()
asyncio.run(scenario())
+380
View File
@@ -0,0 +1,380 @@
from __future__ import annotations
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock
from opc.core.active_task_runs import ActiveTaskRunRegistry
from opc.core.models import ExecutionCheckpoint, Task
from opc.plugins.office_ui.ws_handler import WSHandler
def test_ws_shutdown_checkpoints_before_cancelling_and_awaiting_sessions() -> None:
async def scenario() -> None:
events: list[str] = []
started = asyncio.Event()
async def execution() -> None:
started.set()
try:
await asyncio.Event().wait()
finally:
events.append("execution_finally")
async def prepare() -> list[dict]:
events.append("checkpoint")
assert not execution_task.done()
return []
execution_task = asyncio.create_task(execution())
await started.wait()
handler = WSHandler.__new__(WSHandler)
handler.engine = SimpleNamespace()
handler._root_engine = SimpleNamespace(
prepare_active_company_runtimes_for_shutdown=prepare,
)
handler._shutting_down = False
handler._progress_flush_task = None
handler._clients = set()
handler._active_message_tasks = set()
handler._background_tasks = {execution_task}
handler._task_bg_context = {execution_task: {"task_id": "runtime-task"}}
handler._task_bg_map = {"runtime-task": {execution_task}}
await handler.shutdown(timeout=1.0)
assert events == ["checkpoint", "execution_finally"]
assert execution_task.done()
asyncio.run(scenario())
def test_ws_shutdown_checkpoint_failure_does_not_cancel_execution_or_close_the_gap() -> None:
async def scenario() -> None:
released = asyncio.Event()
async def execution() -> None:
try:
await asyncio.Event().wait()
finally:
released.set()
execution_task = asyncio.create_task(execution())
await asyncio.sleep(0)
handler = WSHandler.__new__(WSHandler)
handler.engine = SimpleNamespace()
handler._root_engine = SimpleNamespace(
prepare_active_company_runtimes_for_shutdown=AsyncMock(
side_effect=RuntimeError("checkpoint unavailable")
),
)
handler._shutting_down = False
handler._progress_flush_task = None
handler._clients = set()
handler._active_message_tasks = set()
handler._background_tasks = {execution_task}
handler._task_bg_context = {execution_task: {"task_id": "runtime-task"}}
handler._task_bg_map = {"runtime-task": {execution_task}}
try:
await handler.shutdown(timeout=1.0)
except RuntimeError as exc:
assert str(exc) == "checkpoint unavailable"
else:
raise AssertionError("shutdown must fail closed when checkpointing fails")
assert not released.is_set()
assert not execution_task.done()
execution_task.cancel()
await asyncio.gather(execution_task, return_exceptions=True)
asyncio.run(scenario())
def test_ws_shutdown_rejects_background_work_scheduled_by_late_ingress() -> None:
async def scenario() -> None:
entered = False
async def late_work() -> None:
nonlocal entered
entered = True
handler = WSHandler.__new__(WSHandler)
handler._shutting_down = True
handler._background_tasks = set()
task = handler._track(late_work())
await asyncio.gather(task, return_exceptions=True)
await asyncio.sleep(0)
assert task.cancelled()
assert entered is False
assert task not in handler._background_tasks
asyncio.run(scenario())
def test_ws_shutdown_drains_queued_duplicate_handoff_before_checkpointing() -> None:
async def scenario() -> None:
registry = ActiveTaskRunRegistry()
runtime_lock = asyncio.Lock()
execution_registered = asyncio.Event()
prepare_called = asyncio.Event()
execution_released = asyncio.Event()
async def prepare() -> list[dict]:
assert registry.is_active("project-a", "runtime-task")
assert registry.pending_handoff_count == 0
prepare_called.set()
return []
root_engine = SimpleNamespace(
_active_task_run_registry=registry,
prepare_active_company_runtimes_for_shutdown=prepare,
)
handler = WSHandler.__new__(WSHandler)
handler.engine = root_engine
handler._root_engine = root_engine
handler._shutting_down = False
handler._progress_flush_task = None
handler._clients = set()
handler._active_message_tasks = set()
handler._background_tasks = set()
handler._task_bg_context = {}
handler._task_bg_map = {}
handler._handoff_route_tasks = {}
async def execution() -> None:
async with runtime_lock:
attempt_token = registry.register("project-a", "runtime-task")
execution_registered.set()
try:
await asyncio.Event().wait()
finally:
registry.unregister("project-a", "runtime-task", attempt_token)
execution_released.set()
async def queued_duplicate() -> None:
async with runtime_lock:
attempt_token = registry.register("project-a", "runtime-task")
try:
await asyncio.Event().wait()
finally:
registry.unregister("project-a", "runtime-task", attempt_token)
first_handoff = registry.reserve_handoff()
with registry.bind_handoff(first_handoff):
first = handler._track_session(
"runtime-task",
execution(),
project_id="project-a",
engine=root_engine,
)
registry.release_handoff(first_handoff)
await execution_registered.wait()
second_handoff = registry.reserve_handoff()
with registry.bind_handoff(second_handoff):
second = handler._track_session(
"runtime-task",
queued_duplicate(),
project_id="project-a",
engine=root_engine,
)
registry.release_handoff(second_handoff)
await asyncio.sleep(0)
assert registry.pending_handoff_count == 1
await asyncio.wait_for(handler.shutdown(timeout=1.0), timeout=1.0)
assert prepare_called.is_set()
assert execution_released.is_set()
assert first.cancelled()
assert second.cancelled()
assert registry.pending_handoff_count == 0
assert not registry.is_active("project-a", "runtime-task")
asyncio.run(scenario())
def test_ws_shutdown_fails_closed_while_execution_cleanup_is_still_running() -> None:
async def scenario() -> None:
cancellation_started = asyncio.Event()
allow_cleanup = asyncio.Event()
async def execution() -> None:
try:
await asyncio.Event().wait()
except asyncio.CancelledError:
cancellation_started.set()
await allow_cleanup.wait()
execution_task = asyncio.create_task(execution())
await asyncio.sleep(0)
handler = WSHandler.__new__(WSHandler)
handler.engine = SimpleNamespace()
handler._root_engine = SimpleNamespace(
prepare_active_company_runtimes_for_shutdown=AsyncMock(return_value=[]),
)
handler._shutting_down = False
handler._progress_flush_task = None
handler._clients = set()
handler._active_message_tasks = set()
handler._background_tasks = {execution_task}
handler._task_bg_context = {
execution_task: {"task_id": "runtime-task", "execution_handoff": True}
}
handler._task_bg_map = {"runtime-task": {execution_task}}
try:
await handler.shutdown(timeout=0.01)
except RuntimeError as exc:
assert "execution task(s)" in str(exc)
else:
raise AssertionError("shutdown must not close resources before execution cleanup")
assert cancellation_started.is_set()
assert not execution_task.done()
allow_cleanup.set()
await execution_task
asyncio.run(scenario())
def test_ws_shutdown_cancels_execution_before_waiting_for_client_close() -> None:
async def scenario() -> None:
execution_released = asyncio.Event()
close_entered = asyncio.Event()
allow_close = asyncio.Event()
async def execution() -> None:
try:
await asyncio.Event().wait()
finally:
execution_released.set()
class BlockingWebSocket:
async def close(self, **_kwargs: object) -> None:
close_entered.set()
await allow_close.wait()
execution_task = asyncio.create_task(execution())
await asyncio.sleep(0)
client = BlockingWebSocket()
handler = WSHandler.__new__(WSHandler)
handler.engine = SimpleNamespace()
handler._root_engine = SimpleNamespace(
prepare_active_company_runtimes_for_shutdown=AsyncMock(return_value=[]),
)
handler._shutting_down = False
handler._progress_flush_task = None
handler._clients = {client}
handler._active_message_tasks = set()
handler._background_tasks = {execution_task}
handler._task_bg_context = {execution_task: {"task_id": "runtime-task"}}
handler._task_bg_map = {"runtime-task": {execution_task}}
shutdown_task = asyncio.create_task(handler.shutdown(timeout=1.0))
await close_entered.wait()
assert execution_released.is_set()
assert execution_task.done()
allow_close.set()
await shutdown_task
asyncio.run(scenario())
def test_duplicate_resume_does_not_leave_shutdown_handoff_barrier_queued() -> None:
async def scenario() -> None:
registry = ActiveTaskRunRegistry()
execution_started = asyncio.Event()
execution_released = asyncio.Event()
async def prepare() -> list[dict]:
assert registry.is_active("project-a", "runtime-task")
return []
root_engine = SimpleNamespace(
project_id="project-a",
_active_task_run_registry=registry,
prepare_active_company_runtimes_for_shutdown=prepare,
)
handler = WSHandler.__new__(WSHandler)
handler.engine = root_engine
handler._root_engine = root_engine
handler.chat_store = None
handler._shutting_down = False
handler._progress_flush_task = None
handler._clients = set()
handler._active_message_tasks = set()
handler._background_tasks = set()
handler._task_bg_context = {}
handler._task_bg_map = {}
handler._company_stop_finalize_tasks = {}
handler._company_suspend_reply_locks = {}
checkpoint = ExecutionCheckpoint(
checkpoint_id="checkpoint-1",
project_id="project-a",
session_id="runtime-session",
checkpoint_type="company_runtime_interrupted",
status="pending",
)
task = Task(
id="ui-task",
title="Company chat",
project_id="project-a",
session_id="runtime-session",
metadata={"exec_mode": "company"},
)
target = {
"runtime_session_id": "runtime-session",
"checkpoint": checkpoint,
}
handler._resolve_company_runtime_target = AsyncMock(return_value=target)
async def fake_resume(**_kwargs: object) -> None:
attempt = registry.register("project-a", "runtime-task")
execution_started.set()
try:
await asyncio.Event().wait()
finally:
registry.unregister("project-a", "runtime-task", attempt)
execution_released.set()
handler._process_company_suspend_reply = fake_resume
async def route_once() -> bool:
handoff = registry.reserve_handoff()
try:
with registry.bind_handoff(handoff):
return await handler._route_company_suspend_reply_if_pending(
task_id=task.id,
content="continue",
session_id=task.session_id,
task=task,
attachment_refs=None,
message_metadata=None,
user_message_id=None,
user_message_created_at=None,
run_engine=root_engine,
run_project_id="project-a",
)
finally:
registry.release_handoff(handoff)
assert await route_once() is True
await execution_started.wait()
assert await route_once() is True
assert registry.pending_handoff_count == 0
assert len(handler._background_tasks) == 1
await asyncio.wait_for(handler.shutdown(timeout=1.0), timeout=1.0)
assert execution_released.is_set()
assert registry.pending_handoff_count == 0
asyncio.run(scenario())
+8
View File
@@ -4,6 +4,7 @@ import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock
from opc.core.active_task_runs import ActiveTaskRunRegistry
from opc.core.config import OPCConfig, RoleConfig
from opc.core.org_config import (
build_org_config_payload_from_config,
@@ -137,6 +138,10 @@ def test_custom_runtime_initializes_with_parent_store(monkeypatch, tmp_path) ->
self.store = kwargs.get("store") or object()
self.owns_store = kwargs.get("owns_store")
self.run_startup_reconcile = kwargs.get("run_startup_reconcile")
self.active_task_run_registry = kwargs.get("active_task_run_registry")
self.owns_active_task_run_registry = kwargs.get(
"owns_active_task_run_registry"
)
self.bound_stores = []
self.message_bus = FakeMessageBus(self)
self.company_executor = SimpleNamespace(_signal_dispatcher_wake=lambda: None)
@@ -164,6 +169,7 @@ def test_custom_runtime_initializes_with_parent_store(monkeypatch, tmp_path) ->
parent.config = OPCConfig()
parent.project_id = None
parent.store = parent_store
parent._active_task_run_registry = ActiveTaskRunRegistry()
parent.on_progress = None
parent.on_runtime_event = None
parent.on_escalation = None
@@ -198,6 +204,8 @@ def test_custom_runtime_initializes_with_parent_store(monkeypatch, tmp_path) ->
assert runtime.store is parent_store
assert runtime.owns_store is False
assert runtime.run_startup_reconcile is False
assert runtime.active_task_run_registry is parent._active_task_run_registry
assert runtime.owns_active_task_run_registry is False
assert runtime.bound_stores == []
assert captured["kanban_callback_runtime"] is runtime
assert callable(runtime.company_executor.on_kanban_changed)
@@ -188,10 +188,8 @@ class DirectStatusWriteLintTest(unittest.TestCase):
PATTERN = re.compile(r"\.status\s*=\s*TaskStatus\.(CANCELLED|FAILED)")
MIGRATED_COMPANY_AWARE_FILES = (
"opc/plugins/office_ui/dispatcher.py",
"opc/plugins/office_ui/recovery_manager.py",
"opc/plugins/office_ui/ws_handler.py",
"opc/plugins/cli_board/services/actions.py",
"opc/plugins/cli_board/services/recovery.py",
)
MIGRATED_FIVE_STATUS_PATTERN = re.compile(
r"\.status\s*=\s*TaskStatus\.(PENDING|RUNNING|DONE|FAILED|CANCELLED)"
+517 -28
View File
@@ -27,6 +27,7 @@ from opc.database.store import _SQLiteConnectionAdapter
from opc.layer2_organization import comms as file_comms
from opc.plugins.office_ui.event_adapter import EventAdapter
from opc.plugins.office_ui.chat_store import ChatStore
from opc.plugins.office_ui.services.models import ServiceError
# ═══════════════════════════════════════════════════════════════════════
@@ -1823,6 +1824,126 @@ class TestWSHandlerSessionSend(unittest.IsolatedAsyncioTestCase):
self.assertEqual(len(ended), 1)
self.handler._dispatch_session_message.assert_not_called()
async def _seed_cancelled_markerless_company_anchor(self) -> ExecutionCheckpoint:
anchor = await self.store.get_task(self.task_id)
assert anchor is not None
anchor.status = TaskStatus.CANCELLED
anchor.metadata = {}
await self.store.save_task(anchor)
await self.store.save_task(Task(
id="shared-final-decider",
title="Final decision",
project_id="test-project",
session_id=self.session_id,
parent_session_id=self.session_id,
linked_work_item_id="work-item-final",
status=TaskStatus.BLOCKED,
metadata={
"exec_mode": "company",
"company_profile": "corporate",
"work_item_runtime": True,
"work_item_projection_id": "final",
"shared_role_session": True,
},
))
checkpoint = ExecutionCheckpoint(
checkpoint_id="cp-markerless-anchor",
project_id="test-project",
session_id=self.session_id,
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id="shared-final-decider",
payload={"parent_session_id": self.session_id},
)
await self.store.save_execution_checkpoint(checkpoint)
return checkpoint
async def test_cancelled_markerless_company_anchor_text_routes_by_checkpoint_identity(self) -> None:
checkpoint = await self._seed_cancelled_markerless_company_anchor()
self.handler._process_company_suspend_reply = AsyncMock()
fake_bg = object()
def _close_and_track(_task_id: str, coro: Any, **_kwargs: Any) -> object:
coro.close()
self.handler._task_bg_context[fake_bg] = {}
return fake_bg
self.handler._track_session = MagicMock(side_effect=_close_and_track)
ws = MagicMock()
await self.handler._handle_session_send(ws, {
"project_id": "test-project",
"task_id": self.task_id,
"content": "继续恢复这个 runtime。",
})
self.handler._process_company_suspend_reply.assert_called_once()
routed = self.handler._process_company_suspend_reply.call_args.kwargs
self.assertEqual(routed["ui_task_id"], self.task_id)
self.assertEqual(routed["runtime_session_id"], self.session_id)
self.assertEqual(routed["checkpoint"].checkpoint_id, checkpoint.checkpoint_id)
self.handler._track_session.assert_called_once()
anchor = await self.store.get_task(self.task_id)
assert anchor is not None
self.assertEqual(anchor.status, TaskStatus.CANCELLED)
async def test_cancelled_company_anchor_with_resuming_checkpoint_is_not_ended_early(self) -> None:
checkpoint = await self._seed_cancelled_markerless_company_anchor()
checkpoint.status = "resuming"
await self.store.save_execution_checkpoint(checkpoint)
self.handler._process_company_suspend_reply = AsyncMock()
self.handler._process_session_message = AsyncMock()
ws = MagicMock()
await self.handler._handle_session_send(ws, {
"project_id": "test-project",
"task_id": self.task_id,
"content": "不要重复恢复。",
})
ended = [
call for call in self.handler._send_ack.await_args_list
if call.kwargs.get("error") == "session_ended"
]
self.assertEqual(ended, [])
self.handler._process_company_suspend_reply.assert_not_called()
self.handler._process_session_message.assert_not_called()
async def test_cancelled_markerless_company_anchor_button_routes_by_checkpoint_identity(self) -> None:
checkpoint = await self._seed_cancelled_markerless_company_anchor()
self.handler._process_company_suspend_reply = AsyncMock()
fake_bg = object()
def _close_and_track(_task_id: str, coro: Any, **_kwargs: Any) -> object:
coro.close()
self.handler._task_bg_context[fake_bg] = {}
return fake_bg
self.handler._track_session = MagicMock(side_effect=_close_and_track)
ws = MagicMock()
await self.handler._handle_session_resume(ws, {
"project_id": "test-project",
"task_id": self.task_id,
"runtime_session_id": self.session_id,
"checkpoint_id": checkpoint.checkpoint_id,
})
self.handler._process_company_suspend_reply.assert_called_once()
routed = self.handler._process_company_suspend_reply.call_args.kwargs
self.assertEqual(routed["ui_task_id"], self.task_id)
self.assertEqual(routed["runtime_session_id"], self.session_id)
self.assertEqual(routed["checkpoint"].checkpoint_id, checkpoint.checkpoint_id)
self.handler._send_ack.assert_awaited_with(
ws,
ok=True,
runtime_session_id=self.session_id,
checkpoint_id=checkpoint.checkpoint_id,
)
anchor = await self.store.get_task(self.task_id)
assert anchor is not None
self.assertEqual(anchor.status, TaskStatus.CANCELLED)
async def test_done_company_session_send_is_reopened_for_followup(self) -> None:
"""Completed company chats can continue in the same CEO/company context."""
ws = MagicMock()
@@ -1980,23 +2101,23 @@ class TestWSHandlerSessionSend(unittest.IsolatedAsyncioTestCase):
}
await self.store.save_task(task)
self.engine.get_active_company_runtime_suspend_checkpoint = AsyncMock(
return_value=SimpleNamespace(
checkpoint_id="cp-suspended",
checkpoint_type="company_runtime_suspended",
status="pending",
payload={},
)
)
await self.store.save_execution_checkpoint(ExecutionCheckpoint(
checkpoint_id="cp-suspended",
project_id="test-project",
session_id=self.session_id,
checkpoint_type="company_runtime_suspended",
status="pending",
payload={"parent_session_id": self.session_id},
))
self.handler._process_company_suspend_reply = AsyncMock()
fake_bg = object()
def _close_and_track(coro: Any) -> object:
def _close_and_track(_task_id: str, coro: Any, **_kwargs: Any) -> object:
coro.close()
self.handler._task_bg_context[fake_bg] = {}
return fake_bg
self.handler._track = MagicMock(side_effect=_close_and_track)
self.handler._track_session = MagicMock()
self.handler._track_session = MagicMock(side_effect=_close_and_track)
await self.handler._handle_session_send(ws, {
"project_id": "test-project",
@@ -2004,13 +2125,39 @@ class TestWSHandlerSessionSend(unittest.IsolatedAsyncioTestCase):
"content": "改成 Sapphire Tide Runner,并让 CEO 自己修改/删除/新增 work item。",
})
self.handler._track_session.assert_not_called()
self.handler._track_session.assert_called_once()
self.handler._process_company_suspend_reply.assert_called_once()
call = self.handler._process_company_suspend_reply.call_args.kwargs
self.assertEqual(call["parent_task_id"], self.task_id)
self.assertEqual(call["parent_session_id"], self.session_id)
self.assertEqual(call["ui_task_id"], self.task_id)
self.assertEqual(call["runtime_session_id"], self.session_id)
self.assertEqual(call["checkpoint"].checkpoint_id, "cp-suspended")
self.assertEqual(call["content"], "改成 Sapphire Tide Runner,并让 CEO 自己修改/删除/新增 work item。")
async def test_rejected_company_resume_refreshes_optimistic_runtime_control(self) -> None:
task = await self.store.get_task(self.task_id)
assert task is not None
task.metadata = {"exec_mode": "company", "company_profile": "corporate"}
await self.store.save_task(task)
self.handler._refresh_runtime_control_for_client = AsyncMock()
ws = MagicMock()
await self.handler._handle_session_resume(ws, {
"project_id": "test-project",
"task_id": self.task_id,
"runtime_session_id": self.session_id,
})
self.handler._send_ack.assert_awaited_once_with(
ws,
ok=False,
error="missing_checkpoint_id",
)
self.handler._refresh_runtime_control_for_client.assert_awaited_once_with(
ws,
engine=self.engine,
project_id="test-project",
)
async def test_session_send_persists_attachment_refs_and_dispatches_them(self) -> None:
"""Uploaded session attachments should be stored and forwarded into engine execution."""
ws = MagicMock()
@@ -3264,6 +3411,37 @@ class TestWSHandlerSessionStop(unittest.IsolatedAsyncioTestCase):
self.assertTrue(any(msg["payload"]["task_id"] == "stop-parent" and msg["payload"]["status"] == "idle" for msg in status_updates))
self.assertTrue(any(msg["payload"]["task_id"] == "stop-child" and msg["payload"]["status"] == "cancelled" for msg in status_updates))
async def test_company_identity_failure_is_rejected_without_task_tree_cancel(self) -> None:
ws = MagicMock()
task = Task(
id="company-stop-mismatch",
title="Company runtime",
session_id="company-stop-session",
project_id="test-project",
status=TaskStatus.RUNNING,
metadata={"exec_mode": "company", "company_profile": "corporate"},
)
await self.store.save_task(task)
self.handler._resolve_company_runtime_target = AsyncMock(return_value=None)
self.handler._cancel_task_tree = AsyncMock()
await self.handler._handle_session_stop(
ws,
{"project_id": "test-project", "task_id": task.id},
)
self.handler._cancel_task_tree.assert_not_awaited()
self.handler._send_ack.assert_awaited_once_with(
ws,
ok=False,
error="company_runtime_identity_mismatch",
project_id="test-project",
task_id=task.id,
)
persisted = await self.store.get_task(task.id)
assert persisted is not None
self.assertEqual(persisted.status, TaskStatus.RUNNING)
# ═══════════════════════════════════════════════════════════════════════
# Test 6: WSHandler — on_opc_event child_session_created
@@ -3711,7 +3889,7 @@ class TestWSHandlerSessionDetail(unittest.IsolatedAsyncioTestCase):
"execution_checkpoint_lifecycle",
)
async def test_session_detail_includes_runtime_control_state(self) -> None:
async def test_session_detail_uses_controller_registry_for_runtime_control_state(self) -> None:
ws = MagicMock()
ws.send_json = AsyncMock()
task = Task(
@@ -3735,8 +3913,17 @@ class TestWSHandlerSessionDetail(unittest.IsolatedAsyncioTestCase):
payload = ws.send_json.await_args.args[0]["payload"]
self.assertTrue(payload["ok"])
self.assertEqual(payload["session_state"].get("runtime_control_state"), "running")
self.assertTrue(payload["session_state"].get("can_stop"))
self.assertEqual(payload["session_state"].get("runtime_control_state"), "idle")
self.assertFalse(payload["session_state"].get("can_stop"))
self.engine._task_runtime_is_live = AsyncMock(return_value=True)
await self.handler._handle_session_detail(
ws,
{"project_id": "test-project", "task_id": "custom-running-task"},
)
live_payload = ws.send_json.await_args.args[0]["payload"]
self.assertEqual(live_payload["session_state"].get("runtime_control_state"), "running")
self.assertTrue(live_payload["session_state"].get("can_stop"))
async def test_session_detail_prefers_task_description_for_role_prompt_context(self) -> None:
ws = MagicMock()
@@ -4582,10 +4769,13 @@ class TestWSHandlerShutdown(unittest.IsolatedAsyncioTestCase):
ws.send_json.assert_awaited_once()
async def test_shutdown_closes_clients_and_waits_for_active_messages(self) -> None:
async def test_shutdown_closes_clients_and_cancels_non_handoff_messages(self) -> None:
ws = MagicMock()
ws.closed = False
ws.closing = False
self.handler._root_engine.prepare_active_company_runtimes_for_shutdown = AsyncMock(
return_value=[]
)
async def _close(*_args: Any, **_kwargs: Any) -> None:
ws.closed = True
@@ -4593,23 +4783,16 @@ class TestWSHandlerShutdown(unittest.IsolatedAsyncioTestCase):
ws.close = AsyncMock(side_effect=_close)
self.handler._clients.add(ws)
blocker = asyncio.Event()
async def _active_message() -> None:
await blocker.wait()
await asyncio.Event().wait()
active_task = asyncio.create_task(_active_message())
self.handler._active_message_tasks.add(active_task)
shutdown_task = asyncio.create_task(self.handler.shutdown(timeout=0.5))
await asyncio.sleep(0.05)
self.assertFalse(shutdown_task.done())
blocker.set()
await shutdown_task
await active_task
await self.handler.shutdown(timeout=0.5)
ws.close.assert_awaited_once()
self.assertTrue(active_task.cancelled())
self.assertTrue(self.handler._shutting_down)
@@ -4967,6 +5150,312 @@ class TestOfficeServiceExecutionIdentity(unittest.IsolatedAsyncioTestCase):
async def asyncTearDown(self) -> None:
await self.chat_store._db.close()
async def test_continue_uses_scope_config_for_markerless_cancelled_ui_anchor(self) -> None:
anchor = Task(
id="service-ui-anchor",
title="Company chat",
session_id="service-runtime-session",
project_id="test-project",
status=TaskStatus.CANCELLED,
metadata={},
)
final_decider = Task(
id="service-final-decider",
title="Final decision",
session_id="service-runtime-session",
parent_session_id="service-runtime-session",
project_id="test-project",
status=TaskStatus.BLOCKED,
linked_work_item_id="service-work-item",
metadata={
"exec_mode": "company",
"company_profile": "corporate",
"work_item_runtime": True,
"work_item_projection_id": "final",
"shared_role_session": True,
},
)
checkpoint = ExecutionCheckpoint(
checkpoint_id="service-runtime-checkpoint",
project_id="test-project",
session_id="service-runtime-session",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id=final_decider.id,
payload={"parent_session_id": "service-runtime-session"},
)
await self.store.save_task(anchor)
await self.store.save_task(final_decider)
await self.store.save_execution_checkpoint(checkpoint)
await self.session_service.continue_run(
project_id="test-project",
task_id=anchor.id,
runtime_session_id="service-runtime-session",
checkpoint_id=checkpoint.checkpoint_id,
content="continue",
)
call = self.engine.process_message.await_args
self.assertEqual(call.kwargs["mode"], "company")
self.assertEqual(call.kwargs["session_id"], "service-runtime-session")
self.assertEqual(call.kwargs["origin_task_id"], anchor.id)
self.assertEqual(
call.kwargs["message_metadata"]["response_to_checkpoint_id"],
checkpoint.checkpoint_id,
)
persisted_anchor = await self.store.get_task(anchor.id)
assert persisted_anchor is not None
self.assertEqual(persisted_anchor.status, TaskStatus.CANCELLED)
async def test_company_continue_preserves_requested_work_item_as_ui_channel(self) -> None:
anchor = Task(
id="service-channel-anchor",
title="Company chat",
session_id="service-channel-runtime",
project_id="test-project",
status=TaskStatus.CANCELLED,
metadata={"exec_mode": "company", "company_profile": "corporate"},
)
work_item = Task(
id="service-channel-work-item",
title="Shared final decision",
session_id="service-channel-runtime",
parent_session_id="service-channel-runtime",
project_id="test-project",
status=TaskStatus.BLOCKED,
linked_work_item_id="service-channel-wi",
metadata={
"exec_mode": "company",
"company_profile": "corporate",
"work_item_runtime": True,
"work_item_projection_id": "final",
"shared_role_session": True,
},
)
checkpoint = ExecutionCheckpoint(
checkpoint_id="service-channel-checkpoint",
project_id="test-project",
session_id="service-channel-runtime",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id=work_item.id,
payload={"parent_session_id": "service-channel-runtime"},
)
await self.store.save_task(anchor)
await self.store.save_task(work_item)
await self.store.save_execution_checkpoint(checkpoint)
result = await self.session_service.continue_run(
project_id="test-project",
task_id=work_item.id,
runtime_session_id="service-channel-runtime",
checkpoint_id=checkpoint.checkpoint_id,
content="continue",
)
call = self.engine.process_message.await_args
self.assertEqual(result.payload["task_id"], work_item.id)
self.assertEqual(call.kwargs["session_id"], "service-channel-runtime")
self.assertEqual(call.kwargs["origin_task_id"], anchor.id)
async def test_company_identity_failure_never_falls_back_to_task_mode_control(self) -> None:
from opc.plugins.office_ui.services.models import ServiceError
task = Task(
id="service-company-control",
title="Company control",
session_id="service-company-session",
project_id="test-project",
status=TaskStatus.RUNNING,
metadata={"exec_mode": "company", "company_profile": "corporate"},
)
await self.store.save_task(task)
mismatch = ServiceError(
"company_runtime_identity_mismatch",
"identity mismatch",
)
self.session_service._resolve_company_runtime_target = AsyncMock(
side_effect=mismatch,
)
with self.assertRaises(ServiceError) as stop_error:
await self.session_service.stop(
project_id="test-project",
task_id=task.id,
)
self.assertEqual(stop_error.exception.code, "company_runtime_identity_mismatch")
with self.assertRaises(ServiceError) as continue_error:
await self.session_service.continue_run(
project_id="test-project",
task_id=task.id,
)
self.assertEqual(continue_error.exception.code, "company_runtime_identity_mismatch")
persisted = await self.store.get_task(task.id)
assert persisted is not None
self.assertEqual(persisted.status, TaskStatus.RUNNING)
async def test_session_send_from_work_item_uses_runtime_checkpoint_identity(self) -> None:
anchor = Task(
id="service-send-anchor",
title="Company chat",
session_id="service-send-runtime",
project_id="test-project",
status=TaskStatus.CANCELLED,
metadata={},
)
final_decider = Task(
id="service-send-final",
title="Final decider",
session_id="service-send-runtime",
parent_session_id="service-send-runtime",
project_id="test-project",
linked_work_item_id="service-send-final-wi",
metadata={
"exec_mode": "company",
"company_profile": "corporate",
"work_item_runtime": True,
"work_item_projection_id": "final",
"shared_role_session": True,
},
)
worker = Task(
id="service-send-worker",
title="Worker",
session_id="service-send-runtime:role:worker",
parent_session_id="service-send-runtime",
project_id="test-project",
linked_work_item_id="service-send-worker-wi",
metadata={
"exec_mode": "company",
"company_profile": "corporate",
"work_item_runtime": True,
"work_item_projection_id": "worker",
},
)
original_worker_metadata = dict(worker.metadata)
checkpoint = ExecutionCheckpoint(
checkpoint_id="service-send-checkpoint",
project_id="test-project",
session_id="service-send-runtime",
checkpoint_type="company_runtime_interrupted",
status="pending",
task_id=final_decider.id,
payload={"parent_session_id": "service-send-runtime"},
)
for task in (anchor, final_decider, worker):
await self.store.save_task(task)
await self.store.save_execution_checkpoint(checkpoint)
result = await self.session_service.send(
project_id="test-project",
task_id=worker.id,
content="revise and continue",
)
call = self.engine.process_message.await_args
self.assertEqual(result.payload["task_id"], worker.id)
self.assertEqual(result.payload["session_id"], "service-send-runtime")
self.assertEqual(call.kwargs["session_id"], "service-send-runtime")
self.assertEqual(call.kwargs["origin_task_id"], anchor.id)
self.assertEqual(call.kwargs["mode"], "company")
self.assertEqual(call.kwargs["message_metadata"], {
"response_to_checkpoint_id": checkpoint.checkpoint_id,
"response_to_checkpoint_type": checkpoint.checkpoint_type,
})
persisted_worker = await self.store.get_task(worker.id)
assert persisted_worker is not None
self.assertEqual(persisted_worker.metadata, original_worker_metadata)
async def test_session_send_rejects_resuming_checkpoint_without_engine_fallback(self) -> None:
anchor = Task(
id="service-resuming-anchor",
title="Company chat",
session_id="service-resuming-runtime",
project_id="test-project",
status=TaskStatus.CANCELLED,
metadata={"exec_mode": "company", "company_profile": "corporate"},
)
checkpoint = ExecutionCheckpoint(
checkpoint_id="service-resuming-checkpoint",
project_id="test-project",
session_id="service-resuming-runtime",
checkpoint_type="company_runtime_suspended",
status="resuming",
task_id=anchor.id,
payload={"parent_session_id": "service-resuming-runtime"},
)
await self.store.save_task(anchor)
await self.store.save_execution_checkpoint(checkpoint)
with self.assertRaises(ServiceError) as raised:
await self.session_service.send(
project_id="test-project",
task_id=anchor.id,
content="continue twice",
)
self.assertEqual(
raised.exception.code,
"company_runtime_checkpoint_not_pending",
)
self.engine.process_message.assert_not_called()
async def test_session_send_rejects_cancelled_task_mode_session(self) -> None:
task = Task(
id="service-cancelled-task-mode",
title="Cancelled task chat",
session_id="service-cancelled-task-session",
project_id="test-project",
status=TaskStatus.CANCELLED,
metadata={
"mode": "task",
"execution_mode": "task_mode",
"origin_task_id": "service-cancelled-task-mode",
},
)
await self.store.save_task(task)
with self.assertRaises(ServiceError) as raised:
await self.session_service.send(
project_id="test-project",
task_id=task.id,
content="must stay cancelled",
)
self.assertEqual(raised.exception.code, "session_ended")
self.engine.process_message.assert_not_called()
async def test_session_send_company_identity_mismatch_fails_closed(self) -> None:
task = Task(
id="service-send-mismatch",
title="Company chat",
session_id="service-send-mismatch-runtime",
project_id="test-project",
metadata={"exec_mode": "company", "company_profile": "corporate"},
)
await self.store.save_task(task)
self.session_service._resolve_company_runtime_target = AsyncMock(
side_effect=ServiceError(
"company_runtime_identity_mismatch",
"identity mismatch",
),
)
with self.assertRaises(ServiceError) as raised:
await self.session_service.send(
project_id="test-project",
task_id=task.id,
content="do not fall back",
)
self.assertEqual(
raised.exception.code,
"company_runtime_identity_mismatch",
)
self.engine.process_message.assert_not_called()
async def test_session_send_prefers_persisted_org_identity_over_call_defaults(self) -> None:
task = Task(
id="task-org-send",
-1
View File
@@ -352,7 +352,6 @@ class DeliveryCardPhaseSyncTests(unittest.IsolatedAsyncioTestCase):
executor._kanban_broadcast_task = None
executor._kanban_debounce_sec = 0.2
executor.runtime = MagicMock()
executor._active_task_runs = set()
executor._runtime_invariant_issue_keys = set()
return executor
-1
View File
@@ -384,7 +384,6 @@ def test_office_ui_work_item_paths_do_not_use_legacy_projection_identity_names()
[
REPO_ROOT / "opc/plugins/office_ui/ws_handler.py",
REPO_ROOT / "opc/plugins/office_ui/snapshot_builder.py",
REPO_ROOT / "opc/plugins/office_ui/recovery_manager.py",
],
)
assert not backend_matches, _format_matches(backend_matches)