fix: unify company runtime recovery lifecycle
This commit is contained in:
@@ -385,7 +385,7 @@ Company Mode turns one brief into a runtime session plus role-owned work items.
|
||||
| `Agents` | Role rollup: active/waiting/pending/done roles, current tool, role work items, filters, search, and links to detailed execution progress. |
|
||||
| `Info` | Status, assignees, role identity, employee assignment, selected execution agent, timing, and developer details. |
|
||||
| `Comms` | Role inboxes, unread/read/sent messages, meetings, decisions, and recent communication failures. |
|
||||
| `Team` | Runtime cockpit: teams, seats, approvals, unread communication, recovery state, and stop controls for the current run. |
|
||||
| `Team` | Runtime cockpit: teams, seats, approvals, unread communication, run state, and stop controls for the current run. |
|
||||
|
||||
To inspect the detailed workflow for a role, open a company-mode session and click a role/work item in the `Chat` progress card or `Agents` tab. The Execution Progress panel shows each work item, its status, activity sections, tool progress, handoffs, review targets, and execution turn metadata.
|
||||
|
||||
@@ -522,7 +522,6 @@ See [`docs/cli-chat-slash.md`](docs/cli-chat-slash.md) for the full command tabl
|
||||
| `opc talent` | `list`, `employees`, `import`, `hire`, `scan`, `import-selected`, `employee-detail`, `import-agent` |
|
||||
| `opc market` | `presets`, `browse`, `preview`, `apply-preset`, `export`, `install`, `list`, `uninstall --yes` |
|
||||
| `opc runtime` | `status`, `checkpoints`, `logs`, `run` |
|
||||
| `opc recovery` | `scan`, `resume`, `cancel --yes`, `retry` |
|
||||
| `opc channels` | `status`, `login`, `start`, `stop` |
|
||||
|
||||
Most service-style commands accept `--project/-p` and `--json`.
|
||||
|
||||
@@ -522,7 +522,6 @@ opc talent hire <template_id> <role_id> -p demo
|
||||
| `opc talent` | `list`、`employees`、`import`、`hire`、`scan`、`import-selected`、`employee-detail`、`import-agent` |
|
||||
| `opc market` | `presets`、`browse`、`preview`、`apply-preset`、`export`、`install`、`list`、`uninstall --yes` |
|
||||
| `opc runtime` | `status`、`checkpoints`、`logs`、`run` |
|
||||
| `opc recovery` | `scan`、`resume`、`cancel --yes`、`retry` |
|
||||
| `opc channels` | `status`、`login`、`start`、`stop` |
|
||||
|
||||
大多数服务类命令都支持 `--project/-p` 与 `--json`。
|
||||
|
||||
@@ -13,7 +13,7 @@ The source of truth for this table is `_SLASH_COMMANDS` in `opc/cli/app.py`.
|
||||
- `/mode [task|company] [corporate|custom]` changes how future natural-language messages run.
|
||||
- `/agent [native|codex|claude_code|cursor|opencode|none]` sets or clears the preferred execution agent.
|
||||
- `/domains [domain ...|clear]` sets or clears domain hints.
|
||||
- Aliases: `/p` is `/project`, `/s` is `/session`, `/t` is `/task`, `/checkpoint` is `/checkpoints`, `/recovery` is `/recover`.
|
||||
- Aliases: `/p` is `/project`, `/s` is `/session`, `/t` is `/task`, and `/checkpoint` is `/checkpoints`.
|
||||
|
||||
## Project And Session
|
||||
|
||||
@@ -30,8 +30,6 @@ The source of truth for this table is `_SLASH_COMMANDS` in `opc/cli/app.py`.
|
||||
- `/tasks [status] [--limit N] [--full]` lists project tasks.
|
||||
- `/task show|move|done|rename|delete` inspects and updates persisted tasks.
|
||||
- `/runtime [--limit N] [--full]` shows live runtime, active tasks, external sessions, and checkpoints.
|
||||
- `/recover [--limit N] [--full]` lists interrupted runtimes and resumable checkpoints.
|
||||
- `/recover resume|cancel|retry <parent_task_id>` acts on interrupted company runtimes.
|
||||
- `/logs <task_id|session_id> [--limit N] [--full]` shows execution logs, runtime events, tools, and transcript.
|
||||
- `/comms <task_id> [--limit N] [--full]` shows company-mode messages, handoffs, review notes, and handoff context.
|
||||
- `/attachments [--limit N] [--full]` lists current-session attachment references.
|
||||
|
||||
+256
-232
@@ -2118,33 +2118,6 @@ def runtime_run(task_id: str = typer.Argument(...), project: Optional[str] = typ
|
||||
asyncio.run(_run_service_command(project, lambda svc: svc.runtime.run_task(project_id=project or "default", task_id=task_id), json_output=json_output))
|
||||
|
||||
|
||||
recovery_app = typer.Typer(help="Recover interrupted company runtimes")
|
||||
app.add_typer(recovery_app, name="recovery")
|
||||
|
||||
|
||||
@recovery_app.command("scan")
|
||||
def recovery_scan(project: Optional[str] = typer.Option(None, "--project", "-p"), json_output: bool = typer.Option(False, "--json")):
|
||||
asyncio.run(_run_service_command(project, lambda svc: svc.runtime.recovery_scan(project_id=project or "default"), json_output=json_output))
|
||||
|
||||
|
||||
@recovery_app.command("resume")
|
||||
def recovery_resume(parent_task_id: str = typer.Argument(...), project: Optional[str] = typer.Option(None, "--project", "-p"), json_output: bool = typer.Option(False, "--json")):
|
||||
asyncio.run(_run_service_command(project, lambda svc: svc.runtime.recovery_action(project_id=project or "default", action="resume", parent_task_id=parent_task_id), json_output=json_output))
|
||||
|
||||
|
||||
@recovery_app.command("cancel")
|
||||
def recovery_cancel(parent_task_id: str = typer.Argument(...), yes: bool = typer.Option(False, "--yes", "-y"), project: Optional[str] = typer.Option(None, "--project", "-p"), json_output: bool = typer.Option(False, "--json")):
|
||||
if not yes:
|
||||
console.print("[warning]Destructive command requires --yes.[/warning]")
|
||||
raise typer.Exit(code=1)
|
||||
asyncio.run(_run_service_command(project, lambda svc: svc.runtime.recovery_action(project_id=project or "default", action="cancel", parent_task_id=parent_task_id), json_output=json_output))
|
||||
|
||||
|
||||
@recovery_app.command("retry")
|
||||
def recovery_retry(parent_task_id: str = typer.Argument(...), project: Optional[str] = typer.Option(None, "--project", "-p"), json_output: bool = typer.Option(False, "--json")):
|
||||
asyncio.run(_run_service_command(project, lambda svc: svc.runtime.recovery_action(project_id=project or "default", action="retry", parent_task_id=parent_task_id), json_output=json_output))
|
||||
|
||||
|
||||
comms_app = typer.Typer(help="Inspect company-mode comms")
|
||||
app.add_typer(comms_app, name="comms")
|
||||
|
||||
@@ -2834,6 +2807,17 @@ class ChatTurnController:
|
||||
self._closing = True
|
||||
self.queue.clear()
|
||||
await self.stop_kanban_watch(silent=True)
|
||||
prepare = getattr(
|
||||
self.state.engine,
|
||||
"prepare_active_company_runtimes_for_shutdown",
|
||||
None,
|
||||
)
|
||||
if callable(prepare):
|
||||
# Persist company checkpoints and put the engine into
|
||||
# infrastructure-shutdown mode before cancelling the active turn.
|
||||
# Engine.shutdown() repeats this idempotently when it closes the
|
||||
# remaining stores and subsystems.
|
||||
await prepare()
|
||||
task = self.active_task
|
||||
if task is None or task.done():
|
||||
return
|
||||
@@ -2994,8 +2978,6 @@ _SLASH_COMMANDS: tuple[_SlashCommandSpec, ...] = (
|
||||
_SlashCommandSpec("Tasks", "/task rename <task_id> <title>", "Rename a task and its session title."),
|
||||
_SlashCommandSpec("Tasks", "/task delete <task_id> --yes", "Hard-delete a task and its persisted lifecycle data."),
|
||||
_SlashCommandSpec("Runtime", "/runtime [--limit N] [--full]", "Show live runtime, active tasks, external sessions, and checkpoints."),
|
||||
_SlashCommandSpec("Runtime", "/recover [--limit N] [--full]", "Show interrupted runtime and resumable checkpoints.", ("scan", "resume", "cancel", "retry")),
|
||||
_SlashCommandSpec("Runtime", "/recover resume|cancel|retry <parent_task_id>", "Act on an interrupted company runtime."),
|
||||
_SlashCommandSpec("Runtime", "/logs <task_id|session_id> [--limit N] [--full]", "Show execution logs, runtime events, tools, and transcript."),
|
||||
_SlashCommandSpec("Runtime", "/comms <task_id> [--limit N] [--full]", "Show company-mode messages, handoffs, and review notes."),
|
||||
_SlashCommandSpec("Runtime", "/attachments [--limit N] [--full]", "List current session attachment references."),
|
||||
@@ -3027,7 +3009,7 @@ _SLASH_COMMANDS: tuple[_SlashCommandSpec, ...] = (
|
||||
_SlashCommandSpec("Diagnostics", "/checkpoints [--limit N] [--full]", "List pending execution checkpoints."),
|
||||
)
|
||||
CommandSpec = _SlashCommandSpec
|
||||
_SLASH_ALIASES = {"p": "project", "s": "session", "t": "task", "checkpoint": "checkpoints", "recovery": "recover", "work-item": "work-items", "workitems": "work-items"}
|
||||
_SLASH_ALIASES = {"p": "project", "s": "session", "t": "task", "checkpoint": "checkpoints", "work-item": "work-items", "workitems": "work-items"}
|
||||
|
||||
|
||||
def _initial_company_profile(config: OPCConfig) -> str:
|
||||
@@ -3862,6 +3844,8 @@ async def _resolve_session_or_task(state: _InteractiveChatState, token: str) ->
|
||||
|
||||
|
||||
async def _resolve_runtime_control_target(state: _InteractiveChatState, target: str = "") -> tuple[str, str]:
|
||||
from opc.layer2_organization.company_runtime_identity import load_company_runtime_identity_index
|
||||
|
||||
store = _require_chat_store(state, label="Session store")
|
||||
if store is None:
|
||||
return "", ""
|
||||
@@ -3869,24 +3853,38 @@ async def _resolve_runtime_control_target(state: _InteractiveChatState, target:
|
||||
if not raw_target:
|
||||
console.print("[warning]No current session. Use /session list or /session create first.[/warning]")
|
||||
return "", ""
|
||||
project_id = _current_project_id(state.engine)
|
||||
identity_index = await load_company_runtime_identity_index(store, project_id)
|
||||
task = await store.get_task(raw_target) if hasattr(store, "get_task") else None
|
||||
if task is not None:
|
||||
project_id = str(getattr(task, "project_id", "") or "default")
|
||||
if project_id != _current_project_id(state.engine):
|
||||
console.print(f"[warning]Target belongs to project '{project_id}'. Switch project first.[/warning]")
|
||||
task_project_id = str(getattr(task, "project_id", "") or "default")
|
||||
if task_project_id != project_id:
|
||||
console.print(f"[warning]Target belongs to project '{task_project_id}'. Switch project first.[/warning]")
|
||||
return "", ""
|
||||
runtime_identity = identity_index.resolve(task_id=raw_target)
|
||||
if runtime_identity is not None:
|
||||
return raw_target, runtime_identity.runtime_session_id
|
||||
return str(getattr(task, "id", "") or ""), str(getattr(task, "session_id", "") or getattr(task, "parent_session_id", "") or "")
|
||||
runtime_identity = (
|
||||
identity_index.resolve(runtime_session_id=raw_target)
|
||||
or identity_index.resolve(task_session_id=raw_target)
|
||||
)
|
||||
if runtime_identity is not None:
|
||||
control_task_id = (
|
||||
runtime_identity.ui_anchor_task_id
|
||||
or runtime_identity.config_source_task_id
|
||||
)
|
||||
if control_task_id:
|
||||
return control_task_id, runtime_identity.runtime_session_id
|
||||
session = await store.get_session(raw_target) if hasattr(store, "get_session") else None
|
||||
if session is None:
|
||||
console.print(f"[warning]Task or session not found: {raw_target}[/warning]")
|
||||
return "", ""
|
||||
project_id = str(getattr(session, "project_id", "") or "default")
|
||||
if project_id != _current_project_id(state.engine):
|
||||
console.print(f"[warning]Target belongs to project '{project_id}'. Switch project first.[/warning]")
|
||||
session_project_id = str(getattr(session, "project_id", "") or "default")
|
||||
if session_project_id != project_id:
|
||||
console.print(f"[warning]Target belongs to project '{session_project_id}'. Switch project first.[/warning]")
|
||||
return "", ""
|
||||
if raw_target in state.session_to_task:
|
||||
return state.session_to_task[raw_target], raw_target
|
||||
tasks = await store.get_tasks(project_id=project_id) if hasattr(store, "get_tasks") else []
|
||||
tasks = list(identity_index.tasks)
|
||||
candidates = [
|
||||
item for item in tasks
|
||||
if str(getattr(item, "session_id", "") or "") == raw_target
|
||||
@@ -3894,8 +3892,15 @@ async def _resolve_runtime_control_target(state: _InteractiveChatState, target:
|
||||
if not candidates:
|
||||
console.print(f"[warning]Session is not task-backed: {raw_target}[/warning]")
|
||||
return "", raw_target
|
||||
candidates.sort(key=lambda item: bool(str(getattr(item, "parent_session_id", "") or "")))
|
||||
return str(getattr(candidates[0], "id", "") or ""), raw_target
|
||||
task_mode_anchor = min(
|
||||
candidates,
|
||||
key=lambda item: (
|
||||
bool(str(getattr(item, "parent_session_id", "") or "")),
|
||||
str(getattr(item, "created_at", "") or ""),
|
||||
str(getattr(item, "id", "") or ""),
|
||||
),
|
||||
)
|
||||
return str(getattr(task_mode_anchor, "id", "") or ""), raw_target
|
||||
|
||||
|
||||
def _make_cli_runtime_control_context(state: _InteractiveChatState, controller: ChatTurnController | None = None) -> Any:
|
||||
@@ -3923,35 +3928,66 @@ def _make_cli_runtime_control_context(state: _InteractiveChatState, controller:
|
||||
return context
|
||||
|
||||
|
||||
async def _latest_company_suspend_checkpoint(state: _InteractiveChatState) -> Any | None:
|
||||
session_id = str(state.session_id or "").strip()
|
||||
if not session_id:
|
||||
return None
|
||||
for name in ("get_active_company_runtime_suspend_checkpoint", "get_pending_company_runtime_suspend_checkpoint"):
|
||||
getter = getattr(state.engine, name, None)
|
||||
if callable(getter):
|
||||
try:
|
||||
checkpoint = await getter(session_id)
|
||||
except Exception:
|
||||
checkpoint = None
|
||||
if checkpoint is not None and str(getattr(checkpoint, "status", "") or "pending") == "pending":
|
||||
return checkpoint
|
||||
async def _company_runtime_identity_for_session(
|
||||
state: _InteractiveChatState,
|
||||
session_id: str | None = None,
|
||||
) -> Any | None:
|
||||
from opc.layer2_organization.company_runtime_identity import (
|
||||
load_company_runtime_identity_index,
|
||||
)
|
||||
|
||||
runtime_session_id = str(session_id or state.session_id or "").strip()
|
||||
store = getattr(state.engine, "store", None)
|
||||
if store is not None and hasattr(store, "get_pending_checkpoints"):
|
||||
try:
|
||||
checkpoints = await store.get_pending_checkpoints(
|
||||
project_id=_current_project_id(state.engine),
|
||||
session_id=session_id,
|
||||
checkpoint_types=["company_runtime_suspended", "company_runtime_interrupted"],
|
||||
)
|
||||
except TypeError:
|
||||
checkpoints = await store.get_pending_checkpoints(project_id=_current_project_id(state.engine), session_id=session_id)
|
||||
except Exception:
|
||||
checkpoints = []
|
||||
for checkpoint in list(checkpoints or []):
|
||||
if str(getattr(checkpoint, "checkpoint_type", "") or "") in {"company_runtime_suspended", "company_runtime_interrupted"}:
|
||||
return checkpoint
|
||||
return None
|
||||
if not runtime_session_id or store is None:
|
||||
return None
|
||||
try:
|
||||
identity_index = await load_company_runtime_identity_index(
|
||||
store,
|
||||
_current_project_id(state.engine),
|
||||
)
|
||||
# The interactive session may be a role/work-item session. Resolve it
|
||||
# as a task session after trying the canonical root session so both UI
|
||||
# roots and child channels converge on the same company runtime.
|
||||
identity = (
|
||||
identity_index.resolve(runtime_session_id=runtime_session_id)
|
||||
or identity_index.resolve(task_session_id=runtime_session_id)
|
||||
)
|
||||
except Exception:
|
||||
return None
|
||||
return identity
|
||||
|
||||
|
||||
async def _latest_company_suspend_checkpoint(
|
||||
state: _InteractiveChatState,
|
||||
session_id: str | None = None,
|
||||
) -> Any | None:
|
||||
identity = await _company_runtime_identity_for_session(state, session_id)
|
||||
return identity.checkpoint if identity is not None and identity.resumable else None
|
||||
|
||||
|
||||
async def _company_runtime_execution_identity(
|
||||
state: _InteractiveChatState,
|
||||
runtime_identity: Any,
|
||||
) -> Any:
|
||||
"""Read execution configuration from the durable runtime config source."""
|
||||
from opc.plugins.office_ui.execution_identity import execution_identity_from_task
|
||||
|
||||
config_source_task_id = str(
|
||||
getattr(runtime_identity, "config_source_task_id", "") or ""
|
||||
).strip()
|
||||
store = getattr(state.engine, "store", None)
|
||||
if not config_source_task_id or store is None or not hasattr(store, "get_task"):
|
||||
raise RuntimeError("Company runtime has no durable configuration source.")
|
||||
config_task = await store.get_task(config_source_task_id)
|
||||
if config_task is None:
|
||||
raise RuntimeError("Company runtime configuration source no longer exists.")
|
||||
return execution_identity_from_task(
|
||||
config_task,
|
||||
default_exec_mode=state.mode,
|
||||
default_company_profile=state.company_profile,
|
||||
default_preferred_agent=state.preferred_agent or "native",
|
||||
default_org_id=state.org_id,
|
||||
)
|
||||
|
||||
|
||||
async def _handle_stop_slash(state: _InteractiveChatState, args: list[str], controller: ChatTurnController | None = None) -> None:
|
||||
@@ -3972,22 +4008,35 @@ async def _handle_stop_slash(state: _InteractiveChatState, args: list[str], cont
|
||||
return
|
||||
payload = dict(result.payload)
|
||||
state.runtime_control_state = str(payload.get("runtime_control_state") or payload.get("status") or "stopped")
|
||||
state.runtime_control_task_id = str(payload.get("resume_parent_task_id") or payload.get("task_id") or task_id)
|
||||
state.runtime_control_task_id = str(payload.get("task_id") or task_id)
|
||||
state.runtime_control_session_id = str(payload.get("resume_parent_session_id") or payload.get("session_id") or session_id)
|
||||
state.runtime_control_checkpoint_id = str(payload.get("checkpoint_id") or "")
|
||||
console.print("[success]Stopped.[/success] [dim]Send a message to revise, or /continue to resume.[/dim]")
|
||||
|
||||
|
||||
async def _runtime_control_identity_for_task(state: _InteractiveChatState, task_id: str) -> Any:
|
||||
async def _runtime_control_execution_identity(state: _InteractiveChatState, task_id: str) -> Any:
|
||||
from opc.layer2_organization.company_runtime_identity import load_company_runtime_identity_index
|
||||
from opc.plugins.office_ui.execution_identity import execution_identity_from_task
|
||||
|
||||
store = getattr(state.engine, "store", None)
|
||||
task = None
|
||||
if store is not None and hasattr(store, "get_task") and task_id:
|
||||
if store is not None and task_id:
|
||||
try:
|
||||
task = await store.get_task(task_id)
|
||||
identity_index = await load_company_runtime_identity_index(
|
||||
store,
|
||||
_current_project_id(state.engine),
|
||||
)
|
||||
runtime_identity = identity_index.resolve(task_id=task_id)
|
||||
task = identity_index.task(
|
||||
runtime_identity.config_source_task_id if runtime_identity is not None else task_id
|
||||
)
|
||||
except Exception:
|
||||
task = None
|
||||
if task is None and hasattr(store, "get_task"):
|
||||
try:
|
||||
task = await store.get_task(task_id)
|
||||
except Exception:
|
||||
task = None
|
||||
return execution_identity_from_task(
|
||||
task,
|
||||
default_exec_mode=state.mode,
|
||||
@@ -4014,13 +4063,24 @@ async def _handle_continue_slash(state: _InteractiveChatState, args: list[str],
|
||||
task_id, session_id = await _resolve_runtime_control_target(state, target)
|
||||
if not task_id:
|
||||
return
|
||||
identity = await _runtime_control_identity_for_task(state, task_id)
|
||||
identity = await _runtime_control_execution_identity(state, task_id)
|
||||
content = " ".join(message_parts).strip() or "Resume the existing runtime."
|
||||
checkpoint = await _latest_company_suspend_checkpoint(state, session_id)
|
||||
if identity.exec_mode in {"company", "org", "custom"} and checkpoint is None:
|
||||
console.print(
|
||||
"[warning]No suspended or interrupted company runtime is available to continue.[/warning]"
|
||||
)
|
||||
return
|
||||
if controller is not None and controller.is_busy:
|
||||
checkpoint = await _latest_company_suspend_checkpoint(state)
|
||||
if checkpoint is None and state.runtime_control_state not in {"suspended", "stopped"}:
|
||||
console.print("[warning]Busy: wait for the current turn or /stop it before /continue.[/warning]")
|
||||
return
|
||||
metadata: dict[str, Any] = {"ui_force_resume": True}
|
||||
if checkpoint is not None:
|
||||
metadata["response_to_checkpoint_id"] = str(getattr(checkpoint, "checkpoint_id", "") or "")
|
||||
metadata["response_to_checkpoint_type"] = str(
|
||||
getattr(checkpoint, "checkpoint_type", "") or "company_runtime_suspended"
|
||||
)
|
||||
item = QueuedChatInput(
|
||||
text=content,
|
||||
project_id=_current_project_id(state.engine),
|
||||
@@ -4030,26 +4090,29 @@ async def _handle_continue_slash(state: _InteractiveChatState, args: list[str],
|
||||
org_id=identity.org_id,
|
||||
preferred_agent=identity.preferred_agent,
|
||||
domains=list(state.domains),
|
||||
message_metadata={"ui_force_resume": True},
|
||||
message_metadata=metadata,
|
||||
)
|
||||
state.runtime_control_state = "resuming"
|
||||
state.runtime_control_task_id = task_id
|
||||
state.runtime_control_session_id = session_id or state.session_id
|
||||
state.runtime_control_checkpoint_id = ""
|
||||
state.runtime_control_checkpoint_id = str(getattr(checkpoint, "checkpoint_id", "") or "")
|
||||
if controller is not None:
|
||||
await controller.submit_item(item)
|
||||
else:
|
||||
previous_session_id = state.session_id
|
||||
previous_mode = state.mode
|
||||
previous_profile = state.company_profile
|
||||
previous_org_id = state.org_id
|
||||
previous_agent = state.preferred_agent
|
||||
state.session_id = session_id or state.session_id
|
||||
state.mode = identity.exec_mode
|
||||
state.company_profile = identity.company_profile
|
||||
state.org_id = identity.org_id
|
||||
state.preferred_agent = identity.preferred_agent
|
||||
try:
|
||||
await _process_interactive_chat_message(state, content, message_metadata={"ui_force_resume": True})
|
||||
await _process_interactive_chat_message(state, content, message_metadata=metadata)
|
||||
finally:
|
||||
state.session_id = previous_session_id
|
||||
state.mode = previous_mode
|
||||
state.company_profile = previous_profile
|
||||
state.org_id = previous_org_id
|
||||
@@ -5630,140 +5693,6 @@ async def _handle_runtime_slash(state: _InteractiveChatState, args: list[str]) -
|
||||
_render_checkpoint_table(checkpoints[:limit], title="Pending Checkpoints", full=full)
|
||||
|
||||
|
||||
class _ChatRecoveryFacade:
|
||||
def __init__(self, state: _InteractiveChatState) -> None:
|
||||
self._state = state
|
||||
self.project_id = _current_project_id(state.engine)
|
||||
|
||||
@property
|
||||
def store(self): # noqa: ANN201 - preserve recovery facade shape
|
||||
return getattr(self._state.engine, "store", None)
|
||||
|
||||
@property
|
||||
def opc_home(self) -> Path:
|
||||
return Path(getattr(self._state.engine, "opc_home", get_opc_home()))
|
||||
|
||||
async def ensure_ready(self):
|
||||
return self._state.engine
|
||||
|
||||
|
||||
def _get_chat_recovery_manager(state: _InteractiveChatState) -> Any:
|
||||
cached = getattr(state, "_cli_chat_recovery_manager", None)
|
||||
engine_id = id(state.engine)
|
||||
if cached and cached[0] == engine_id:
|
||||
return cached[1]
|
||||
from opc.plugins.cli_board.services.recovery import CliRecoveryManager
|
||||
|
||||
manager = CliRecoveryManager(_ChatRecoveryFacade(state))
|
||||
setattr(state, "_cli_chat_recovery_manager", (engine_id, manager))
|
||||
return manager
|
||||
|
||||
|
||||
def _render_recovery_status(status: Any, *, limit: int, full: bool = False) -> None:
|
||||
workflows = list(getattr(status, "interrupted", []) or [])[:limit]
|
||||
if workflows:
|
||||
table = Table(title=f"Interrupted Company Runtimes ({len(workflows)})")
|
||||
table.add_column("Parent Task")
|
||||
table.add_column("Parent Session")
|
||||
table.add_column("Title")
|
||||
table.add_column("Profile")
|
||||
table.add_column("Interrupted")
|
||||
table.add_column("Work Items")
|
||||
for workflow in workflows:
|
||||
interrupted_items = [
|
||||
item for item in list(getattr(workflow, "work_items", []) or [])
|
||||
if bool(getattr(item, "interrupted", False))
|
||||
]
|
||||
table.add_row(
|
||||
str(getattr(workflow, "parent_task_id", "") or ""),
|
||||
str(getattr(workflow, "parent_session_id", "") or ""),
|
||||
_clip_text(getattr(workflow, "title", "") or "", 70, full=full),
|
||||
str(getattr(workflow, "profile", "") or ""),
|
||||
str(getattr(workflow, "interrupted_at", "") or ""),
|
||||
f"{len(interrupted_items)}/{len(list(getattr(workflow, 'work_items', []) or []))}",
|
||||
)
|
||||
console.print(table)
|
||||
else:
|
||||
console.print("[info]No interrupted company runtimes found.[/info]")
|
||||
|
||||
active = list(getattr(status, "active_recoveries", []) or [])
|
||||
if active:
|
||||
console.print(f"[info]Active recoveries: {', '.join(active)}[/info]")
|
||||
|
||||
|
||||
async def _find_checkpoint_by_id(store: Any, project_id: str, checkpoint_id: str) -> Any | None:
|
||||
if hasattr(store, "get_execution_checkpoints"):
|
||||
checkpoints = await store.get_execution_checkpoints(project_id=project_id)
|
||||
else:
|
||||
checkpoints = await store.get_pending_checkpoints(project_id=project_id)
|
||||
return next((item for item in checkpoints if str(getattr(item, "checkpoint_id", "") or "") == checkpoint_id), None)
|
||||
|
||||
|
||||
async def _handle_recover_slash(state: _InteractiveChatState, args: list[str]) -> None:
|
||||
store = _require_chat_store(state, label="Recovery store")
|
||||
if store is None:
|
||||
return
|
||||
if args and args[0].lower() == "resume":
|
||||
if len(args) != 2:
|
||||
console.print("[warning]Usage: /recover resume <parent_task_id>. Try /recover.[/warning]")
|
||||
return
|
||||
manager = _get_chat_recovery_manager(state)
|
||||
result = await manager.resume(args[1])
|
||||
if result.get("ok"):
|
||||
resumed = result.get("resumed_work_item_projection_ids", []) or []
|
||||
console.print(f"[success]Recovery started for {args[1]}: {', '.join(resumed) or 'runtime queued'}.[/success]")
|
||||
return
|
||||
if result.get("error") == "not_found":
|
||||
checkpoint = await _find_checkpoint_by_id(store, _current_project_id(state.engine), args[1])
|
||||
if checkpoint:
|
||||
session_id = str(getattr(checkpoint, "session_id", "") or "")
|
||||
suffix = f" Try /session resume {session_id}." if session_id else " Try /checkpoints."
|
||||
console.print(f"[warning]Checkpoint {args[1]} is not resumed directly.{suffix}[/warning]")
|
||||
return
|
||||
console.print(f"[warning]Recovery could not start: {result.get('error', 'unknown_error')}[/warning]")
|
||||
return
|
||||
if args and args[0].lower() in {"cancel", "retry"}:
|
||||
action = args[0].lower()
|
||||
if len(args) < 2:
|
||||
console.print(f"[warning]Usage: /recover {action} <parent_task_id>{' --yes' if action == 'cancel' else ''}. Try /recover.[/warning]")
|
||||
return
|
||||
if action == "cancel":
|
||||
remaining, ok = _require_yes_arg(args[2:], usage="/recover cancel <parent_task_id> --yes")
|
||||
if not ok or remaining:
|
||||
return
|
||||
elif len(args) > 2:
|
||||
console.print(f"[warning]Usage: /recover {action} <parent_task_id>[/warning]")
|
||||
return
|
||||
payload = await _run_chat_office_service(
|
||||
state,
|
||||
lambda svc: svc.runtime.recovery_action(project_id=_current_project_id(state.engine), action=action, parent_task_id=args[1]),
|
||||
)
|
||||
if payload:
|
||||
_emit_payload(payload)
|
||||
return
|
||||
try:
|
||||
if args and args[0].lower() == "scan":
|
||||
args = args[1:]
|
||||
args, limit, full = _parse_view_args(args)
|
||||
except ValueError as exc:
|
||||
console.print(f"[warning]{exc}. Try /recover --limit 20.[/warning]")
|
||||
return
|
||||
if args:
|
||||
console.print("[warning]Usage: /recover [--limit N] [--full] or /recover resume <parent_task_id>.[/warning]")
|
||||
return
|
||||
manager = _get_chat_recovery_manager(state)
|
||||
status = await manager.get_status()
|
||||
_render_recovery_status(status, limit=limit, full=full)
|
||||
if hasattr(store, "get_execution_checkpoints"):
|
||||
checkpoints = await store.get_execution_checkpoints(
|
||||
project_id=_current_project_id(state.engine),
|
||||
statuses=["pending", "resuming"],
|
||||
)
|
||||
else:
|
||||
checkpoints = await store.get_pending_checkpoints(project_id=_current_project_id(state.engine))
|
||||
_render_checkpoint_table(checkpoints[:limit], title="Recovery Checkpoints", full=full)
|
||||
|
||||
|
||||
async def _resolve_logs_target(
|
||||
state: _InteractiveChatState,
|
||||
target: str,
|
||||
@@ -7487,10 +7416,6 @@ def _busy_slash_policy(command: str, args: list[str]) -> BusyCommandPolicy:
|
||||
}
|
||||
if command in readonly_roots:
|
||||
return BusyCommandPolicy.IMMEDIATE_READONLY
|
||||
if command == "recover":
|
||||
if not args or args[0].lower() in {"scan", "status", "list"}:
|
||||
return BusyCommandPolicy.IMMEDIATE_READONLY
|
||||
return BusyCommandPolicy.BLOCKED_WHEN_BUSY
|
||||
if command == "task":
|
||||
if args and args[0].lower() == "show":
|
||||
return BusyCommandPolicy.IMMEDIATE_READONLY
|
||||
@@ -7810,8 +7735,6 @@ async def _handle_chat_slash_command(state: _InteractiveChatState, user_input: s
|
||||
await _handle_reorg_slash(state, args)
|
||||
elif command == "runtime":
|
||||
await _handle_runtime_slash(state, args)
|
||||
elif command == "recover":
|
||||
await _handle_recover_slash(state, args)
|
||||
elif command in {"work-items", "work-item"}:
|
||||
await _handle_work_items_slash(state, args)
|
||||
elif command == "logs":
|
||||
@@ -7835,6 +7758,13 @@ async def _sync_runtime_checkpoint_hint(state: _InteractiveChatState) -> None:
|
||||
display = state.runtime_display
|
||||
if not hasattr(display, "set_checkpoint_hint"):
|
||||
return
|
||||
runtime_identity = await _company_runtime_identity_for_session(state)
|
||||
if runtime_identity is not None and runtime_identity.checkpoint is not None:
|
||||
checkpoint = runtime_identity.checkpoint
|
||||
checkpoint_type = str(getattr(checkpoint, "checkpoint_type", "") or "pending")
|
||||
checkpoint_id = str(getattr(checkpoint, "checkpoint_id", "") or "")
|
||||
display.set_checkpoint_hint(checkpoint_type or checkpoint_id)
|
||||
return
|
||||
getter = getattr(state.engine, "get_latest_pending_checkpoint_for_session", None)
|
||||
if not callable(getter) or not state.session_id:
|
||||
display.set_checkpoint_hint("")
|
||||
@@ -7862,24 +7792,110 @@ async def _process_interactive_chat_message(
|
||||
try:
|
||||
state.runtime_display.begin_turn()
|
||||
effective_metadata = message_metadata
|
||||
if effective_metadata is None:
|
||||
suspend_checkpoint = await _latest_company_suspend_checkpoint(state)
|
||||
if suspend_checkpoint is not None:
|
||||
effective_metadata = {
|
||||
"response_to_checkpoint_id": str(getattr(suspend_checkpoint, "checkpoint_id", "") or ""),
|
||||
"response_to_checkpoint_type": str(getattr(suspend_checkpoint, "checkpoint_type", "") or "company_runtime_suspended"),
|
||||
}
|
||||
elif state.mode == "company":
|
||||
execution_session_id = state.session_id
|
||||
execution_origin_task_id: str | None = None
|
||||
execution_mode = state.mode
|
||||
execution_company_profile = state.company_profile
|
||||
execution_org_id = state.org_id
|
||||
execution_preferred_agent = state.preferred_agent
|
||||
handoff_identity = None
|
||||
runtime_identity = await _company_runtime_identity_for_session(state)
|
||||
runtime_checkpoint = (
|
||||
runtime_identity.checkpoint
|
||||
if runtime_identity is not None
|
||||
else None
|
||||
)
|
||||
explicit_runtime_type = str(
|
||||
(effective_metadata or {}).get("response_to_checkpoint_type", "") or ""
|
||||
).strip()
|
||||
explicit_runtime_id = str(
|
||||
(effective_metadata or {}).get("response_to_checkpoint_id", "") or ""
|
||||
).strip()
|
||||
is_explicit_runtime_handoff = explicit_runtime_type in {
|
||||
"company_runtime_suspended",
|
||||
"company_runtime_interrupted",
|
||||
}
|
||||
|
||||
if is_explicit_runtime_handoff:
|
||||
if runtime_identity is None or runtime_checkpoint is None:
|
||||
raise RuntimeError(
|
||||
"Company runtime checkpoint does not match the current session."
|
||||
)
|
||||
checkpoint_id = str(
|
||||
getattr(runtime_checkpoint, "checkpoint_id", "") or ""
|
||||
).strip()
|
||||
checkpoint_type = str(
|
||||
getattr(runtime_checkpoint, "checkpoint_type", "") or ""
|
||||
).strip()
|
||||
checkpoint_status = str(
|
||||
getattr(runtime_checkpoint, "status", "") or ""
|
||||
).strip().lower()
|
||||
if (
|
||||
not explicit_runtime_id
|
||||
or explicit_runtime_id != checkpoint_id
|
||||
or explicit_runtime_type != checkpoint_type
|
||||
):
|
||||
raise RuntimeError(
|
||||
"Company runtime checkpoint identity mismatch; refresh before continuing."
|
||||
)
|
||||
if checkpoint_status != "pending":
|
||||
raise RuntimeError(
|
||||
f"Company runtime checkpoint is {checkpoint_status or 'not pending'}."
|
||||
)
|
||||
execution_session_id = runtime_identity.runtime_session_id
|
||||
execution_origin_task_id = runtime_identity.ui_anchor_task_id or None
|
||||
handoff_identity = runtime_identity
|
||||
elif effective_metadata is None and runtime_checkpoint is not None:
|
||||
checkpoint_status = str(
|
||||
getattr(runtime_checkpoint, "status", "") or ""
|
||||
).strip().lower()
|
||||
if checkpoint_status != "pending":
|
||||
raise RuntimeError(
|
||||
f"Company runtime checkpoint is {checkpoint_status or 'not pending'}."
|
||||
)
|
||||
if runtime_identity is None:
|
||||
raise RuntimeError(
|
||||
"Company runtime checkpoint does not match the current session."
|
||||
)
|
||||
effective_metadata = {
|
||||
"response_to_checkpoint_id": str(getattr(runtime_checkpoint, "checkpoint_id", "") or ""),
|
||||
"response_to_checkpoint_type": str(getattr(runtime_checkpoint, "checkpoint_type", "") or "company_runtime_suspended"),
|
||||
}
|
||||
execution_session_id = runtime_identity.runtime_session_id
|
||||
execution_origin_task_id = runtime_identity.ui_anchor_task_id or None
|
||||
handoff_identity = runtime_identity
|
||||
elif effective_metadata is None:
|
||||
if runtime_identity is not None and runtime_identity.pending_checkpoint_id:
|
||||
# An active checkpoint may only be pending or resuming. Never
|
||||
# let a malformed identity/status fall through as a new turn.
|
||||
raise RuntimeError(
|
||||
"Company runtime checkpoint is not available for a new turn."
|
||||
)
|
||||
if state.mode == "company":
|
||||
effective_metadata = {"company_preflight": "manual"}
|
||||
if handoff_identity is not None:
|
||||
runtime_execution_identity = await _company_runtime_execution_identity(
|
||||
state,
|
||||
handoff_identity,
|
||||
)
|
||||
execution_mode = runtime_execution_identity.exec_mode
|
||||
execution_company_profile = runtime_execution_identity.company_profile
|
||||
execution_org_id = runtime_execution_identity.org_id
|
||||
execution_preferred_agent = runtime_execution_identity.preferred_agent
|
||||
response = await state.engine.process_message(
|
||||
user_input,
|
||||
project_id=getattr(state.engine, "project_id", None),
|
||||
session_id=state.session_id,
|
||||
mode=state.mode,
|
||||
org_id=state.org_id or None,
|
||||
company_profile=state.company_profile if state.mode == "company" else None,
|
||||
preferred_agent=state.preferred_agent,
|
||||
session_id=execution_session_id,
|
||||
mode=execution_mode,
|
||||
org_id=execution_org_id or None,
|
||||
company_profile=(
|
||||
execution_company_profile
|
||||
if execution_mode == "company"
|
||||
else None
|
||||
),
|
||||
preferred_agent=execution_preferred_agent,
|
||||
domains=list(state.domains),
|
||||
origin_task_id=execution_origin_task_id,
|
||||
message_metadata=effective_metadata,
|
||||
)
|
||||
await state.runtime_display.flush()
|
||||
@@ -7910,7 +7926,15 @@ async def _process_interactive_chat_message(
|
||||
"company_runtime_interrupted",
|
||||
}
|
||||
):
|
||||
state.runtime_control_state = "running"
|
||||
remaining_checkpoint = await _latest_company_suspend_checkpoint(state)
|
||||
if remaining_checkpoint is not None:
|
||||
state.runtime_control_state = "suspended"
|
||||
state.runtime_control_checkpoint_id = str(
|
||||
getattr(remaining_checkpoint, "checkpoint_id", "") or ""
|
||||
)
|
||||
else:
|
||||
state.runtime_control_state = "idle"
|
||||
state.runtime_control_checkpoint_id = ""
|
||||
except KeyboardInterrupt:
|
||||
console.print("\n[warning]Interrupted. Type /quit to exit.[/warning]")
|
||||
except Exception as e:
|
||||
|
||||
@@ -0,0 +1,269 @@
|
||||
"""Controller-local ownership of active task execution attempts.
|
||||
|
||||
Persisted task rows describe durable workflow state; they cannot prove that
|
||||
the controller which owns an execution coroutine is still alive. This
|
||||
registry intentionally stays in memory and is shared by all engines owned by
|
||||
one controller.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
|
||||
|
||||
_CURRENT_HANDOFF: ContextVar[tuple[object, str] | None] = ContextVar(
|
||||
"opc_active_task_run_handoff",
|
||||
default=None,
|
||||
)
|
||||
_CURRENT_DRIVER_ATTEMPT: ContextVar[tuple[object, str] | None] = ContextVar(
|
||||
"opc_active_task_run_driver_attempt",
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
class ActiveTaskRunAdmissionClosed(RuntimeError):
|
||||
"""Raised when execution registration starts after shutdown admission closes."""
|
||||
|
||||
|
||||
class ActiveTaskRunRegistry:
|
||||
"""Track active execution attempts by ``(project_id, task_id)``.
|
||||
|
||||
A task can briefly have overlapping attempts while cancellation and a new
|
||||
dispatch cross. Each registration therefore receives its own token and a
|
||||
task remains active until its last token is removed.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._attempts: dict[tuple[str, str], set[str]] = {}
|
||||
self._scope_locks: dict[tuple[str, str], asyncio.Lock] = {}
|
||||
self._handoff_refs: dict[str, int] = {}
|
||||
self._handoffs_drained = asyncio.Event()
|
||||
self._handoffs_drained.set()
|
||||
self._admission_closed = False
|
||||
|
||||
@staticmethod
|
||||
def _key(project_id: str | None, task_id: str | None) -> tuple[str, str]:
|
||||
project = str(project_id or "default").strip() or "default"
|
||||
task = str(task_id or "").strip()
|
||||
if not task:
|
||||
raise ValueError("task_id is required")
|
||||
return project, task
|
||||
|
||||
def register(self, project_id: str | None, task_id: str | None) -> str:
|
||||
handoff_token = self._current_pending_handoff_token()
|
||||
driver_attempt_token = self._current_driver_attempt_token()
|
||||
if (
|
||||
self._admission_closed
|
||||
and handoff_token is None
|
||||
and driver_attempt_token is None
|
||||
):
|
||||
raise ActiveTaskRunAdmissionClosed(
|
||||
"task execution admission is closed for controller shutdown"
|
||||
)
|
||||
key = self._key(project_id, task_id)
|
||||
attempt_token = uuid.uuid4().hex
|
||||
self._attempts.setdefault(key, set()).add(attempt_token)
|
||||
# A pre-shutdown WS request is handed off once its first real execution
|
||||
# coroutine is registered. The reservation itself is deliberately not
|
||||
# reported by is_active()/active_task_ids(); only this attempt is.
|
||||
if handoff_token is not None:
|
||||
self._settle_handoff(handoff_token)
|
||||
return attempt_token
|
||||
|
||||
@contextmanager
|
||||
def bind_driver_attempt(self, attempt_token: str) -> Iterator[None]:
|
||||
"""Allow nested attempts while their live scheduler owns the scope.
|
||||
|
||||
Closing controller admission rejects new ingress, but a scheduler that
|
||||
was already running may be between its atomic WorkItem claim and child
|
||||
coroutine creation. Its nested registrations remain admissible until
|
||||
that scheduler attempt ends, so shutdown can snapshot the still-live
|
||||
scope instead of creating an orphan RUNNING claim.
|
||||
"""
|
||||
|
||||
if not self._attempt_token_is_active(attempt_token):
|
||||
raise ValueError("driver attempt is not active")
|
||||
context_token = _CURRENT_DRIVER_ATTEMPT.set((self, attempt_token))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_CURRENT_DRIVER_ATTEMPT.reset(context_token)
|
||||
|
||||
def reserve_handoff(self) -> str:
|
||||
"""Reserve one accepted ingress request until execution is registered.
|
||||
|
||||
Reservations bridge the short scheduling gap between the WS router and
|
||||
``register``. They are controller-local synchronization only and never
|
||||
become a second liveness source.
|
||||
"""
|
||||
|
||||
if self._admission_closed:
|
||||
raise ActiveTaskRunAdmissionClosed(
|
||||
"task execution admission is closed for controller shutdown"
|
||||
)
|
||||
token = uuid.uuid4().hex
|
||||
self._handoff_refs[token] = 1
|
||||
self._handoffs_drained.clear()
|
||||
return token
|
||||
|
||||
@contextmanager
|
||||
def bind_handoff(self, handoff_token: str) -> Iterator[None]:
|
||||
"""Propagate a reservation through tasks spawned by an ingress handler."""
|
||||
|
||||
if handoff_token not in self._handoff_refs:
|
||||
raise ValueError("handoff reservation is not pending")
|
||||
context_token = _CURRENT_HANDOFF.set((self, handoff_token))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_CURRENT_HANDOFF.reset(context_token)
|
||||
|
||||
def retain_current_handoff(self) -> str | None:
|
||||
"""Retain the bound reservation for a newly scheduled coroutine."""
|
||||
|
||||
handoff_token = self._current_pending_handoff_token()
|
||||
if handoff_token is None:
|
||||
return None
|
||||
self._handoff_refs[handoff_token] += 1
|
||||
return handoff_token
|
||||
|
||||
def release_current_handoff(self) -> bool:
|
||||
"""Release an accepted request that will not start an execution."""
|
||||
|
||||
handoff_token = self._current_pending_handoff_token()
|
||||
if handoff_token is None:
|
||||
return False
|
||||
return self.release_handoff(handoff_token)
|
||||
|
||||
def release_handoff(self, handoff_token: str) -> bool:
|
||||
"""Release one owner, draining a request that exited before execution."""
|
||||
|
||||
refs = self._handoff_refs.get(handoff_token)
|
||||
if refs is None:
|
||||
return False
|
||||
if refs > 1:
|
||||
self._handoff_refs[handoff_token] = refs - 1
|
||||
return True
|
||||
self._settle_handoff(handoff_token)
|
||||
return True
|
||||
|
||||
def revoke_handoff(self, handoff_token: str) -> bool:
|
||||
"""Invalidate every retained owner of a queued ingress handoff.
|
||||
|
||||
Controller shutdown uses this after synchronously cancelling a request
|
||||
which has not registered its first execution attempt. Revocation is
|
||||
intentionally stronger than ``release_handoff``: callbacks may be
|
||||
delayed by cancellation cleanup, but the revoked request must neither
|
||||
keep the shutdown barrier open nor register work after admission has
|
||||
closed.
|
||||
"""
|
||||
|
||||
if handoff_token not in self._handoff_refs:
|
||||
return False
|
||||
self._settle_handoff(handoff_token)
|
||||
return True
|
||||
|
||||
def _current_pending_handoff_token(self) -> str | None:
|
||||
binding = _CURRENT_HANDOFF.get()
|
||||
if binding is None or binding[0] is not self:
|
||||
return None
|
||||
token = binding[1]
|
||||
return token if token in self._handoff_refs else None
|
||||
|
||||
def _current_driver_attempt_token(self) -> str | None:
|
||||
binding = _CURRENT_DRIVER_ATTEMPT.get()
|
||||
if binding is None or binding[0] is not self:
|
||||
return None
|
||||
token = binding[1]
|
||||
return token if self._attempt_token_is_active(token) else None
|
||||
|
||||
def _attempt_token_is_active(self, attempt_token: str) -> bool:
|
||||
return any(
|
||||
attempt_token in attempts
|
||||
for attempts in self._attempts.values()
|
||||
)
|
||||
|
||||
def _settle_handoff(self, handoff_token: str) -> None:
|
||||
self._handoff_refs.pop(handoff_token, None)
|
||||
if not self._handoff_refs:
|
||||
self._handoffs_drained.set()
|
||||
|
||||
@property
|
||||
def admission_closed(self) -> bool:
|
||||
return self._admission_closed
|
||||
|
||||
def close_admission(self) -> None:
|
||||
"""Reject future attempts without dropping attempts already in flight."""
|
||||
|
||||
self._admission_closed = True
|
||||
|
||||
async def close_admission_and_wait_for_handoffs(self) -> None:
|
||||
"""Close ingress and wait until every already-accepted request hands off.
|
||||
|
||||
A bound pending reservation may still call ``register`` after admission
|
||||
closes. That registration atomically drains the reservation and turns
|
||||
the real coroutine into the sole active fact. This wait therefore ends
|
||||
at handoff, never at completion of the potentially long execution.
|
||||
"""
|
||||
|
||||
self.close_admission()
|
||||
while self._handoff_refs:
|
||||
await self._handoffs_drained.wait()
|
||||
|
||||
@property
|
||||
def pending_handoff_count(self) -> int:
|
||||
return len(self._handoff_refs)
|
||||
|
||||
def is_handoff_pending(self, handoff_token: str | None) -> bool:
|
||||
return bool(handoff_token and handoff_token in self._handoff_refs)
|
||||
|
||||
def scope_lock(
|
||||
self,
|
||||
project_id: str | None,
|
||||
runtime_session_id: str | None,
|
||||
) -> asyncio.Lock:
|
||||
"""Return the controller-shared lock for one durable runtime scope."""
|
||||
|
||||
project = str(project_id or "default").strip() or "default"
|
||||
session = str(runtime_session_id or "").strip()
|
||||
if not session:
|
||||
raise ValueError("runtime_session_id is required")
|
||||
key = (project, session)
|
||||
lock = self._scope_locks.get(key)
|
||||
if lock is None:
|
||||
lock = asyncio.Lock()
|
||||
self._scope_locks[key] = lock
|
||||
return lock
|
||||
|
||||
def unregister(
|
||||
self,
|
||||
project_id: str | None,
|
||||
task_id: str | None,
|
||||
attempt_token: str,
|
||||
) -> bool:
|
||||
key = self._key(project_id, task_id)
|
||||
attempts = self._attempts.get(key)
|
||||
if not attempts or attempt_token not in attempts:
|
||||
return False
|
||||
attempts.remove(attempt_token)
|
||||
if not attempts:
|
||||
self._attempts.pop(key, None)
|
||||
return True
|
||||
|
||||
def is_active(self, project_id: str | None, task_id: str | None) -> bool:
|
||||
return bool(self._attempts.get(self._key(project_id, task_id)))
|
||||
|
||||
def active_task_ids(self, project_id: str | None) -> set[str]:
|
||||
project = str(project_id or "default").strip() or "default"
|
||||
return {
|
||||
task_id
|
||||
for (candidate_project, task_id), attempts in self._attempts.items()
|
||||
if candidate_project == project and attempts
|
||||
}
|
||||
|
||||
def attempt_count(self, project_id: str | None, task_id: str | None) -> int:
|
||||
return len(self._attempts.get(self._key(project_id, task_id), ()))
|
||||
+358
-57
@@ -2862,32 +2862,6 @@ class OPCStore:
|
||||
await self._db.commit()
|
||||
return cursor.rowcount > 0
|
||||
|
||||
async def renew_task_lock(self, task_id: str) -> bool:
|
||||
"""Refresh ``execution_locked_at`` for a live running task.
|
||||
|
||||
This is the heartbeat used by the WS handler while it is actively
|
||||
processing a session message. It only updates rows whose ``status`` is
|
||||
still ``running`` (anything else has already ended and must not keep a
|
||||
live timestamp), and deliberately ignores ``execution_lock``: that bit
|
||||
is set only by delegation checkout, while the office_ui dispatch path
|
||||
serializes through an in-memory asyncio.Lock and never flips it. The
|
||||
refreshed ``execution_locked_at`` is what ``reset_orphan_running_tasks``
|
||||
compares against on startup to distinguish live work from abandoned
|
||||
``running`` rows.
|
||||
|
||||
Returns ``True`` when the row was still ``status=running`` (heartbeat
|
||||
extended), ``False`` otherwise (task ended or gone — caller should
|
||||
stop heartbeating).
|
||||
"""
|
||||
assert self._db
|
||||
now_iso = datetime.now().isoformat()
|
||||
async with self._db.execute(
|
||||
"UPDATE tasks SET execution_locked_at = ? WHERE id = ? AND status = 'running'",
|
||||
(now_iso, task_id),
|
||||
) as cursor:
|
||||
await self._db.commit()
|
||||
return cursor.rowcount > 0
|
||||
|
||||
async def release_task_lock(self, task_id: str) -> None:
|
||||
assert self._db
|
||||
await self._db.execute(
|
||||
@@ -2896,37 +2870,6 @@ class OPCStore:
|
||||
)
|
||||
await self._db.commit()
|
||||
|
||||
async def reset_orphan_running_tasks(self, *, lease_seconds: int = 300) -> dict[str, int]:
|
||||
"""Reset ``status=running`` tasks abandoned by a prior process.
|
||||
|
||||
Safe single-process assumption: when this runs during server startup,
|
||||
any task that still claims ``status=running`` must be an orphan — its
|
||||
worker coroutine died with the previous process. We revert it to
|
||||
``idle`` so the UI can Continue it, and clear any stale execution lock
|
||||
regardless of status when the lease has expired.
|
||||
|
||||
Returns a summary dict with keys ``statuses_reset`` and
|
||||
``locks_cleared`` giving the affected row counts.
|
||||
"""
|
||||
assert self._db
|
||||
cutoff_iso = (datetime.now() - timedelta(seconds=int(lease_seconds))).isoformat()
|
||||
async with self._db.execute(
|
||||
"UPDATE tasks SET status = 'idle', execution_lock = 0, execution_locked_at = NULL "
|
||||
"WHERE status = 'running' AND ("
|
||||
"execution_locked_at IS NULL OR execution_locked_at < ?"
|
||||
")",
|
||||
(cutoff_iso,),
|
||||
) as cursor:
|
||||
statuses_reset = cursor.rowcount or 0
|
||||
async with self._db.execute(
|
||||
"UPDATE tasks SET execution_lock = 0, execution_locked_at = NULL "
|
||||
"WHERE execution_lock = 1 AND execution_locked_at IS NOT NULL AND execution_locked_at < ?",
|
||||
(cutoff_iso,),
|
||||
) as cursor:
|
||||
locks_cleared = cursor.rowcount or 0
|
||||
await self._db.commit()
|
||||
return {"statuses_reset": statuses_reset, "locks_cleared": locks_cleared}
|
||||
|
||||
def _row_to_task(self, row: Any, description: Any) -> Task:
|
||||
cols = [d[0] for d in description]
|
||||
data = dict(zip(cols, row))
|
||||
@@ -5509,6 +5452,108 @@ class OPCStore:
|
||||
await self.save_delegation_work_item(item)
|
||||
return item
|
||||
|
||||
async def claim_delegation_work_item_if_dispatchable(
|
||||
self,
|
||||
work_item_id: str,
|
||||
*,
|
||||
expected_phase: Phase | str,
|
||||
role_runtime_session_id: str,
|
||||
seat_id: str,
|
||||
task_id: str,
|
||||
work_item_revision: int = 0,
|
||||
) -> DelegationWorkItem | None:
|
||||
"""Atomically claim an unheld, unowned dispatchable WorkItem.
|
||||
|
||||
The dispatcher may be operating on a snapshot loaded before a Stop or
|
||||
shutdown transition. Keeping the phase, claim, queue, and durable
|
||||
hold predicates in the same UPDATE prevents that stale snapshot from
|
||||
resurrecting a suspended WorkItem.
|
||||
"""
|
||||
|
||||
phase = coerce_phase(expected_phase)
|
||||
dispatchable_phases = {
|
||||
Phase.READY,
|
||||
Phase.READY_FOR_REWORK,
|
||||
*IN_PROGRESS_PHASES,
|
||||
}
|
||||
if phase not in dispatchable_phases:
|
||||
return None
|
||||
if phase != Phase.RUNNING:
|
||||
validate_transition(phase, Phase.RUNNING)
|
||||
role_session_id = str(role_runtime_session_id or "").strip()
|
||||
claimed_task_id = str(task_id or "").strip()
|
||||
if not work_item_id or not role_session_id or not claimed_task_id:
|
||||
return None
|
||||
|
||||
updated_at = datetime.now()
|
||||
db = self._require_db()
|
||||
cursor = await db.execute(
|
||||
"""UPDATE delegation_work_items
|
||||
SET phase = ?,
|
||||
role_runtime_session_id = ?,
|
||||
claimed_by_role_runtime_session_id = ?,
|
||||
claimed_by_seat_id = ?,
|
||||
metadata = json_set(
|
||||
COALESCE(NULLIF(metadata, ''), '{}'),
|
||||
'$.claimed_by_role_session_id', ?,
|
||||
'$.claimed_task_id', ?,
|
||||
'$.claimed_work_item_revision', ?
|
||||
),
|
||||
updated_at = ?
|
||||
WHERE work_item_id = ?
|
||||
AND phase = ?
|
||||
AND COALESCE(claimed_by_role_runtime_session_id, '') = ''
|
||||
AND COALESCE(claimed_by_seat_id, '') = ''
|
||||
AND COALESCE(json_extract(metadata, '$.claimed_by_role_session_id'), '') = ''
|
||||
AND COALESCE(json_extract(metadata, '$.claimed_task_id'), '') = ''
|
||||
AND COALESCE(json_extract(metadata, '$.dispatch_hold'), '') = ''
|
||||
AND COALESCE(json_extract(metadata, '$.queued_behind_session'), '') = ''
|
||||
AND COALESCE(
|
||||
CAST(json_extract(metadata, '$.manager_mutation_revision') AS INTEGER),
|
||||
0
|
||||
) = ?""",
|
||||
(
|
||||
Phase.RUNNING.value,
|
||||
role_session_id,
|
||||
role_session_id,
|
||||
str(seat_id or "").strip(),
|
||||
role_session_id,
|
||||
claimed_task_id,
|
||||
int(work_item_revision or 0),
|
||||
updated_at.isoformat(),
|
||||
str(work_item_id or "").strip(),
|
||||
phase.value,
|
||||
int(work_item_revision or 0),
|
||||
),
|
||||
)
|
||||
await db.commit()
|
||||
if not (getattr(cursor, "rowcount", 0) or 0):
|
||||
return None
|
||||
persisted = await self.get_delegation_work_item(work_item_id)
|
||||
if persisted is None:
|
||||
return None
|
||||
persisted_metadata = dict(persisted.metadata or {})
|
||||
if (
|
||||
persisted.phase != Phase.RUNNING
|
||||
or str(persisted.claimed_by_role_runtime_session_id or "").strip()
|
||||
!= role_session_id
|
||||
or str(persisted_metadata.get("claimed_by_role_session_id", "") or "").strip()
|
||||
!= role_session_id
|
||||
or str(persisted_metadata.get("claimed_task_id", "") or "").strip()
|
||||
!= claimed_task_id
|
||||
or str(persisted_metadata.get("dispatch_hold", "") or "").strip()
|
||||
or str(persisted_metadata.get("queued_behind_session", "") or "").strip()
|
||||
):
|
||||
# Stop/shutdown may have won immediately after the CAS commit and
|
||||
# cleared this claim while adding a durable hold. The committed
|
||||
# UPDATE is not permission to spawn once that newer state exists.
|
||||
return None
|
||||
# Do not fire the generic phase hooks here. The executor's first
|
||||
# idempotent RUNNING transition performs projection. Deferring it
|
||||
# avoids a post-commit hook racing after Stop and projecting the Task
|
||||
# back to RUNNING from an already-held WorkItem.
|
||||
return persisted
|
||||
|
||||
async def apply_delegation_review_resolution(
|
||||
self,
|
||||
work_item_id: str,
|
||||
@@ -7148,6 +7193,262 @@ class OPCStore:
|
||||
)
|
||||
await self._db.commit()
|
||||
|
||||
async def get_or_create_active_execution_checkpoint(
|
||||
self,
|
||||
checkpoint: ExecutionCheckpoint,
|
||||
*,
|
||||
checkpoint_types: list[str] | tuple[str, ...] | set[str],
|
||||
create_if_missing: bool = True,
|
||||
) -> tuple[ExecutionCheckpoint | None, bool]:
|
||||
"""Atomically reuse or create one active checkpoint for a session scope.
|
||||
|
||||
``BEGIN IMMEDIATE`` serializes checkpoint creation across independent
|
||||
controller/store connections. It also repairs historical duplicate
|
||||
active rows in the same transaction, so concurrent startup/shutdown
|
||||
reconcilers can never supersede each other's newly-created checkpoint
|
||||
and leave the scope without a durable recovery point.
|
||||
"""
|
||||
|
||||
assert self._db
|
||||
clean_types = sorted(
|
||||
{
|
||||
str(item).strip()
|
||||
for item in checkpoint_types
|
||||
if str(item).strip()
|
||||
}
|
||||
)
|
||||
project_id = str(checkpoint.project_id or "default").strip() or "default"
|
||||
session_id = str(checkpoint.session_id or "").strip()
|
||||
checkpoint_type = str(checkpoint.checkpoint_type or "").strip()
|
||||
if not session_id:
|
||||
raise ValueError("active execution checkpoint requires session_id")
|
||||
if checkpoint_type not in clean_types:
|
||||
raise ValueError(
|
||||
"checkpoint_type must be included in the active checkpoint scope"
|
||||
)
|
||||
|
||||
placeholders = ", ".join("?" for _ in clean_types)
|
||||
try:
|
||||
await self._db.execute("BEGIN IMMEDIATE")
|
||||
async with self._db.execute(
|
||||
f"""SELECT * FROM execution_checkpoints
|
||||
WHERE project_id = ?
|
||||
AND session_id = ?
|
||||
AND checkpoint_type IN ({placeholders})
|
||||
AND status IN ('pending', 'resuming')
|
||||
ORDER BY updated_at DESC, created_at DESC, checkpoint_id DESC""",
|
||||
(project_id, session_id, *clean_types),
|
||||
) as cursor:
|
||||
rows = await cursor.fetchall()
|
||||
cols = [description[0] for description in cursor.description]
|
||||
|
||||
if rows:
|
||||
decoded = [dict(zip(cols, row)) for row in rows]
|
||||
winner_data = decoded[0]
|
||||
winner_id = str(winner_data["checkpoint_id"])
|
||||
now = datetime.now().isoformat()
|
||||
for duplicate in decoded[1:]:
|
||||
duplicate_id = str(duplicate.get("checkpoint_id", "") or "").strip()
|
||||
if not duplicate_id:
|
||||
continue
|
||||
payload = _json_loads(duplicate.get("payload"), {})
|
||||
payload["superseded_at"] = now
|
||||
payload["superseded_by_checkpoint_id"] = winner_id
|
||||
await self._db.execute(
|
||||
"""UPDATE execution_checkpoints
|
||||
SET status = 'superseded', payload = ?, updated_at = ?
|
||||
WHERE checkpoint_id = ? AND status IN ('pending', 'resuming')""",
|
||||
(_json_dumps(payload), now, duplicate_id),
|
||||
)
|
||||
await self._db.commit()
|
||||
return (
|
||||
ExecutionCheckpoint(
|
||||
checkpoint_id=winner_id,
|
||||
project_id=str(winner_data["project_id"]),
|
||||
session_id=winner_data.get("session_id"),
|
||||
checkpoint_type=str(winner_data["checkpoint_type"]),
|
||||
status=str(winner_data["status"]),
|
||||
task_id=winner_data.get("task_id"),
|
||||
payload=_json_loads(winner_data.get("payload"), {}),
|
||||
created_at=datetime.fromisoformat(str(winner_data["created_at"])),
|
||||
updated_at=datetime.fromisoformat(str(winner_data["updated_at"])),
|
||||
),
|
||||
False,
|
||||
)
|
||||
|
||||
if not create_if_missing:
|
||||
await self._db.commit()
|
||||
return None, False
|
||||
|
||||
await self._db.execute(
|
||||
"""INSERT INTO execution_checkpoints
|
||||
(checkpoint_id, project_id, session_id, checkpoint_type, status,
|
||||
task_id, payload, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
(
|
||||
checkpoint.checkpoint_id,
|
||||
project_id,
|
||||
session_id,
|
||||
checkpoint_type,
|
||||
checkpoint.status,
|
||||
checkpoint.task_id,
|
||||
_json_dumps(checkpoint.payload),
|
||||
checkpoint.created_at.isoformat(),
|
||||
checkpoint.updated_at.isoformat(),
|
||||
),
|
||||
)
|
||||
await self._db.commit()
|
||||
return checkpoint, True
|
||||
except Exception:
|
||||
await self._db.rollback()
|
||||
raise
|
||||
|
||||
async def normalize_active_execution_checkpoints(
|
||||
self,
|
||||
*,
|
||||
project_id: str,
|
||||
session_id: str,
|
||||
checkpoint_types: list[str] | tuple[str, ...] | set[str],
|
||||
) -> ExecutionCheckpoint | None:
|
||||
"""Return one active scope owner while superseding duplicate rows.
|
||||
|
||||
Unlike creation, startup reconciliation must not manufacture a
|
||||
checkpoint merely because none exists. This wrapper reuses the same
|
||||
serialized transaction and duplicate winner rule with insertion
|
||||
explicitly disabled.
|
||||
"""
|
||||
|
||||
clean_types = sorted(
|
||||
{
|
||||
str(item).strip()
|
||||
for item in checkpoint_types
|
||||
if str(item).strip()
|
||||
}
|
||||
)
|
||||
if not clean_types or not str(session_id or "").strip():
|
||||
return None
|
||||
candidate = ExecutionCheckpoint(
|
||||
project_id=str(project_id or "default").strip() or "default",
|
||||
session_id=str(session_id or "").strip(),
|
||||
checkpoint_type=clean_types[0],
|
||||
)
|
||||
checkpoint, _created = await self.get_or_create_active_execution_checkpoint(
|
||||
candidate,
|
||||
checkpoint_types=clean_types,
|
||||
create_if_missing=False,
|
||||
)
|
||||
return checkpoint
|
||||
|
||||
async def compare_and_set_execution_checkpoint(
|
||||
self,
|
||||
checkpoint_id: str,
|
||||
*,
|
||||
expected_statuses: list[str] | tuple[str, ...] | set[str],
|
||||
status: str,
|
||||
payload: dict[str, Any],
|
||||
updated_at: datetime | None = None,
|
||||
) -> bool:
|
||||
"""Atomically transition a checkpoint only from an expected state.
|
||||
|
||||
The conditional UPDATE is the cross-controller guard for checkpoint
|
||||
consumption. In-memory scope locks serialize one Office controller;
|
||||
this CAS prevents a standalone CLI/controller from claiming the same
|
||||
pending runtime concurrently.
|
||||
"""
|
||||
|
||||
assert self._db
|
||||
expected = [
|
||||
str(item).strip()
|
||||
for item in expected_statuses
|
||||
if str(item).strip()
|
||||
]
|
||||
if not checkpoint_id or not expected:
|
||||
return False
|
||||
placeholders = ", ".join("?" for _ in expected)
|
||||
cursor = await self._db.execute(
|
||||
f"""UPDATE execution_checkpoints
|
||||
SET status = ?, payload = ?, updated_at = ?
|
||||
WHERE checkpoint_id = ? AND status IN ({placeholders})""",
|
||||
(
|
||||
str(status or "").strip(),
|
||||
_json_dumps(dict(payload or {})),
|
||||
(updated_at or datetime.now()).isoformat(),
|
||||
checkpoint_id,
|
||||
*expected,
|
||||
),
|
||||
)
|
||||
await self._db.commit()
|
||||
return cursor.rowcount == 1
|
||||
|
||||
async def complete_execution_checkpoint_and_reopen_ui_anchor(
|
||||
self,
|
||||
checkpoint_id: str,
|
||||
*,
|
||||
project_id: str,
|
||||
session_id: str,
|
||||
expected_status: str,
|
||||
status: str,
|
||||
payload: dict[str, Any],
|
||||
ui_anchor_task_id: str = "",
|
||||
updated_at: datetime | None = None,
|
||||
) -> bool:
|
||||
"""Atomically complete a runtime handoff and reopen its UI anchor.
|
||||
|
||||
The anchor must never become chat-runnable if a concurrent Stop already
|
||||
took checkpoint ownership back. Keeping both writes in one SQLite
|
||||
transaction makes the checkpoint CAS the gate for the UI projection.
|
||||
"""
|
||||
|
||||
assert self._db
|
||||
checkpoint_id = str(checkpoint_id or "").strip()
|
||||
project_id = str(project_id or "default").strip() or "default"
|
||||
session_id = str(session_id or "").strip()
|
||||
expected_status = str(expected_status or "").strip()
|
||||
status = str(status or "").strip()
|
||||
ui_anchor_task_id = str(ui_anchor_task_id or "").strip()
|
||||
if not checkpoint_id or not session_id or not expected_status or not status:
|
||||
return False
|
||||
now = updated_at or datetime.now()
|
||||
try:
|
||||
await self._db.execute("BEGIN IMMEDIATE")
|
||||
cursor = await self._db.execute(
|
||||
"""UPDATE execution_checkpoints
|
||||
SET status = ?, payload = ?, updated_at = ?
|
||||
WHERE checkpoint_id = ?
|
||||
AND project_id = ?
|
||||
AND session_id = ?
|
||||
AND status = ?""",
|
||||
(
|
||||
status,
|
||||
_json_dumps(dict(payload or {})),
|
||||
now.isoformat(),
|
||||
checkpoint_id,
|
||||
project_id,
|
||||
session_id,
|
||||
expected_status,
|
||||
),
|
||||
)
|
||||
if cursor.rowcount != 1:
|
||||
await self._db.rollback()
|
||||
return False
|
||||
if ui_anchor_task_id:
|
||||
await self._db.execute(
|
||||
"""UPDATE tasks
|
||||
SET status = ?, execution_lock = 0, execution_locked_at = NULL
|
||||
WHERE id = ? AND project_id = ? AND status = ?""",
|
||||
(
|
||||
TaskStatus.IDLE.value,
|
||||
ui_anchor_task_id,
|
||||
project_id,
|
||||
TaskStatus.CANCELLED.value,
|
||||
),
|
||||
)
|
||||
await self._db.commit()
|
||||
return True
|
||||
except Exception:
|
||||
await self._db.rollback()
|
||||
raise
|
||||
|
||||
async def get_execution_checkpoints(
|
||||
self,
|
||||
project_id: str = "default",
|
||||
|
||||
+1470
-550
File diff suppressed because it is too large
Load Diff
@@ -17,6 +17,10 @@ from typing import Any, Awaitable, Callable
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from opc.core.active_task_runs import (
|
||||
ActiveTaskRunAdmissionClosed,
|
||||
ActiveTaskRunRegistry,
|
||||
)
|
||||
from opc.core.config import DEFAULT_EXTERNAL_AGENT_STARTUP_TIMEOUT_SECONDS, DEFAULT_ORGANIZATION_ID
|
||||
from opc.core.models import (
|
||||
AdaptiveRoleProfile,
|
||||
@@ -456,6 +460,26 @@ class WorkItemOutputBundle:
|
||||
summary: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompanyExecutorDriverOwnership:
|
||||
"""One registry attempt covering a complete company scheduler run."""
|
||||
|
||||
registry: ActiveTaskRunRegistry
|
||||
project_id: str
|
||||
task_id: str
|
||||
attempt_token: str
|
||||
|
||||
def bind(self):
|
||||
return self.registry.bind_driver_attempt(self.attempt_token)
|
||||
|
||||
def release(self) -> bool:
|
||||
return self.registry.unregister(
|
||||
self.project_id,
|
||||
self.task_id,
|
||||
self.attempt_token,
|
||||
)
|
||||
|
||||
|
||||
def serialize_company_runtime_spec(spec: CompanyRuntimeSpec | None) -> dict[str, Any]:
|
||||
if spec is None:
|
||||
return {}
|
||||
@@ -1354,6 +1378,7 @@ class CompanyWorkItemExecutor:
|
||||
store: Any | None = None,
|
||||
llm: Any | None = None,
|
||||
role_prompt_runner: Callable[[Task, str, dict[str, Any], str, bool], Awaitable[str | None]] | None = None,
|
||||
active_task_run_registry: ActiveTaskRunRegistry | None = None,
|
||||
) -> None:
|
||||
self.org_engine = org_engine
|
||||
self.communication = communication
|
||||
@@ -1373,6 +1398,7 @@ class CompanyWorkItemExecutor:
|
||||
self.on_kanban_changed = on_kanban_changed
|
||||
self.work_item_timeout = work_item_timeout
|
||||
self.role_prompt_runner = role_prompt_runner
|
||||
self.active_task_run_registry = active_task_run_registry
|
||||
self._default_run_state = CompanyExecutorRunState()
|
||||
self._run_state_var: ContextVar[CompanyExecutorRunState | None] = ContextVar(
|
||||
f"company-executor-run-state:{id(self)}",
|
||||
@@ -3368,12 +3394,6 @@ class CompanyWorkItemExecutor:
|
||||
"cell_id": work_item.cell_id,
|
||||
"parent_work_item_id": work_item.parent_work_item_id,
|
||||
}
|
||||
if work_item.phase == Phase.PAUSED:
|
||||
task.metadata.setdefault("interrupted_recovery", {
|
||||
"reason": "work_item_interrupted",
|
||||
"detected_at": datetime.now().isoformat(),
|
||||
})
|
||||
|
||||
if task.metadata != before_metadata:
|
||||
changed = True
|
||||
return changed
|
||||
@@ -4359,14 +4379,76 @@ class CompanyWorkItemExecutor:
|
||||
return "dispatch_required"
|
||||
return "worker_execute"
|
||||
|
||||
async def execute(self, plan: CompanyWorkItemRuntimePlan, tasks: list[Task]) -> str:
|
||||
plan = _coerce_company_work_item_runtime_plan(plan) or CompanyWorkItemRuntimePlan()
|
||||
plan.metadata = {
|
||||
**dict(plan.metadata or {}),
|
||||
"execution_model": "multi_team_org",
|
||||
"runtime_model": "multi_team_org",
|
||||
@staticmethod
|
||||
def _driver_ownership_task(
|
||||
tasks: list[Task],
|
||||
*,
|
||||
preferred_task_ids: set[str] | None = None,
|
||||
) -> Task | None:
|
||||
preferred = {
|
||||
str(task_id or "").strip()
|
||||
for task_id in set(preferred_task_ids or set())
|
||||
if str(task_id or "").strip()
|
||||
}
|
||||
return await self._execute_multi_team_org(plan, tasks)
|
||||
candidates = [
|
||||
task
|
||||
for task in tasks
|
||||
if str(getattr(task, "id", "") or "").strip()
|
||||
and (not preferred or task.id in preferred)
|
||||
]
|
||||
for task in candidates:
|
||||
if linked_work_item_id_for_task(task):
|
||||
return task
|
||||
for task in candidates:
|
||||
if is_work_item_runtime_metadata(dict(task.metadata or {})):
|
||||
return task
|
||||
return candidates[0] if candidates else None
|
||||
|
||||
def acquire_driver_ownership(
|
||||
self,
|
||||
tasks: list[Task],
|
||||
*,
|
||||
preferred_task_ids: set[str] | None = None,
|
||||
) -> CompanyExecutorDriverOwnership | None:
|
||||
registry = self.active_task_run_registry
|
||||
task = self._driver_ownership_task(
|
||||
tasks,
|
||||
preferred_task_ids=preferred_task_ids,
|
||||
)
|
||||
if registry is None or task is None:
|
||||
return None
|
||||
project_id = str(task.project_id or "default").strip() or "default"
|
||||
try:
|
||||
attempt_token = registry.register(project_id, task.id)
|
||||
except ActiveTaskRunAdmissionClosed as exc:
|
||||
raise asyncio.CancelledError(str(exc)) from exc
|
||||
return CompanyExecutorDriverOwnership(
|
||||
registry=registry,
|
||||
project_id=project_id,
|
||||
task_id=task.id,
|
||||
attempt_token=attempt_token,
|
||||
)
|
||||
|
||||
async def execute(
|
||||
self,
|
||||
plan: CompanyWorkItemRuntimePlan,
|
||||
tasks: list[Task],
|
||||
) -> str:
|
||||
ownership = self.acquire_driver_ownership(tasks)
|
||||
try:
|
||||
plan = _coerce_company_work_item_runtime_plan(plan) or CompanyWorkItemRuntimePlan()
|
||||
plan.metadata = {
|
||||
**dict(plan.metadata or {}),
|
||||
"execution_model": "multi_team_org",
|
||||
"runtime_model": "multi_team_org",
|
||||
}
|
||||
if ownership is None:
|
||||
return await self._execute_multi_team_org(plan, tasks)
|
||||
with ownership.bind():
|
||||
return await self._execute_multi_team_org(plan, tasks)
|
||||
finally:
|
||||
if ownership is not None:
|
||||
ownership.release()
|
||||
|
||||
async def _execute_multi_team_org(
|
||||
self,
|
||||
@@ -4501,11 +4583,11 @@ class CompanyWorkItemExecutor:
|
||||
# Claim whatever is immediately claimable and spawn each
|
||||
# work item as an independent asyncio.Task so the loop no
|
||||
# longer blocks on the slowest sibling.
|
||||
claims = await self.runtime.claim_runnable_tasks(tasks, work_items=work_items)
|
||||
for member_session, claimed_task in claims:
|
||||
work_item_coro = self._run_claimed_work_item(member_session, claimed_task, {})
|
||||
work_item_task = asyncio.create_task(work_item_coro)
|
||||
active_work_item_tasks[work_item_task] = (member_session, claimed_task)
|
||||
claims = await self._claim_and_create_work_item_tasks(
|
||||
tasks,
|
||||
work_items,
|
||||
active_work_item_tasks,
|
||||
)
|
||||
# Termination: only when nothing is in-flight AND nothing
|
||||
# else is runnable. If work items are still running, even an
|
||||
# "empty runnable" snapshot may become non-empty within
|
||||
@@ -4608,10 +4690,6 @@ class CompanyWorkItemExecutor:
|
||||
self._schedule_kanban_notification()
|
||||
except asyncio.CancelledError:
|
||||
claimed_pairs = list(active_work_item_tasks.values())
|
||||
claimed_tasks = [
|
||||
claimed_task
|
||||
for _member_session, claimed_task in claimed_pairs
|
||||
]
|
||||
for work_item_task in list(active_work_item_tasks.keys()):
|
||||
if not work_item_task.done():
|
||||
work_item_task.cancel()
|
||||
@@ -4660,63 +4738,6 @@ class CompanyWorkItemExecutor:
|
||||
)
|
||||
except Exception:
|
||||
logger.opt(exception=True).debug("company runtime cancellation: failed session idle reset")
|
||||
for claimed_task in claimed_tasks:
|
||||
if claimed_task.status in {TaskStatus.DONE, TaskStatus.FAILED, TaskStatus.CANCELLED}:
|
||||
continue
|
||||
claimed_task.metadata = dict(claimed_task.metadata or {})
|
||||
claimed_task.metadata["company_runtime_suspended_at"] = datetime.now().isoformat()
|
||||
claimed_task.metadata.setdefault("last_stop_reason", "runtime_cancelled")
|
||||
claimed_task.metadata["company_runtime_stop_state"] = "suspended"
|
||||
claimed_task.metadata["company_runtime_stop_marked_at"] = (
|
||||
claimed_task.metadata.get("company_runtime_stop_marked_at") or datetime.now().isoformat()
|
||||
)
|
||||
claimed_task.metadata.setdefault(
|
||||
"suspended_task_status",
|
||||
claimed_task.status.value if isinstance(claimed_task.status, TaskStatus) else str(claimed_task.status or ""),
|
||||
)
|
||||
work_item_id = linked_work_item_id_for_task(claimed_task)
|
||||
try:
|
||||
if work_item_id and self._store_is_ready(self.store) and hasattr(self.store, "get_delegation_work_item"):
|
||||
work_item = await self.store.get_delegation_work_item(work_item_id)
|
||||
else:
|
||||
work_item = None
|
||||
if work_item is not None and getattr(work_item, "phase", None) not in {Phase.APPROVED, Phase.FAILED, Phase.CANCELLED}:
|
||||
phase = getattr(work_item, "phase", Phase.RUNNING)
|
||||
phase_value = phase.value if isinstance(phase, Phase) else str(phase or "")
|
||||
original_claim = {
|
||||
"claimed_by_role_runtime_session_id": str(getattr(work_item, "claimed_by_role_runtime_session_id", "") or ""),
|
||||
"claimed_by_seat_id": str(getattr(work_item, "claimed_by_seat_id", "") or ""),
|
||||
"claimed_by_role_session_id": str((getattr(work_item, "metadata", {}) or {}).get("claimed_by_role_session_id", "") or ""),
|
||||
"claimed_task_id": str((getattr(work_item, "metadata", {}) or {}).get("claimed_task_id", "") or claimed_task.id),
|
||||
}
|
||||
await self.store.update_delegation_work_item(
|
||||
work_item_id,
|
||||
metadata_updates={
|
||||
"dispatch_hold": "company_runtime_suspended",
|
||||
"suspended_at": datetime.now().isoformat(),
|
||||
"suspend_reason": claimed_task.metadata.get("last_stop_reason", "runtime_cancelled"),
|
||||
"suspended_phase": phase_value,
|
||||
"suspended_task_status": claimed_task.metadata.get("suspended_task_status", ""),
|
||||
"suspended_claim": original_claim,
|
||||
"claimed_by_role_session_id": "",
|
||||
"claimed_task_id": "",
|
||||
},
|
||||
claimed_by_role_runtime_session_id="",
|
||||
claimed_by_seat_id="",
|
||||
)
|
||||
claimed_task.metadata["dispatch_hold"] = "company_runtime_suspended"
|
||||
claimed_task.metadata["suspended_phase"] = phase_value
|
||||
except Exception:
|
||||
logger.opt(exception=True).debug(
|
||||
"company runtime cancellation: failed suspend hold release",
|
||||
)
|
||||
if self.save_task and self._store_is_ready(self.store):
|
||||
try:
|
||||
await self.save_task(claimed_task)
|
||||
except Exception:
|
||||
logger.opt(exception=True).debug(
|
||||
"company runtime cancellation: failed suspended task save",
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
# Drain any work items still running (shouldn't happen given the
|
||||
@@ -4739,6 +4760,97 @@ class CompanyWorkItemExecutor:
|
||||
active_work_item_tasks.clear()
|
||||
return self._summarize_multi_team_org_results(tasks)
|
||||
|
||||
@staticmethod
|
||||
def _runtime_scope_for_tasks(tasks: list[Task]) -> tuple[str, str]:
|
||||
project_id = "default"
|
||||
runtime_session_id = ""
|
||||
for task in tasks:
|
||||
project_id = str(task.project_id or project_id).strip() or "default"
|
||||
metadata = dict(task.metadata or {})
|
||||
runtime_session_id = str(
|
||||
getattr(task, "parent_session_id", "")
|
||||
or metadata.get("company_runtime_root_session_id")
|
||||
or metadata.get("parent_session_id")
|
||||
or ""
|
||||
).strip()
|
||||
if runtime_session_id:
|
||||
return project_id, runtime_session_id
|
||||
for task in tasks:
|
||||
runtime_session_id = str(getattr(task, "session_id", "") or "").strip()
|
||||
if runtime_session_id:
|
||||
return project_id, runtime_session_id
|
||||
return project_id, ""
|
||||
|
||||
async def _claim_and_create_work_item_tasks(
|
||||
self,
|
||||
tasks: list[Task],
|
||||
work_items: list[DelegationWorkItem],
|
||||
active_work_item_tasks: dict[
|
||||
asyncio.Task[TaskResult | None],
|
||||
tuple[CompanyMemberSession, Task],
|
||||
],
|
||||
) -> list[tuple[CompanyMemberSession, Task]]:
|
||||
"""Keep durable claim and coroutine ownership in one scope boundary."""
|
||||
|
||||
async def claim_and_create() -> list[tuple[CompanyMemberSession, Task]]:
|
||||
claims = await self.runtime.claim_runnable_tasks(
|
||||
tasks,
|
||||
work_items=work_items,
|
||||
)
|
||||
for member_session, claimed_task in claims:
|
||||
work_item_task = self._create_claimed_work_item_task(
|
||||
member_session,
|
||||
claimed_task,
|
||||
{},
|
||||
)
|
||||
active_work_item_tasks[work_item_task] = (
|
||||
member_session,
|
||||
claimed_task,
|
||||
)
|
||||
return claims
|
||||
|
||||
registry = self.active_task_run_registry
|
||||
project_id, runtime_session_id = self._runtime_scope_for_tasks(tasks)
|
||||
if registry is None or not runtime_session_id:
|
||||
return await claim_and_create()
|
||||
async with registry.scope_lock(project_id, runtime_session_id):
|
||||
return await claim_and_create()
|
||||
|
||||
def _create_claimed_work_item_task(
|
||||
self,
|
||||
member_session: CompanyMemberSession,
|
||||
task: Task,
|
||||
task_by_projection_id: dict[str, Task],
|
||||
) -> asyncio.Task[TaskResult | None]:
|
||||
"""Register ownership before scheduling the full claimed-item coroutine."""
|
||||
|
||||
registry = self.active_task_run_registry
|
||||
project_id = str(task.project_id or "default").strip() or "default"
|
||||
attempt_token = ""
|
||||
if registry is not None:
|
||||
try:
|
||||
attempt_token = registry.register(project_id, task.id)
|
||||
except ActiveTaskRunAdmissionClosed as exc:
|
||||
raise asyncio.CancelledError(str(exc)) from exc
|
||||
|
||||
async def run_owned() -> TaskResult | None:
|
||||
try:
|
||||
return await self._run_claimed_work_item(
|
||||
member_session,
|
||||
task,
|
||||
task_by_projection_id,
|
||||
)
|
||||
finally:
|
||||
if registry is not None and attempt_token:
|
||||
registry.unregister(project_id, task.id, attempt_token)
|
||||
|
||||
try:
|
||||
return asyncio.create_task(run_owned())
|
||||
except BaseException:
|
||||
if registry is not None and attempt_token:
|
||||
registry.unregister(project_id, task.id, attempt_token)
|
||||
raise
|
||||
|
||||
async def _run_claimed_work_item(
|
||||
self,
|
||||
member_session: CompanyMemberSession,
|
||||
@@ -5298,69 +5410,10 @@ class CompanyWorkItemExecutor:
|
||||
timeout=self.work_item_timeout,
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
task.metadata = dict(task.metadata or {})
|
||||
task.metadata["company_runtime_suspended_at"] = datetime.now().isoformat()
|
||||
task.metadata.setdefault("last_stop_reason", "runtime_cancelled")
|
||||
task.metadata["company_runtime_stop_state"] = "suspended"
|
||||
task.metadata["company_runtime_stop_marked_at"] = (
|
||||
task.metadata.get("company_runtime_stop_marked_at") or datetime.now().isoformat()
|
||||
)
|
||||
task.metadata.setdefault(
|
||||
"suspended_task_status",
|
||||
task.status.value if isinstance(task.status, TaskStatus) else str(task.status or ""),
|
||||
)
|
||||
work_item_id = linked_work_item_id_for_task(task)
|
||||
store_ready = self._store_is_ready(self.store)
|
||||
if work_item_id and self.store and store_ready:
|
||||
task.metadata.pop("progress_log", None)
|
||||
await append_work_item_progress(
|
||||
self.store,
|
||||
work_item_id,
|
||||
"Work item suspended by runtime cancellation.",
|
||||
)
|
||||
else:
|
||||
progress = list(task.metadata.get("progress_log", []) or [])
|
||||
progress.append("Work item suspended by runtime cancellation.")
|
||||
task.metadata["progress_log"] = progress[-20:]
|
||||
try:
|
||||
work_item = (
|
||||
await self.store.get_delegation_work_item(work_item_id)
|
||||
if work_item_id and store_ready and hasattr(self.store, "get_delegation_work_item")
|
||||
else None
|
||||
)
|
||||
if work_item is not None and getattr(work_item, "phase", None) not in {Phase.APPROVED, Phase.FAILED, Phase.CANCELLED}:
|
||||
phase = getattr(work_item, "phase", Phase.RUNNING)
|
||||
phase_value = phase.value if isinstance(phase, Phase) else str(phase or "")
|
||||
original_claim = {
|
||||
"claimed_by_role_runtime_session_id": str(getattr(work_item, "claimed_by_role_runtime_session_id", "") or ""),
|
||||
"claimed_by_seat_id": str(getattr(work_item, "claimed_by_seat_id", "") or ""),
|
||||
"claimed_by_role_session_id": str((getattr(work_item, "metadata", {}) or {}).get("claimed_by_role_session_id", "") or ""),
|
||||
"claimed_task_id": str((getattr(work_item, "metadata", {}) or {}).get("claimed_task_id", "") or task.id),
|
||||
}
|
||||
await self.store.update_delegation_work_item(
|
||||
work_item_id,
|
||||
metadata_updates={
|
||||
"dispatch_hold": "company_runtime_suspended",
|
||||
"suspended_at": datetime.now().isoformat(),
|
||||
"suspend_reason": task.metadata.get("last_stop_reason", "runtime_cancelled"),
|
||||
"suspended_phase": phase_value,
|
||||
"suspended_task_status": task.metadata.get("suspended_task_status", ""),
|
||||
"suspended_claim": original_claim,
|
||||
"claimed_by_role_session_id": "",
|
||||
"claimed_task_id": "",
|
||||
},
|
||||
claimed_by_role_runtime_session_id="",
|
||||
claimed_by_seat_id="",
|
||||
)
|
||||
task.metadata["dispatch_hold"] = "company_runtime_suspended"
|
||||
task.metadata["suspended_phase"] = phase_value
|
||||
task.status = task_status_for_phase(phase) if isinstance(phase, Phase) else task.status
|
||||
except Exception:
|
||||
logger.opt(exception=True).debug(
|
||||
"company runtime cancellation: failed to apply suspend hold",
|
||||
)
|
||||
if self.save_task and self._store_is_ready(self.store):
|
||||
await self.save_task(task)
|
||||
# Suspension is a checkpoint transition owned by OPCEngine.
|
||||
# This task object may be stale by the time cancellation is
|
||||
# observed, so persisting it here can erase the canonical
|
||||
# checkpoint type, stop intent, or WorkItem hold.
|
||||
raise
|
||||
except asyncio.TimeoutError:
|
||||
logger.error(f"Company work item {projection_id} timed out after {self.work_item_timeout}s")
|
||||
|
||||
@@ -1373,7 +1373,6 @@ class CompanyRuntime:
|
||||
_skip("no task materialized for work_item this tick",
|
||||
session=session_label, work_item_id=work_item_id)
|
||||
continue
|
||||
self._claimed_work_item_ids.add(work_item_id)
|
||||
else:
|
||||
task_id = queued_item_id
|
||||
self._queued_task_ids.discard(task_id)
|
||||
@@ -1392,6 +1391,30 @@ class CompanyRuntime:
|
||||
status=getattr(task, "status", None))
|
||||
continue
|
||||
self._claimed_task_ids.add(task_id)
|
||||
if work_item is not None:
|
||||
claimed = await self._claim_role_session_work_item(
|
||||
session,
|
||||
work_item,
|
||||
task,
|
||||
)
|
||||
if not claimed:
|
||||
fresh_work_item = None
|
||||
get_work_item = getattr(
|
||||
self.store,
|
||||
"get_delegation_work_item",
|
||||
None,
|
||||
)
|
||||
if callable(get_work_item):
|
||||
fresh_work_item = await get_work_item(work_item_id)
|
||||
if fresh_work_item is not None:
|
||||
work_item_map[work_item_id] = fresh_work_item
|
||||
_skip(
|
||||
"atomic WorkItem claim lost to a phase/hold/owner update",
|
||||
session=session_label,
|
||||
work_item_id=work_item_id,
|
||||
)
|
||||
continue
|
||||
self._claimed_work_item_ids.add(work_item_id)
|
||||
if can_soft_wake and (
|
||||
bool((task.metadata or {}).get("review_task", False))
|
||||
or bool((task.metadata or {}).get("review_execution_work_item", False))
|
||||
@@ -1412,7 +1435,6 @@ class CompanyRuntime:
|
||||
self.prepare_task_for_session(session, task)
|
||||
await self._sync_current_turn_mode_to_work_item(task, session.current_turn_mode)
|
||||
if work_item is not None:
|
||||
await self._claim_role_session_work_item(session, work_item, task)
|
||||
# #7: mirror the claim onto the in-memory work_item so
|
||||
# subsequent iterations in the same claim pass see it
|
||||
# and is_dispatchable returns False (race safety after
|
||||
@@ -1908,21 +1930,65 @@ class CompanyRuntime:
|
||||
self.role_sessions[role_session_id] = role_session
|
||||
return role_session
|
||||
|
||||
async def _claim_role_session_work_item(self, session: CompanyMemberSession, work_item: Any, task: Task) -> None:
|
||||
async def _claim_role_session_work_item(
|
||||
self,
|
||||
session: CompanyMemberSession,
|
||||
work_item: Any,
|
||||
task: Task,
|
||||
) -> bool:
|
||||
"""Atomically claim ``work_item`` for the role-instance behind
|
||||
``session``.
|
||||
|
||||
In the role-instance model the claim identity is
|
||||
``role_runtime_session_id``. Seat / manager-seat columns are
|
||||
still written for org-chart lookups but they are NOT part of
|
||||
the claim key — only the role session is.
|
||||
the claim key — only the role session is. Pure in-memory runtimes have
|
||||
no durable race to arbitrate and keep the same local claim semantics.
|
||||
"""
|
||||
work_item_id = str(getattr(work_item, "work_item_id", "") or "").strip()
|
||||
if not work_item_id:
|
||||
return
|
||||
return False
|
||||
role_session = self._ensure_role_session(task)
|
||||
if role_session is None:
|
||||
return
|
||||
return False
|
||||
work_item_revision = 0
|
||||
try:
|
||||
work_item_revision = int((getattr(work_item, "metadata", {}) or {}).get("manager_mutation_revision") or 0)
|
||||
except (TypeError, ValueError):
|
||||
work_item_revision = 0
|
||||
store_ready = self.store is not None and bool(
|
||||
getattr(self.store, "is_ready", False)
|
||||
)
|
||||
claim = (
|
||||
getattr(self.store, "claim_delegation_work_item_if_dispatchable", None)
|
||||
if store_ready
|
||||
else None
|
||||
)
|
||||
if store_ready:
|
||||
# A durable runtime must win the store CAS before it mutates any
|
||||
# in-memory scheduling state. This is the Stop/shutdown race
|
||||
# boundary: a missing CAS API is a failed claim, not permission to
|
||||
# fall back to the former best-effort update path.
|
||||
if not callable(claim):
|
||||
return False
|
||||
persisted = await claim(
|
||||
work_item_id,
|
||||
expected_phase=getattr(work_item, "phase", Phase.READY),
|
||||
role_runtime_session_id=role_session.role_session_id,
|
||||
seat_id=str(getattr(session, "seat_id", "") or "").strip(),
|
||||
task_id=task.id,
|
||||
work_item_revision=work_item_revision,
|
||||
)
|
||||
if persisted is None:
|
||||
return False
|
||||
|
||||
work_item.phase = persisted.phase
|
||||
work_item.role_runtime_session_id = persisted.role_runtime_session_id
|
||||
work_item.claimed_by_role_runtime_session_id = (
|
||||
persisted.claimed_by_role_runtime_session_id
|
||||
)
|
||||
work_item.claimed_by_seat_id = persisted.claimed_by_seat_id
|
||||
work_item.metadata = dict(persisted.metadata or {})
|
||||
ready_background_ids = [
|
||||
item_id
|
||||
for item_id in list(role_session.background_work_item_ids or [])
|
||||
@@ -1937,39 +2003,11 @@ class CompanyRuntime:
|
||||
session.background_work_item_ids = list(role_session.background_work_item_ids)
|
||||
task.metadata = dict(task.metadata)
|
||||
task.metadata["delegation_role_session_id"] = role_session.role_session_id
|
||||
work_item_revision = 0
|
||||
try:
|
||||
work_item_revision = int((getattr(work_item, "metadata", {}) or {}).get("manager_mutation_revision") or 0)
|
||||
except (TypeError, ValueError):
|
||||
work_item_revision = 0
|
||||
task.metadata["started_work_item_revision"] = work_item_revision
|
||||
task.metadata["claimed_work_item_revision"] = work_item_revision
|
||||
if self.store and bool(getattr(self.store, "is_ready", False)) and hasattr(self.store, "update_delegation_work_item"):
|
||||
# Do not regress a work item that is already in a review
|
||||
# phase (AWAITING_MANAGER_REVIEW / AWAITING_HUMAN) back to
|
||||
# RUNNING: the DB phase validator rejects that transition
|
||||
# and the error bubbles up to the session loop. This
|
||||
# happens when the reactivation sweeper wakes a task whose
|
||||
# work item has already been promoted to review — treat
|
||||
# the claim as "refresh the task/role-session bindings
|
||||
# only" and leave the phase alone.
|
||||
current_phase = getattr(work_item, "phase", None)
|
||||
phase_to_write: Phase | None = Phase.RUNNING
|
||||
if current_phase in IN_REVIEW_PHASES:
|
||||
phase_to_write = None
|
||||
await self.store.update_delegation_work_item(
|
||||
work_item_id,
|
||||
phase=phase_to_write,
|
||||
role_runtime_session_id=role_session.role_session_id,
|
||||
claimed_by_role_runtime_session_id=role_session.role_session_id,
|
||||
metadata_updates={
|
||||
"claimed_by_role_session_id": role_session.role_session_id,
|
||||
"claimed_task_id": task.id,
|
||||
"claimed_work_item_revision": work_item_revision,
|
||||
},
|
||||
)
|
||||
if self.store and bool(getattr(self.store, "is_ready", False)) and hasattr(self.store, "save_delegation_role_session"):
|
||||
await self.store.save_delegation_role_session(role_session)
|
||||
return True
|
||||
|
||||
def ensure_role_instance_session(
|
||||
self, task: Task
|
||||
|
||||
@@ -0,0 +1,391 @@
|
||||
"""Canonical company-runtime identity derived from durable records.
|
||||
|
||||
Company-mode Tasks are execution envelopes, not the identity of a run. A
|
||||
runtime is owned by its root session and an active suspend checkpoint. This
|
||||
module deliberately has no UI dependencies so every surface can resolve the
|
||||
same scope without relying on process-local task maps.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Any, Iterable
|
||||
|
||||
from opc.layer2_organization.work_item_links import linked_work_item_id_for_task
|
||||
from opc.layer2_organization.work_item_runtime import is_work_item_runtime_metadata
|
||||
|
||||
|
||||
COMPANY_RUNTIME_CHECKPOINT_TYPES: frozenset[str] = frozenset({
|
||||
"company_runtime_suspended",
|
||||
"company_runtime_interrupted",
|
||||
})
|
||||
ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES: frozenset[str] = frozenset({
|
||||
"pending",
|
||||
"resuming",
|
||||
})
|
||||
|
||||
|
||||
def _text(value: Any) -> str:
|
||||
return str(value or "").strip()
|
||||
|
||||
|
||||
def _metadata(task: Any) -> dict[str, Any]:
|
||||
return dict(getattr(task, "metadata", {}) or {})
|
||||
|
||||
|
||||
def _task_id(task: Any) -> str:
|
||||
return _text(getattr(task, "id", ""))
|
||||
|
||||
|
||||
def _task_session_id(task: Any) -> str:
|
||||
return _text(getattr(task, "session_id", ""))
|
||||
|
||||
|
||||
def _task_parent_session_id(task: Any) -> str:
|
||||
metadata = _metadata(task)
|
||||
return _text(
|
||||
getattr(task, "parent_session_id", "")
|
||||
or metadata.get("company_runtime_root_session_id")
|
||||
or metadata.get("parent_session_id")
|
||||
)
|
||||
|
||||
|
||||
def _has_company_runtime_marker(task: Any) -> bool:
|
||||
metadata = _metadata(task)
|
||||
exec_mode = _text(metadata.get("exec_mode")).lower()
|
||||
mode = _text(metadata.get("mode")).lower()
|
||||
execution_mode = _text(metadata.get("execution_mode")).lower()
|
||||
if exec_mode in {"company", "org", "custom"} or mode in {"company", "org", "custom"}:
|
||||
return True
|
||||
if execution_mode in {"company", "company_mode", "multi_team_org"}:
|
||||
return True
|
||||
if is_work_item_runtime_metadata(metadata):
|
||||
return True
|
||||
if linked_work_item_id_for_task(task):
|
||||
return True
|
||||
if any(
|
||||
metadata.get(key) not in (None, "", [], {})
|
||||
for key in (
|
||||
"company_work_item_plan",
|
||||
"company_runtime_root_session_id",
|
||||
"delegation_run_id",
|
||||
"work_item_projection_id",
|
||||
"work_item_projection_ref",
|
||||
"work_item_role_id",
|
||||
"shared_role_session",
|
||||
)
|
||||
):
|
||||
return True
|
||||
# Old company records may predate exec_mode. An explicit task-mode marker
|
||||
# wins over the legacy profile hint.
|
||||
explicitly_task_mode = (
|
||||
exec_mode in {"task", "project", "single"}
|
||||
or mode == "task"
|
||||
or execution_mode in {"task", "task_mode", "project"}
|
||||
or _text(metadata.get("task_mode_contract")) == "single_full_capability_main_agent"
|
||||
)
|
||||
return not explicitly_task_mode and bool(_text(metadata.get("company_profile")))
|
||||
|
||||
|
||||
def is_company_runtime_task(task: Any) -> bool:
|
||||
"""Return whether durable Task metadata identifies company-owned work."""
|
||||
|
||||
return _has_company_runtime_marker(task)
|
||||
|
||||
|
||||
def is_pure_company_ui_anchor(task: Any, runtime_session_id: str) -> bool:
|
||||
"""Return whether *task* is the user-facing container for a runtime.
|
||||
|
||||
A shared final-decider Task can have the same ``session_id`` as the UI
|
||||
anchor. Work-item, role, or parent links therefore disqualify a Task even
|
||||
when its session id is an exact match.
|
||||
"""
|
||||
|
||||
session_id = _text(runtime_session_id)
|
||||
if not session_id or _task_session_id(task) != session_id:
|
||||
return False
|
||||
if _text(getattr(task, "parent_session_id", "")) or _text(getattr(task, "parent_id", "")):
|
||||
return False
|
||||
if linked_work_item_id_for_task(task):
|
||||
return False
|
||||
metadata = _metadata(task)
|
||||
return not any(
|
||||
metadata.get(key) not in (None, "", [], {}, False)
|
||||
for key in (
|
||||
"work_item_runtime",
|
||||
"work_item_projection_id",
|
||||
"work_item_projection_ref",
|
||||
"work_item_id",
|
||||
"work_item_role_id",
|
||||
"delegation_role_session_id",
|
||||
"shared_role_session",
|
||||
"shared_role_id",
|
||||
"company_runtime_root_session_id",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _created_sort_key(value: Any) -> tuple[float, str]:
|
||||
created_at = getattr(value, "created_at", None)
|
||||
if isinstance(created_at, datetime):
|
||||
timestamp = created_at.timestamp()
|
||||
elif hasattr(created_at, "timestamp"):
|
||||
try:
|
||||
timestamp = float(created_at.timestamp())
|
||||
except Exception:
|
||||
timestamp = 0.0
|
||||
else:
|
||||
timestamp = 0.0
|
||||
return timestamp, _task_id(value)
|
||||
|
||||
|
||||
def _checkpoint_sort_key(checkpoint: Any) -> tuple[float, float, str]:
|
||||
def _timestamp(value: Any) -> float:
|
||||
if isinstance(value, datetime):
|
||||
return value.timestamp()
|
||||
if hasattr(value, "timestamp"):
|
||||
try:
|
||||
return float(value.timestamp())
|
||||
except Exception:
|
||||
return 0.0
|
||||
return 0.0
|
||||
|
||||
return (
|
||||
_timestamp(getattr(checkpoint, "updated_at", None)),
|
||||
_timestamp(getattr(checkpoint, "created_at", None)),
|
||||
_text(getattr(checkpoint, "checkpoint_id", "")),
|
||||
)
|
||||
|
||||
|
||||
def _checkpoint_runtime_session_id(checkpoint: Any) -> str:
|
||||
payload = dict(getattr(checkpoint, "payload", {}) or {})
|
||||
return _text(
|
||||
getattr(checkpoint, "session_id", "")
|
||||
or payload.get("parent_session_id")
|
||||
or payload.get("session_id")
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CompanyRuntimeIdentity:
|
||||
"""Resolved identity for one company runtime scope."""
|
||||
|
||||
project_id: str
|
||||
runtime_session_id: str
|
||||
runtime_task_ids: tuple[str, ...]
|
||||
ui_anchor_task_id: str = ""
|
||||
config_source_task_id: str = ""
|
||||
pending_checkpoint_id: str = ""
|
||||
pending_checkpoint_type: str = ""
|
||||
pending_checkpoint_status: str = ""
|
||||
resumable: bool = False
|
||||
checkpoint: Any | None = field(default=None, repr=False, compare=False)
|
||||
|
||||
|
||||
class CompanyRuntimeIdentityIndex:
|
||||
"""Session-first index over preloaded Tasks and checkpoints."""
|
||||
|
||||
def __init__(self, tasks: Iterable[Any], checkpoints: Iterable[Any] = ()) -> None:
|
||||
self.tasks = tuple(tasks or ())
|
||||
self.checkpoints = tuple(checkpoints or ())
|
||||
self.tasks_by_id = {
|
||||
_task_id(task): task
|
||||
for task in self.tasks
|
||||
if _task_id(task)
|
||||
}
|
||||
self.checkpoints_by_id = {
|
||||
_text(getattr(checkpoint, "checkpoint_id", "")): checkpoint
|
||||
for checkpoint in self.checkpoints
|
||||
if _text(getattr(checkpoint, "checkpoint_id", ""))
|
||||
}
|
||||
self._identities_by_session = self._build_identities()
|
||||
self._runtime_session_by_task_id: dict[str, str] = {}
|
||||
runtime_sessions_by_task_session_id: dict[str, set[str]] = {}
|
||||
for runtime_session_id, identity in self._identities_by_session.items():
|
||||
for task_id in identity.runtime_task_ids:
|
||||
self._runtime_session_by_task_id[task_id] = runtime_session_id
|
||||
task_session_id = _task_session_id(self.tasks_by_id.get(task_id))
|
||||
if task_session_id:
|
||||
runtime_sessions_by_task_session_id.setdefault(
|
||||
task_session_id,
|
||||
set(),
|
||||
).add(runtime_session_id)
|
||||
self._runtime_session_by_task_session_id = {
|
||||
task_session_id: next(iter(runtime_session_ids))
|
||||
for task_session_id, runtime_session_ids in runtime_sessions_by_task_session_id.items()
|
||||
if len(runtime_session_ids) == 1
|
||||
}
|
||||
|
||||
@property
|
||||
def identities(self) -> tuple[CompanyRuntimeIdentity, ...]:
|
||||
return tuple(self._identities_by_session.values())
|
||||
|
||||
def task(self, task_id: str) -> Any | None:
|
||||
return self.tasks_by_id.get(_text(task_id))
|
||||
|
||||
def resolve(
|
||||
self,
|
||||
*,
|
||||
task_id: str = "",
|
||||
task_session_id: str = "",
|
||||
runtime_session_id: str = "",
|
||||
checkpoint_id: str = "",
|
||||
) -> CompanyRuntimeIdentity | None:
|
||||
requested_task_id = _text(task_id)
|
||||
requested_task_session_id = _text(task_session_id)
|
||||
requested_session_id = _text(runtime_session_id)
|
||||
requested_checkpoint_id = _text(checkpoint_id)
|
||||
|
||||
task_scope_id = self._runtime_session_by_task_id.get(requested_task_id, "")
|
||||
session_scope_id = self._runtime_session_by_task_session_id.get(
|
||||
requested_task_session_id,
|
||||
"",
|
||||
)
|
||||
checkpoint = self.checkpoints_by_id.get(requested_checkpoint_id) if requested_checkpoint_id else None
|
||||
checkpoint_session_id = _checkpoint_runtime_session_id(checkpoint) if checkpoint is not None else ""
|
||||
|
||||
candidates = {
|
||||
value
|
||||
for value in (
|
||||
requested_session_id,
|
||||
task_scope_id,
|
||||
session_scope_id,
|
||||
checkpoint_session_id,
|
||||
)
|
||||
if value
|
||||
}
|
||||
if requested_task_session_id and not session_scope_id:
|
||||
return None
|
||||
if len(candidates) != 1:
|
||||
return None
|
||||
resolved_session_id = next(iter(candidates))
|
||||
identity = self._identities_by_session.get(resolved_session_id)
|
||||
if identity is None:
|
||||
return None
|
||||
if requested_task_id and requested_task_id not in identity.runtime_task_ids:
|
||||
return None
|
||||
if requested_checkpoint_id and requested_checkpoint_id != identity.pending_checkpoint_id:
|
||||
return None
|
||||
return identity
|
||||
|
||||
def _build_identities(self) -> dict[str, CompanyRuntimeIdentity]:
|
||||
active_checkpoints_by_session: dict[str, list[Any]] = {}
|
||||
for checkpoint in self.checkpoints:
|
||||
checkpoint_type = _text(getattr(checkpoint, "checkpoint_type", ""))
|
||||
checkpoint_status = _text(getattr(checkpoint, "status", "")).lower()
|
||||
if (
|
||||
checkpoint_type not in COMPANY_RUNTIME_CHECKPOINT_TYPES
|
||||
or checkpoint_status not in ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES
|
||||
):
|
||||
continue
|
||||
runtime_session_id = _checkpoint_runtime_session_id(checkpoint)
|
||||
if runtime_session_id:
|
||||
active_checkpoints_by_session.setdefault(runtime_session_id, []).append(checkpoint)
|
||||
|
||||
known_sessions = set(active_checkpoints_by_session)
|
||||
for task in self.tasks:
|
||||
if not _has_company_runtime_marker(task):
|
||||
continue
|
||||
runtime_session_id = _task_parent_session_id(task) or _task_session_id(task)
|
||||
if runtime_session_id:
|
||||
known_sessions.add(runtime_session_id)
|
||||
|
||||
tasks_by_session: dict[str, list[Any]] = {session_id: [] for session_id in known_sessions}
|
||||
for task in self.tasks:
|
||||
task_id = _task_id(task)
|
||||
if not task_id:
|
||||
continue
|
||||
parent_session_id = _task_parent_session_id(task)
|
||||
own_session_id = _task_session_id(task)
|
||||
runtime_session_id = parent_session_id or own_session_id
|
||||
if runtime_session_id not in known_sessions:
|
||||
continue
|
||||
if not (
|
||||
_has_company_runtime_marker(task)
|
||||
or runtime_session_id in active_checkpoints_by_session
|
||||
or is_pure_company_ui_anchor(task, runtime_session_id)
|
||||
):
|
||||
continue
|
||||
tasks_by_session.setdefault(runtime_session_id, []).append(task)
|
||||
|
||||
identities: dict[str, CompanyRuntimeIdentity] = {}
|
||||
for runtime_session_id in sorted(known_sessions):
|
||||
group = sorted(tasks_by_session.get(runtime_session_id, []), key=_created_sort_key)
|
||||
anchor = next(
|
||||
(task for task in group if is_pure_company_ui_anchor(task, runtime_session_id)),
|
||||
None,
|
||||
)
|
||||
def _has_runtime_config(task: Any) -> bool:
|
||||
metadata = _metadata(task)
|
||||
return any(
|
||||
metadata.get(key) not in (None, "", [], {})
|
||||
for key in (
|
||||
"exec_mode",
|
||||
"mode",
|
||||
"company_profile",
|
||||
"org_id",
|
||||
"organization_id",
|
||||
"preferred_agent",
|
||||
"selected_execution_agent",
|
||||
)
|
||||
)
|
||||
|
||||
config_source = (
|
||||
anchor if anchor is not None and _has_runtime_config(anchor) else None
|
||||
) or next(
|
||||
(
|
||||
task for task in group
|
||||
if _has_runtime_config(task)
|
||||
),
|
||||
anchor or (group[0] if group else None),
|
||||
)
|
||||
checkpoint_candidates = active_checkpoints_by_session.get(runtime_session_id, [])
|
||||
checkpoint = max(checkpoint_candidates, key=_checkpoint_sort_key) if checkpoint_candidates else None
|
||||
checkpoint_status = _text(getattr(checkpoint, "status", "")).lower() if checkpoint is not None else ""
|
||||
project_id = _text(
|
||||
getattr(checkpoint, "project_id", "") if checkpoint is not None else ""
|
||||
) or _text(getattr(config_source, "project_id", "") if config_source is not None else "") or "default"
|
||||
identities[runtime_session_id] = CompanyRuntimeIdentity(
|
||||
project_id=project_id,
|
||||
runtime_session_id=runtime_session_id,
|
||||
runtime_task_ids=tuple(_task_id(task) for task in group if _task_id(task)),
|
||||
ui_anchor_task_id=_task_id(anchor) if anchor is not None else "",
|
||||
config_source_task_id=_task_id(config_source) if config_source is not None else "",
|
||||
pending_checkpoint_id=_text(getattr(checkpoint, "checkpoint_id", "")) if checkpoint is not None else "",
|
||||
pending_checkpoint_type=_text(getattr(checkpoint, "checkpoint_type", "")) if checkpoint is not None else "",
|
||||
pending_checkpoint_status=checkpoint_status,
|
||||
resumable=checkpoint_status == "pending",
|
||||
checkpoint=checkpoint,
|
||||
)
|
||||
return identities
|
||||
|
||||
|
||||
def build_company_runtime_identity_index(
|
||||
tasks: Iterable[Any],
|
||||
checkpoints: Iterable[Any] = (),
|
||||
) -> CompanyRuntimeIdentityIndex:
|
||||
return CompanyRuntimeIdentityIndex(tasks, checkpoints)
|
||||
|
||||
|
||||
async def load_company_runtime_identity_index(
|
||||
store: Any,
|
||||
project_id: str,
|
||||
) -> CompanyRuntimeIdentityIndex:
|
||||
"""Load durable records once and build the canonical runtime index."""
|
||||
|
||||
tasks = await store.get_tasks(project_id=project_id)
|
||||
checkpoint_getter = getattr(store, "get_execution_checkpoints", None)
|
||||
if callable(checkpoint_getter):
|
||||
checkpoints = await checkpoint_getter(
|
||||
project_id=project_id,
|
||||
checkpoint_types=sorted(COMPANY_RUNTIME_CHECKPOINT_TYPES),
|
||||
statuses=sorted(ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES),
|
||||
)
|
||||
else:
|
||||
checkpoint_getter = getattr(store, "get_pending_checkpoints", None)
|
||||
checkpoints = await checkpoint_getter(
|
||||
project_id=project_id,
|
||||
checkpoint_types=sorted(COMPANY_RUNTIME_CHECKPOINT_TYPES),
|
||||
) if callable(checkpoint_getter) else []
|
||||
return build_company_runtime_identity_index(tasks, checkpoints)
|
||||
@@ -75,6 +75,8 @@ class CustomRuntimeRunner:
|
||||
store=shared_store,
|
||||
owns_store=shared_store is None,
|
||||
run_startup_reconcile=shared_store is None,
|
||||
active_task_run_registry=getattr(self.parent, "_active_task_run_registry", None),
|
||||
owns_active_task_run_registry=False,
|
||||
on_progress=self.parent.on_progress,
|
||||
on_runtime_event=self.parent.on_runtime_event,
|
||||
on_escalation=self.parent.on_escalation,
|
||||
|
||||
@@ -144,8 +144,10 @@ class EngineSeatExecutor:
|
||||
member_session: CompanyMemberSession | None = None,
|
||||
) -> None:
|
||||
_ = member_session
|
||||
if hasattr(self.host, "_active_task_runs"):
|
||||
self.host._active_task_runs.discard(task.id)
|
||||
# The task coroutine owns its registry attempt token and removes it in
|
||||
# ``finally``. Interrupt requests must not make a still-running
|
||||
# coroutine appear inactive.
|
||||
return None
|
||||
|
||||
async def shutdown(
|
||||
self,
|
||||
|
||||
@@ -1,297 +0,0 @@
|
||||
"""CLI-board-specific company runtime recovery manager.
|
||||
|
||||
Independent from office_ui/recovery_manager.py — same Core Engine APIs,
|
||||
different notification path (TUI event bridge instead of WebSocket broadcast).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from opc.layer2_organization.work_item_identity import work_item_projection_id_from_metadata
|
||||
from opc.layer2_organization.work_item_transition import apply_task_status_transition
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .engine_facade import EngineFacade
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecoverableWorkItem:
|
||||
projection_id: str
|
||||
title: str
|
||||
task_id: str
|
||||
status: str
|
||||
interrupted: bool
|
||||
previous_status: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class InterruptedCompanyRuntime:
|
||||
parent_session_id: str
|
||||
parent_task_id: str
|
||||
project_id: str
|
||||
title: str
|
||||
profile: str
|
||||
interrupted_at: str
|
||||
work_items: list[RecoverableWorkItem] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecoveryStatus:
|
||||
interrupted: list[InterruptedCompanyRuntime] = field(default_factory=list)
|
||||
active_recoveries: list[str] = field(default_factory=list)
|
||||
scanned_at: float = 0.0
|
||||
|
||||
|
||||
def _is_interrupted(task: Any) -> bool:
|
||||
from opc.core.models import TaskStatus
|
||||
if task.status != TaskStatus.FAILED:
|
||||
return False
|
||||
meta = getattr(task, "metadata", {}) or {}
|
||||
if meta.get("interrupted_recovery"):
|
||||
return True
|
||||
result = getattr(task, "result", {}) or {}
|
||||
artifacts = result.get("artifacts", {}) or {}
|
||||
return bool(artifacts.get("interrupted"))
|
||||
|
||||
|
||||
class CliRecoveryManager:
|
||||
"""Scan for interrupted company runtimes and provide resume/cancel."""
|
||||
|
||||
_CACHE_TTL = 10.0
|
||||
|
||||
def __init__(self, facade: EngineFacade) -> None:
|
||||
self._facade = facade
|
||||
self._lock = asyncio.Lock()
|
||||
self._active: dict[str, asyncio.Task[Any]] = {}
|
||||
self._cached: RecoveryStatus | None = None
|
||||
self._cache_until: float = 0.0
|
||||
|
||||
@property
|
||||
def _project_id(self) -> str:
|
||||
return self._facade.project_id or "default"
|
||||
|
||||
async def get_status(self) -> RecoveryStatus:
|
||||
now = time.time()
|
||||
if self._cached is not None and now < self._cache_until:
|
||||
self._cached.active_recoveries = list(self._active.keys())
|
||||
return self._cached
|
||||
status = await self.scan()
|
||||
self._cached = status
|
||||
self._cache_until = now + self._CACHE_TTL
|
||||
return status
|
||||
|
||||
async def scan(self) -> RecoveryStatus:
|
||||
engine = await self._facade.ensure_ready()
|
||||
if not engine.store:
|
||||
return RecoveryStatus()
|
||||
|
||||
try:
|
||||
all_tasks = await engine.store.get_tasks(project_id=self._project_id)
|
||||
except Exception as exc:
|
||||
logger.warning("Recovery scan failed: %s", exc)
|
||||
return RecoveryStatus()
|
||||
|
||||
groups: dict[str, list[Any]] = {}
|
||||
tasks_by_session: dict[str, Any] = {}
|
||||
for task in all_tasks:
|
||||
sid = str(getattr(task, "session_id", "") or "").strip()
|
||||
if sid:
|
||||
tasks_by_session[sid] = task
|
||||
parent_sid = str(getattr(task, "parent_session_id", "") or "").strip()
|
||||
projection_id = work_item_projection_id_from_metadata(getattr(task, "metadata", {}) or {})
|
||||
if parent_sid and projection_id:
|
||||
groups.setdefault(parent_sid, []).append(task)
|
||||
|
||||
from opc.core.models import TaskStatus
|
||||
interrupted: list[InterruptedCompanyRuntime] = []
|
||||
|
||||
for parent_sid, tasks in groups.items():
|
||||
if not any(_is_interrupted(t) for t in tasks):
|
||||
continue
|
||||
non_terminal = [t for t in tasks if t.status not in (TaskStatus.DONE, TaskStatus.CANCELLED)]
|
||||
if not non_terminal:
|
||||
continue
|
||||
|
||||
parent_task = tasks_by_session.get(parent_sid)
|
||||
parent_task_id = parent_task.id if parent_task else parent_sid
|
||||
title = parent_task.title if parent_task else "Unknown company runtime"
|
||||
|
||||
work_items: list[RecoverableWorkItem] = []
|
||||
earliest = ""
|
||||
for t in sorted(tasks, key=lambda x: (x.created_at, x.id)):
|
||||
meta = dict(getattr(t, "metadata", {}) or {})
|
||||
rmeta = meta.get("interrupted_recovery", {})
|
||||
is_int = _is_interrupted(t)
|
||||
if is_int and rmeta.get("detected_at", ""):
|
||||
det = rmeta["detected_at"]
|
||||
if not earliest or det < earliest:
|
||||
earliest = det
|
||||
work_items.append(RecoverableWorkItem(
|
||||
projection_id=work_item_projection_id_from_metadata(meta, fallback=t.id),
|
||||
title=t.title,
|
||||
task_id=t.id,
|
||||
status=t.status.value if hasattr(t.status, "value") else str(t.status),
|
||||
interrupted=is_int,
|
||||
previous_status=rmeta.get("previous_status", ""),
|
||||
))
|
||||
|
||||
profile = ""
|
||||
for t in tasks:
|
||||
p = (getattr(t, "metadata", {}) or {}).get("company_profile", "")
|
||||
if p:
|
||||
profile = p
|
||||
break
|
||||
|
||||
interrupted.append(InterruptedCompanyRuntime(
|
||||
parent_session_id=parent_sid,
|
||||
parent_task_id=parent_task_id,
|
||||
project_id=self._project_id,
|
||||
title=title,
|
||||
profile=profile,
|
||||
interrupted_at=earliest or datetime.now().isoformat(),
|
||||
work_items=work_items,
|
||||
))
|
||||
|
||||
return RecoveryStatus(
|
||||
interrupted=interrupted,
|
||||
active_recoveries=list(self._active.keys()),
|
||||
scanned_at=time.time(),
|
||||
)
|
||||
|
||||
async def resume(self, parent_task_id: str) -> dict[str, Any]:
|
||||
async with self._lock:
|
||||
if parent_task_id in self._active:
|
||||
return {"ok": False, "error": "already_in_progress"}
|
||||
|
||||
status = await self.scan()
|
||||
wf = next((w for w in status.interrupted if w.parent_task_id == parent_task_id), None)
|
||||
if not wf:
|
||||
return {"ok": False, "error": "not_found"}
|
||||
|
||||
engine = await self._facade.ensure_ready()
|
||||
snapshot = await engine._load_company_runtime_snapshot(wf.parent_session_id)
|
||||
if not snapshot:
|
||||
return {"ok": False, "error": "snapshot_unavailable"}
|
||||
|
||||
plan, tasks = snapshot
|
||||
|
||||
await self._clean_checkpoints(wf, tasks)
|
||||
|
||||
from opc.core.models import TaskStatus
|
||||
resumed_ids: list[str] = []
|
||||
for task in tasks:
|
||||
if task.status == TaskStatus.DONE:
|
||||
continue
|
||||
if task.status in (TaskStatus.FAILED, TaskStatus.BLOCKED):
|
||||
task.result = None
|
||||
task.execution_lock = False
|
||||
task.execution_locked_at = None
|
||||
meta = dict(task.metadata)
|
||||
meta.pop("interrupted_recovery", None)
|
||||
progress = list(meta.get("progress_log", []))
|
||||
progress.append(f"[Recovery] Resumed at {datetime.now().isoformat()}")
|
||||
meta["progress_log"] = progress[-20:]
|
||||
task.metadata = meta
|
||||
try:
|
||||
await apply_task_status_transition(
|
||||
engine.store,
|
||||
task,
|
||||
target_status_or_phase=TaskStatus.PENDING,
|
||||
reason="cli_recovery_resume",
|
||||
release_claim=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Recovery resume skipped %s: %s", task.id, exc)
|
||||
continue
|
||||
if task.status != TaskStatus.PENDING:
|
||||
logger.warning("Recovery resume preserved non-runnable phase for %s", task.id)
|
||||
continue
|
||||
await engine.store.save_task(task)
|
||||
resumed_ids.append(work_item_projection_id_from_metadata(meta, fallback=task.id))
|
||||
|
||||
if not resumed_ids:
|
||||
return {"ok": False, "error": "no_work_items_to_resume"}
|
||||
|
||||
self._cache_until = 0.0
|
||||
bg = asyncio.create_task(self._execute(parent_task_id, plan, tasks))
|
||||
self._active[parent_task_id] = bg
|
||||
return {"ok": True, "resumed_work_item_projection_ids": resumed_ids}
|
||||
|
||||
async def cancel(self, parent_task_id: str) -> dict[str, Any]:
|
||||
async with self._lock:
|
||||
bg = self._active.pop(parent_task_id, None)
|
||||
if bg and not bg.done():
|
||||
bg.cancel()
|
||||
|
||||
status = await self.scan()
|
||||
wf = next((w for w in status.interrupted if w.parent_task_id == parent_task_id), None)
|
||||
if not wf:
|
||||
return {"ok": False, "error": "not_found"}
|
||||
|
||||
engine = await self._facade.ensure_ready()
|
||||
snapshot = await engine._load_company_runtime_snapshot(wf.parent_session_id)
|
||||
if not snapshot:
|
||||
return {"ok": False, "error": "snapshot_unavailable"}
|
||||
|
||||
_, tasks = snapshot
|
||||
from opc.core.models import TaskStatus
|
||||
cancelled = 0
|
||||
for task in tasks:
|
||||
if task.status not in (TaskStatus.DONE, TaskStatus.CANCELLED):
|
||||
try:
|
||||
await apply_task_status_transition(
|
||||
engine.store,
|
||||
task,
|
||||
target_status_or_phase=TaskStatus.CANCELLED,
|
||||
reason="cli_recovery_cancel",
|
||||
release_claim=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Recovery cancel skipped %s: %s", task.id, exc)
|
||||
continue
|
||||
if task.status != TaskStatus.CANCELLED:
|
||||
logger.warning("Recovery cancel preserved non-cancelled phase for %s", task.id)
|
||||
continue
|
||||
cancelled += 1
|
||||
|
||||
await self._clean_checkpoints(wf, tasks)
|
||||
self._cache_until = 0.0
|
||||
return {"ok": True, "cancelled_count": cancelled}
|
||||
|
||||
async def _execute(self, parent_task_id: str, plan: Any, tasks: list[Any]) -> None:
|
||||
try:
|
||||
engine = await self._facade.ensure_ready()
|
||||
executor = engine.company_executor
|
||||
if not executor:
|
||||
raise RuntimeError("company_executor not available")
|
||||
await executor.execute(plan, tasks)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception as exc:
|
||||
logger.warning("Recovery execution failed for %s: %s", parent_task_id, exc)
|
||||
finally:
|
||||
self._active.pop(parent_task_id, None)
|
||||
self._cache_until = 0.0
|
||||
|
||||
async def _clean_checkpoints(self, wf: InterruptedCompanyRuntime, tasks: list[Any]) -> None:
|
||||
engine = await self._facade.ensure_ready()
|
||||
if not engine.store:
|
||||
return
|
||||
session_ids = {str(getattr(t, "session_id", "") or "").strip() for t in tasks}
|
||||
session_ids.add(wf.parent_session_id)
|
||||
session_ids.discard("")
|
||||
try:
|
||||
pending = await engine.store.get_pending_checkpoints(project_id=wf.project_id)
|
||||
for cp in pending:
|
||||
if str(cp.session_id or "").strip() in session_ids:
|
||||
await engine.store.resolve_execution_checkpoint(cp.checkpoint_id, status="cancelled")
|
||||
except Exception as exc:
|
||||
logger.debug("Checkpoint cleanup error: %s", exc)
|
||||
@@ -15,7 +15,6 @@ from opc.plugins.cli_board.state.store import BoardStateStore
|
||||
from opc.plugins.cli_board.tui.screens.help import HelpScreen
|
||||
from opc.plugins.cli_board.tui.screens.palette import CommandPaletteScreen, PaletteCommand
|
||||
from opc.plugins.cli_board.tui.screens.prompt import PromptField, PromptScreen
|
||||
from opc.plugins.cli_board.tui.screens.recovery import RecoveryAction, RecoveryScreen
|
||||
from opc.plugins.cli_board.widgets.activity_pane import ActivityPaneWidget
|
||||
from opc.plugins.cli_board.widgets.context_tabs import ContextTabsWidget
|
||||
from opc.plugins.cli_board.widgets.detail_pane import DetailPaneWidget
|
||||
@@ -36,7 +35,6 @@ if TYPE_CHECKING:
|
||||
from opc.plugins.cli_board.services.engine_facade import EngineFacade
|
||||
from opc.plugins.cli_board.services.event_bridge import CliBoardEventBridge
|
||||
from opc.plugins.cli_board.services.reconcile import ReconcileLoop
|
||||
from opc.plugins.cli_board.services.recovery import CliRecoveryManager
|
||||
|
||||
|
||||
class CliBoardApp(App[None]):
|
||||
@@ -74,7 +72,6 @@ class CliBoardApp(App[None]):
|
||||
Binding("x", "cancel_task", "Cancel"),
|
||||
Binding("t", "retry_selected", "Retry"),
|
||||
Binding("e", "checkpoint_feedback", "Feedback"),
|
||||
Binding("w", "recovery_scan", "Recovery"),
|
||||
Binding("R", "rename_session", "Rename", show=False),
|
||||
Binding("D", "delete_session", "Delete", show=False),
|
||||
Binding("E", "switch_mode", "Mode", show=False),
|
||||
@@ -115,7 +112,6 @@ class CliBoardApp(App[None]):
|
||||
self.repository: BoardRepository | None = None
|
||||
self.actions: BoardActions | None = None
|
||||
self.event_bridge: CliBoardEventBridge | None = None
|
||||
self.recovery_manager: CliRecoveryManager | None = None
|
||||
self.reconcile_loop: ReconcileLoop | None = None
|
||||
self.exec_mode = "task"
|
||||
self.company_profile = "corporate"
|
||||
@@ -143,13 +139,10 @@ class CliBoardApp(App[None]):
|
||||
from opc.plugins.cli_board.services.engine_facade import EngineFacade
|
||||
from opc.plugins.cli_board.services.event_bridge import CliBoardEventBridge
|
||||
|
||||
from opc.plugins.cli_board.services.recovery import CliRecoveryManager
|
||||
|
||||
self.facade = EngineFacade(project_id=self.project_id)
|
||||
self.repository = BoardRepository(self.facade, project_id=self.project_id)
|
||||
self.actions = BoardActions(self.facade, project_id=self.project_id)
|
||||
self.event_bridge = CliBoardEventBridge(self._handle_board_event)
|
||||
self.recovery_manager = CliRecoveryManager(self.facade)
|
||||
|
||||
def compose(self) -> ComposeResult:
|
||||
yield Header(show_clock=True)
|
||||
@@ -650,36 +643,6 @@ class CliBoardApp(App[None]):
|
||||
success_message=f"{label} checkpoint for {task.title}{suffix}.",
|
||||
)
|
||||
|
||||
def action_recovery_scan(self) -> None:
|
||||
if self._readonly_guard():
|
||||
return
|
||||
self._action_recovery_scan()
|
||||
|
||||
@work(group="modal", exclusive=True)
|
||||
async def _action_recovery_scan(self) -> None:
|
||||
if self.recovery_manager is None:
|
||||
self.status_widget.set_message("Recovery unavailable.")
|
||||
return
|
||||
status = await self.recovery_manager.get_status()
|
||||
result = await self.push_screen_wait(RecoveryScreen(status))
|
||||
if result is None:
|
||||
return
|
||||
if result.action == "resume":
|
||||
self.status_widget.set_message(f"Resuming {result.parent_task_id}...")
|
||||
outcome = await self.recovery_manager.resume(result.parent_task_id)
|
||||
if outcome.get("ok"):
|
||||
ids = outcome.get("resumed_work_item_projection_ids", [])
|
||||
self.status_widget.set_message(f"Resumed {len(ids)} work item(s).")
|
||||
else:
|
||||
self.status_widget.set_message(f"Resume failed: {outcome.get('error', '?')}.")
|
||||
elif result.action == "cancel":
|
||||
outcome = await self.recovery_manager.cancel(result.parent_task_id)
|
||||
if outcome.get("ok"):
|
||||
self.status_widget.set_message(f"Cancelled {outcome.get('cancelled_count', 0)} task(s).")
|
||||
else:
|
||||
self.status_widget.set_message(f"Cancel failed: {outcome.get('error', '?')}.")
|
||||
await self._refresh_snapshot(reason="recovery", silent=True)
|
||||
|
||||
def action_rename_session(self) -> None:
|
||||
if self._readonly_guard():
|
||||
return
|
||||
@@ -1354,7 +1317,6 @@ class CliBoardApp(App[None]):
|
||||
PaletteCommand("view_focus", "Switch to Focus", "Zoom into the selected task.", "3"),
|
||||
PaletteCommand("view_pipeline", "Switch to Projection", "Show the read-only work-item projection for the selected company run.", "4"),
|
||||
PaletteCommand("view_org", "Switch to Organisation", "Show read-only org structure.", "5"),
|
||||
PaletteCommand("recovery_scan", "Runtime Recovery", "Scan and resume interrupted company runtimes.", "w"),
|
||||
PaletteCommand("rename_session", "Rename Session", "Change the title of the selected task.", "R"),
|
||||
PaletteCommand("delete_session", "Delete Session", "Cancel and remove the selected task.", "D"),
|
||||
PaletteCommand("purge_cancelled", "Purge Cancelled Tasks", "Permanently delete all cancelled/failed tasks.", ""),
|
||||
@@ -1382,7 +1344,6 @@ class CliBoardApp(App[None]):
|
||||
"project_delete",
|
||||
"session_config",
|
||||
"org_add_role",
|
||||
"recovery_scan",
|
||||
"rename_session",
|
||||
"delete_session",
|
||||
"purge_cancelled",
|
||||
|
||||
@@ -49,7 +49,7 @@ class HelpScreen(ModalScreen[None]):
|
||||
" s: reply in session m: move between columns\n"
|
||||
" a / d: approve / deny checkpoint\n"
|
||||
" e: checkpoint feedback (approve/deny with message)\n"
|
||||
" c: done x: cancel t: retry w: runtime recovery\n"
|
||||
" c: done x: cancel t: retry\n"
|
||||
"\n"
|
||||
"Session Management\n"
|
||||
" R: rename session D: delete session\n"
|
||||
|
||||
@@ -1,126 +0,0 @@
|
||||
"""Recovery modal screen for the CLI board."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from textual.app import ComposeResult
|
||||
from textual.containers import Horizontal, Vertical, VerticalScroll
|
||||
from textual.screen import ModalScreen
|
||||
from textual.widgets import Button, Label, Static
|
||||
|
||||
from opc.plugins.cli_board.services.recovery import RecoveryStatus
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecoveryAction:
|
||||
action: str # "resume" | "cancel" | "dismiss"
|
||||
parent_task_id: str = ""
|
||||
|
||||
|
||||
class RecoveryScreen(ModalScreen[RecoveryAction | None]):
|
||||
"""Show interrupted company runtimes with resume/cancel options."""
|
||||
|
||||
DEFAULT_CSS = """
|
||||
RecoveryScreen {
|
||||
align: center middle;
|
||||
}
|
||||
|
||||
.recovery-dialog {
|
||||
width: 88;
|
||||
max-width: 90%;
|
||||
height: auto;
|
||||
max-height: 80%;
|
||||
border: solid $primary;
|
||||
background: $surface;
|
||||
padding: 1 2;
|
||||
}
|
||||
|
||||
.recovery-title {
|
||||
text-style: bold;
|
||||
margin-bottom: 1;
|
||||
}
|
||||
|
||||
.recovery-runtime {
|
||||
margin-bottom: 1;
|
||||
padding: 1;
|
||||
border: round $secondary;
|
||||
}
|
||||
|
||||
.recovery-actions {
|
||||
align-horizontal: right;
|
||||
height: auto;
|
||||
margin-top: 1;
|
||||
}
|
||||
|
||||
.recovery-empty {
|
||||
color: $text-muted;
|
||||
margin: 1;
|
||||
}
|
||||
"""
|
||||
|
||||
BINDINGS = [("escape", "dismiss_screen", "Close")]
|
||||
|
||||
def __init__(self, status: RecoveryStatus) -> None:
|
||||
super().__init__()
|
||||
self.status = status
|
||||
|
||||
def compose(self) -> ComposeResult:
|
||||
with Vertical(classes="recovery-dialog"):
|
||||
yield Static("Interrupted Runtimes", classes="recovery-title")
|
||||
|
||||
if not self.status.interrupted:
|
||||
yield Static("No interrupted runtimes found.", classes="recovery-empty")
|
||||
else:
|
||||
with VerticalScroll():
|
||||
for wf in self.status.interrupted:
|
||||
with Vertical(classes="recovery-runtime"):
|
||||
# Runtime header
|
||||
active = wf.parent_task_id in set(self.status.active_recoveries)
|
||||
status_label = " (recovering...)" if active else ""
|
||||
yield Label(f"{wf.title}{status_label}")
|
||||
yield Static(
|
||||
f" Profile: {wf.profile or 'unknown'} "
|
||||
f"Interrupted: {wf.interrupted_at[:19] if wf.interrupted_at else '?'}"
|
||||
)
|
||||
|
||||
# Work-item summary
|
||||
done = sum(1 for s in wf.work_items if s.status == "done")
|
||||
total = len(wf.work_items)
|
||||
failed = sum(1 for s in wf.work_items if s.interrupted)
|
||||
yield Static(
|
||||
f" Work items: {done}/{total} done, {failed} interrupted"
|
||||
)
|
||||
|
||||
if not active:
|
||||
with Horizontal():
|
||||
yield Button(
|
||||
"Resume",
|
||||
id=f"resume-{wf.parent_task_id}",
|
||||
variant="primary",
|
||||
)
|
||||
yield Button(
|
||||
"Cancel",
|
||||
id=f"cancel-{wf.parent_task_id}",
|
||||
variant="error",
|
||||
)
|
||||
|
||||
with Horizontal(classes="recovery-actions"):
|
||||
yield Button("Close", id="close-recovery")
|
||||
|
||||
def on_button_pressed(self, event: Button.Pressed) -> None:
|
||||
btn_id = event.button.id or ""
|
||||
if btn_id == "close-recovery":
|
||||
self.dismiss(None)
|
||||
return
|
||||
if btn_id.startswith("resume-"):
|
||||
task_id = btn_id[len("resume-"):]
|
||||
self.dismiss(RecoveryAction(action="resume", parent_task_id=task_id))
|
||||
return
|
||||
if btn_id.startswith("cancel-"):
|
||||
task_id = btn_id[len("cancel-"):]
|
||||
self.dismiss(RecoveryAction(action="cancel", parent_task_id=task_id))
|
||||
return
|
||||
|
||||
def action_dismiss_screen(self) -> None:
|
||||
self.dismiss(None)
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
+1
-1
File diff suppressed because one or more lines are too long
@@ -5,9 +5,9 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<link rel="icon" href="data:," />
|
||||
<title>OpenOPC Pixel Office</title>
|
||||
<script type="module" crossorigin src="./assets/index-02tfsorH.js"></script>
|
||||
<script type="module" crossorigin src="./assets/index-Cson66Y4.js"></script>
|
||||
<link rel="modulepreload" crossorigin href="./assets/phaser-DFK5Ua9d.js">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-BCWwLlJm.css">
|
||||
<link rel="stylesheet" crossorigin href="./assets/index-DEvLDWDw.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="root"></div>
|
||||
|
||||
@@ -476,7 +476,6 @@ export default function App() {
|
||||
const [globalCompanyProfile, setGlobalCompanyProfile] = useState<'corporate' | 'custom'>('corporate')
|
||||
const [globalTaskPreferredAgent, setGlobalTaskPreferredAgent] = useState<TaskPreferredAgent>('native')
|
||||
const [orgInfoData, setOrgInfoData] = useState<OrgInfoPayload | null>(null)
|
||||
const [recoveryStatus, setRecoveryStatus] = useState<any>(null)
|
||||
const [commsState, setCommsState] = useState<import('./lib/wsClient').CommsStatePayload | null>(null)
|
||||
const [commsMessage, setCommsMessage] = useState<import('./lib/wsClient').CommsMessagePayload | null>(null)
|
||||
const [talentTemplates, setTalentTemplates] = useState<TalentTemplate[]>([])
|
||||
@@ -1239,7 +1238,6 @@ export default function App() {
|
||||
runtimeControlState: String(payload.runtime_control_state ?? payload.runtimeControlState ?? 'idle') as any,
|
||||
canStop: Boolean(payload.can_stop ?? payload.canStop),
|
||||
canResume: Boolean(payload.can_resume ?? payload.canResume),
|
||||
resumeParentTaskId: String(payload.resume_parent_task_id ?? payload.resumeParentTaskId ?? ''),
|
||||
resumeParentSessionId: String(payload.resume_parent_session_id ?? payload.resumeParentSessionId ?? ''),
|
||||
pendingRuntimeCheckpointId: String(payload.pending_runtime_checkpoint_id ?? payload.pendingRuntimeCheckpointId ?? ''),
|
||||
stopIntentId: String(payload.stop_intent_id ?? payload.stopIntentId ?? ''),
|
||||
@@ -1749,10 +1747,6 @@ export default function App() {
|
||||
clientRef.current?.collabSync(getActiveProjectId(), undefined, projectViewGenerationRef.current)
|
||||
}
|
||||
},
|
||||
onRecoveryStatus: (payload) => {
|
||||
if (!payloadMatchesActiveProject(payload as unknown as Record<string, unknown>, false)) return
|
||||
setRecoveryStatus(payload)
|
||||
},
|
||||
onCommsState: (payload) => {
|
||||
if (!payloadMatchesActiveProject(payload as unknown as Record<string, unknown>, false)) return
|
||||
setCommsState(payload)
|
||||
@@ -1761,13 +1755,6 @@ export default function App() {
|
||||
if (!payloadMatchesActiveProject(payload as unknown as Record<string, unknown>, true)) return
|
||||
setCommsMessage(payload)
|
||||
},
|
||||
onRecoveryResult: (payload) => {
|
||||
if (!payloadMatchesActiveProject(payload as unknown as Record<string, unknown>, false)) return
|
||||
if (payload?.status === 'completed' || payload?.status === 'cancelled') {
|
||||
// Trigger a re-scan
|
||||
clientRef.current?.recoveryAction(getActiveProjectId(), 'scan')
|
||||
}
|
||||
},
|
||||
onTalentList: (payload) => {
|
||||
setTalentTemplates(payload.templates ?? [])
|
||||
if (payload.talent_dir) setDefaultTalentDir(payload.talent_dir)
|
||||
@@ -2242,18 +2229,13 @@ export default function App() {
|
||||
) => {
|
||||
const session = sessionStore.sessions.find(s => s.taskId === taskId)
|
||||
const parentSessionId = session?.resumeParentSessionId ?? session?.parentSessionId ?? session?.sessionId
|
||||
const parentTaskId = session?.resumeParentTaskId
|
||||
?? (parentSessionId ? sessionStore.sessions.find(s => s.sessionId === parentSessionId && !s.parentSessionId)?.taskId : undefined)
|
||||
?? taskId
|
||||
for (const candidate of sessionStore.sessions) {
|
||||
if (
|
||||
candidate.taskId === taskId
|
||||
|| candidate.taskId === parentTaskId
|
||||
|| (!!parentSessionId && (candidate.parentSessionId === parentSessionId || candidate.sessionId === parentSessionId))
|
||||
) {
|
||||
sessionStore.updateSession(candidate.taskId, {
|
||||
...patch,
|
||||
resumeParentTaskId: parentTaskId,
|
||||
resumeParentSessionId: parentSessionId,
|
||||
})
|
||||
}
|
||||
@@ -2278,7 +2260,7 @@ export default function App() {
|
||||
clientRef.current?.sessionStop(getActiveProjectId(), taskId)
|
||||
}, [sessionStore.sessions, markRuntimeControlForTask, getActiveProjectId])
|
||||
|
||||
const handleSessionResume = useCallback((taskId: string) => {
|
||||
const handleSessionResume = useCallback((taskId: string, runtimeSessionId?: string, checkpointId?: string) => {
|
||||
const session = sessionStore.sessions.find(s => s.taskId === taskId)
|
||||
const isCompanyRuntime = session?.execMode === 'company'
|
||||
|| session?.execMode === 'org'
|
||||
@@ -2293,7 +2275,12 @@ export default function App() {
|
||||
canResume: false,
|
||||
})
|
||||
}
|
||||
clientRef.current?.sessionResume(getActiveProjectId(), taskId)
|
||||
clientRef.current?.sessionResume(
|
||||
getActiveProjectId(),
|
||||
taskId,
|
||||
runtimeSessionId ?? session?.resumeParentSessionId ?? session?.parentSessionId ?? session?.sessionId,
|
||||
checkpointId ?? session?.pendingRuntimeCheckpointId,
|
||||
)
|
||||
}, [sessionStore.sessions, markRuntimeControlForTask, getActiveProjectId])
|
||||
|
||||
const handleGlobalModeChange = useCallback((mode: 'task' | 'company' | 'org' | 'custom', profile?: string, orgId?: string) => {
|
||||
@@ -2426,9 +2413,6 @@ export default function App() {
|
||||
activeSavedOrg={activeSavedOrg}
|
||||
onSavedOrgsList={handleSavedOrgsList}
|
||||
onSavedOrgLoad={handleSavedOrgLoad}
|
||||
recoveryStatus={recoveryStatus}
|
||||
onRecoveryResume={(id) => clientRef.current?.recoveryAction(getActiveProjectId(), 'resume', id)}
|
||||
onRecoveryCancel={(id) => clientRef.current?.recoveryAction(getActiveProjectId(), 'cancel', id)}
|
||||
commsState={commsState}
|
||||
commsMessage={commsMessage}
|
||||
onCommsRefresh={(opts) => {
|
||||
|
||||
@@ -676,7 +676,6 @@ export function mapBackendSession(raw: any): Session {
|
||||
runtimeControlState: raw.runtime_control_state ?? raw.runtimeControlState,
|
||||
canStop: raw.can_stop ?? raw.canStop,
|
||||
canResume: raw.can_resume ?? raw.canResume,
|
||||
resumeParentTaskId: raw.resume_parent_task_id ?? raw.resumeParentTaskId,
|
||||
resumeParentSessionId: raw.resume_parent_session_id ?? raw.resumeParentSessionId,
|
||||
pendingRuntimeCheckpointId: raw.pending_runtime_checkpoint_id ?? raw.pendingRuntimeCheckpointId,
|
||||
stopIntentId: raw.stop_intent_id ?? raw.stopIntentId,
|
||||
|
||||
@@ -387,7 +387,6 @@ export function getConversationSessionView(
|
||||
runtimeControlState: runtimeSource.runtimeControlState ?? normalizedActiveSession.runtimeControlState,
|
||||
canStop: runtimeSource.canStop ?? normalizedActiveSession.canStop,
|
||||
canResume: runtimeSource.canResume ?? normalizedActiveSession.canResume,
|
||||
resumeParentTaskId: runtimeSource.resumeParentTaskId ?? normalizedActiveSession.resumeParentTaskId,
|
||||
resumeParentSessionId: runtimeSource.resumeParentSessionId ?? normalizedActiveSession.resumeParentSessionId,
|
||||
pendingRuntimeCheckpointId: runtimeSource.pendingRuntimeCheckpointId ?? normalizedActiveSession.pendingRuntimeCheckpointId,
|
||||
stopIntentId: runtimeSource.stopIntentId ?? normalizedActiveSession.stopIntentId,
|
||||
@@ -447,7 +446,6 @@ export function getConversationHeaderSession(
|
||||
runtimeControlState: runtimeSource.runtimeControlState ?? normalizedActiveSession.runtimeControlState,
|
||||
canStop: runtimeSource.canStop ?? normalizedActiveSession.canStop,
|
||||
canResume: runtimeSource.canResume ?? normalizedActiveSession.canResume,
|
||||
resumeParentTaskId: runtimeSource.resumeParentTaskId ?? normalizedActiveSession.resumeParentTaskId,
|
||||
resumeParentSessionId: runtimeSource.resumeParentSessionId ?? normalizedActiveSession.resumeParentSessionId,
|
||||
pendingRuntimeCheckpointId: runtimeSource.pendingRuntimeCheckpointId ?? normalizedActiveSession.pendingRuntimeCheckpointId,
|
||||
stopIntentId: runtimeSource.stopIntentId ?? normalizedActiveSession.stopIntentId,
|
||||
|
||||
@@ -30,6 +30,17 @@ const flushPromises = async () => {
|
||||
|
||||
const client = new VisualSocketClient('ws://unit.test', {})
|
||||
|
||||
// Company Continue keeps the selected UI channel task separate from the
|
||||
// durable runtime identity used by the checkpoint handoff.
|
||||
client.sessionResume('project-a', 'ui-task', 'runtime-session', 'checkpoint-1')
|
||||
const resumeEnvelope = JSON.parse(
|
||||
(client as unknown as TestSocketClient).pendingQueue.pop() ?? '{}',
|
||||
) as Record<string, unknown>
|
||||
assert.equal(resumeEnvelope.type, 'session_resume')
|
||||
assert.equal(resumeEnvelope.task_id, 'ui-task')
|
||||
assert.equal(resumeEnvelope.runtime_session_id, 'runtime-session')
|
||||
assert.equal(resumeEnvelope.checkpoint_id, 'checkpoint-1')
|
||||
|
||||
// A summary and a full request for the same task are distinct correlations.
|
||||
// Neither Promise may settle merely because the request was queued locally.
|
||||
const summaryPromise = client.sessionDetail('project-a', 'task-1', { detailLevel: 'summary' })
|
||||
|
||||
@@ -41,8 +41,6 @@ interface SocketHandlers {
|
||||
onProjectSwitched?: (payload: { project_id: string; switch_seq?: string }) => void
|
||||
onProjectDeleted?: (payload: { project_id: string }) => void
|
||||
onOrgInfo?: (payload: OrgInfoPayload) => void
|
||||
onRecoveryStatus?: (payload: any) => void
|
||||
onRecoveryResult?: (payload: any) => void
|
||||
onTalentList?: (payload: TalentListPayload) => void
|
||||
onTalentScanLocal?: (payload: { templates: Array<{ template_id: string; name: string; description: string; category: string; domains: string[]; tags: string[] }> }) => void
|
||||
onEmployeeDetail?: (payload: EmployeeDetailPayload) => void
|
||||
@@ -169,7 +167,6 @@ const PROJECT_SCOPED_MESSAGE_TYPES = new Set([
|
||||
'session_update_title',
|
||||
'secretary_send',
|
||||
'project_index',
|
||||
'recovery_action',
|
||||
'comms_state',
|
||||
'comms_read_message',
|
||||
])
|
||||
@@ -420,9 +417,22 @@ export class VisualSocketClient {
|
||||
this.send({ type: 'session_stop', project_id: pid, task_id: taskId })
|
||||
}
|
||||
|
||||
sessionResume(projectId: string, taskId: string, content?: string): void {
|
||||
sessionResume(
|
||||
projectId: string,
|
||||
taskId: string,
|
||||
runtimeSessionId?: string,
|
||||
checkpointId?: string,
|
||||
content?: string,
|
||||
): void {
|
||||
const pid = this.requireProjectId(projectId, 'session_resume')
|
||||
this.send({ type: 'session_resume', project_id: pid, task_id: taskId, content })
|
||||
this.send({
|
||||
type: 'session_resume',
|
||||
project_id: pid,
|
||||
task_id: taskId,
|
||||
runtime_session_id: runtimeSessionId,
|
||||
checkpoint_id: checkpointId,
|
||||
content,
|
||||
})
|
||||
}
|
||||
|
||||
sessionComplete(projectId: string, taskId: string): void {
|
||||
@@ -646,11 +656,6 @@ export class VisualSocketClient {
|
||||
this.send({ type: 'org_saved_delete', name })
|
||||
}
|
||||
|
||||
recoveryAction(projectId: string, action: 'resume' | 'cancel' | 'scan', parentTaskId?: string): void {
|
||||
const pid = this.requireProjectId(projectId, 'recovery_action')
|
||||
this.send({ type: 'recovery_action', project_id: pid, action, parent_task_id: parentTaskId })
|
||||
}
|
||||
|
||||
commsState(projectId: string, opts?: { task_id?: string; session_id?: string }): void {
|
||||
const pid = this.requireProjectId(projectId, 'comms_state')
|
||||
this.send({ type: 'comms_state', project_id: pid, ...(opts || {}) })
|
||||
@@ -801,12 +806,6 @@ export class VisualSocketClient {
|
||||
if (projectId) this.commsState(projectId)
|
||||
} catch { /* ignore */ }
|
||||
break
|
||||
case 'recovery_status':
|
||||
this.handlers.onRecoveryStatus?.(parsed.payload)
|
||||
break
|
||||
case 'recovery_result':
|
||||
this.handlers.onRecoveryResult?.(parsed.payload)
|
||||
break
|
||||
case 'talent_list':
|
||||
this.handlers.onTalentList?.(parsed.payload)
|
||||
break
|
||||
|
||||
@@ -244,7 +244,6 @@ function sessionReducer(state: Session[], action: SessionAction): Session[] {
|
||||
runtimeControlState: guardedRuntimeControl.runtimeControlState ?? existing.runtimeControlState,
|
||||
canStop: guardedRuntimeControl.canStop ?? existing.canStop,
|
||||
canResume: guardedRuntimeControl.canResume ?? existing.canResume,
|
||||
resumeParentTaskId: incoming.resumeParentTaskId ?? existing.resumeParentTaskId,
|
||||
resumeParentSessionId: incoming.resumeParentSessionId ?? existing.resumeParentSessionId,
|
||||
pendingRuntimeCheckpointId: guardedRuntimeControl.pendingRuntimeCheckpointId ?? existing.pendingRuntimeCheckpointId,
|
||||
stopIntentId: guardedRuntimeControl.stopIntentId ?? existing.stopIntentId,
|
||||
@@ -337,7 +336,6 @@ function sessionReducer(state: Session[], action: SessionAction): Session[] {
|
||||
runtimeControlState: guarded.runtimeControlState ?? s.runtimeControlState,
|
||||
canStop: guarded.canStop ?? s.canStop,
|
||||
canResume: guarded.canResume ?? s.canResume,
|
||||
resumeParentTaskId: guarded.resumeParentTaskId ?? s.resumeParentTaskId,
|
||||
resumeParentSessionId: guarded.resumeParentSessionId ?? s.resumeParentSessionId,
|
||||
pendingRuntimeCheckpointId: guarded.pendingRuntimeCheckpointId ?? s.pendingRuntimeCheckpointId,
|
||||
stopIntentId: guarded.stopIntentId ?? s.stopIntentId,
|
||||
|
||||
@@ -316,10 +316,9 @@ export interface Session {
|
||||
originChannel?: string
|
||||
originTaskId?: string
|
||||
runtimeControlState?: 'running' | 'suspending' | 'suspended' | 'resuming' | 'idle'
|
||||
canStop?: boolean
|
||||
canResume?: boolean
|
||||
resumeParentTaskId?: string
|
||||
resumeParentSessionId?: string
|
||||
canStop?: boolean
|
||||
canResume?: boolean
|
||||
resumeParentSessionId?: string
|
||||
pendingRuntimeCheckpointId?: string
|
||||
stopIntentId?: string
|
||||
// Handoff context from upstream work item (Company Mode)
|
||||
|
||||
@@ -100,8 +100,6 @@ export type SocketEnvelope =
|
||||
| { type: 'work_item_batch_updated'; payload: { run_id?: string; work_items: RuntimeWorkItemInfo[]; frontier?: RuntimeFrontierSummary } }
|
||||
| { type: 'project_recovery_updated'; payload: Record<string, unknown> }
|
||||
| { type: 'project_revision_created'; payload: { run_id?: string; revision_links: SessionLinkInfo[] } }
|
||||
| { type: 'recovery_status'; payload: Record<string, unknown> }
|
||||
| { type: 'recovery_result'; payload: Record<string, unknown> }
|
||||
| { type: 'talent_list'; payload: TalentListPayload }
|
||||
| { type: 'talent_scan_local'; payload: { templates: Array<{ template_id: string; name: string; description: string; category: string; domains: string[]; tags: string[] }> } }
|
||||
| { type: 'employee_detail'; payload: EmployeeDetailPayload }
|
||||
|
||||
@@ -77,7 +77,6 @@ interface ContextPanelProps {
|
||||
onCommsRefresh?: () => void
|
||||
onCommsReadMessage?: (path: string) => void
|
||||
orgInfoData?: OrgInfoPayload | null
|
||||
recoveryStatus?: Record<string, unknown> | null
|
||||
canShowTeamTab?: boolean
|
||||
onTeamStopRun?: () => void
|
||||
|
||||
@@ -497,7 +496,6 @@ export function ContextPanel({
|
||||
onCommsRefresh,
|
||||
onCommsReadMessage,
|
||||
orgInfoData,
|
||||
recoveryStatus,
|
||||
canShowTeamTab = false,
|
||||
onTeamStopRun,
|
||||
onTitleChange,
|
||||
@@ -1306,7 +1304,6 @@ export function ContextPanel({
|
||||
<div style={{ flex: 1, overflow: 'auto', minHeight: 0 }}>
|
||||
<ProjectCockpit
|
||||
orgInfoData={orgInfoData ?? null}
|
||||
recoveryStatus={recoveryStatus ?? null}
|
||||
commsState={commsState ?? null}
|
||||
onStopRun={onTeamStopRun}
|
||||
embedded
|
||||
|
||||
@@ -48,7 +48,6 @@ interface TeamCardInfo {
|
||||
|
||||
interface ProjectCockpitProps {
|
||||
orgInfoData?: OrgInfoPayload | null
|
||||
recoveryStatus?: Record<string, unknown> | null
|
||||
commsState?: CommsStatePayload | null
|
||||
onStopRun?: () => void
|
||||
embedded?: boolean
|
||||
@@ -56,7 +55,6 @@ interface ProjectCockpitProps {
|
||||
|
||||
export function ProjectCockpit({
|
||||
orgInfoData,
|
||||
recoveryStatus,
|
||||
commsState,
|
||||
onStopRun,
|
||||
embedded = false,
|
||||
@@ -78,7 +76,6 @@ export function ProjectCockpit({
|
||||
count + asRecordList(asRecord(digest.manager_digest).notification_backlog).length
|
||||
), 0)
|
||||
const unreadCount = actionableCount + protocolCount + notificationCount
|
||||
const interrupted = Array.isArray(recoveryStatus?.interrupted) ? recoveryStatus.interrupted.length : 0
|
||||
|
||||
const communicationItems = [
|
||||
{ label: 'Actionable', value: actionableCount },
|
||||
@@ -198,7 +195,7 @@ export function ProjectCockpit({
|
||||
<span>Seats {runtimeView.runtimeSeats.length}</span>
|
||||
<span>Approvals {pendingDecisionCount}</span>
|
||||
<span>Unread {unreadCount}</span>
|
||||
<span>Recovery {interrupted > 0 ? `${interrupted} interrupted` : summarizeText(asRecord(projectRun?.recovery_pointer).status, 'clean')}</span>
|
||||
<span>Run state {summarizeText(asRecord(projectRun?.recovery_pointer).status, 'clean')}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
@@ -1,116 +0,0 @@
|
||||
import { useState } from 'react'
|
||||
|
||||
export interface RecoverableWorkItem {
|
||||
work_item_projection_id: string
|
||||
title: string
|
||||
task_id: string
|
||||
status: string
|
||||
interrupted: boolean
|
||||
previous_status: string
|
||||
}
|
||||
|
||||
export interface InterruptedWorkItemRuntime {
|
||||
parent_session_id: string
|
||||
parent_task_id: string
|
||||
project_id: string
|
||||
title: string
|
||||
profile: string
|
||||
interrupted_at: string
|
||||
work_items: RecoverableWorkItem[]
|
||||
}
|
||||
|
||||
export interface RecoveryStatusPayload {
|
||||
interrupted: InterruptedWorkItemRuntime[]
|
||||
active_recoveries: string[]
|
||||
scanned_at: number
|
||||
}
|
||||
|
||||
interface WorkItemRecoveryPanelProps {
|
||||
data: RecoveryStatusPayload
|
||||
onResume: (parentTaskId: string) => void
|
||||
onCancel: (parentTaskId: string) => void
|
||||
}
|
||||
|
||||
const STATUS_ICON: Record<string, string> = {
|
||||
done: '\u2713',
|
||||
failed: '\u2717',
|
||||
pending: '\u25CB',
|
||||
blocked: '\u25A0',
|
||||
cancelled: '\u2014',
|
||||
running: '\u25B6',
|
||||
}
|
||||
|
||||
const STATUS_COLOR: Record<string, string> = {
|
||||
done: 'var(--green, #27ae60)',
|
||||
failed: 'var(--red, #e74c3c)',
|
||||
pending: 'var(--text-secondary, #888)',
|
||||
blocked: 'var(--yellow, #f39c12)',
|
||||
cancelled: 'var(--text-dim, #555)',
|
||||
running: 'var(--accent, #3498db)',
|
||||
}
|
||||
|
||||
export function WorkItemRecoveryPanel({ data, onResume, onCancel }: WorkItemRecoveryPanelProps) {
|
||||
const [dismissed, setDismissed] = useState<Set<string>>(new Set())
|
||||
|
||||
if (!data.interrupted.length && !data.active_recoveries.length) return null
|
||||
|
||||
const visible = data.interrupted.filter(w => !dismissed.has(w.parent_task_id))
|
||||
if (!visible.length && !data.active_recoveries.length) return null
|
||||
|
||||
return (
|
||||
<div className="wfr-panel">
|
||||
{visible.map(wf => {
|
||||
const isRecovering = data.active_recoveries.includes(wf.parent_task_id)
|
||||
const doneCount = wf.work_items.filter(item => item.status === 'done').length
|
||||
const failedCount = wf.work_items.filter(item => item.interrupted || item.status === 'failed').length
|
||||
|
||||
return (
|
||||
<div key={wf.parent_task_id} className="wfr-card">
|
||||
<div className="wfr-header">
|
||||
<span className="wfr-icon">⚠</span>
|
||||
<div className="wfr-header-text">
|
||||
<span className="wfr-title">Interrupted: {wf.title}</span>
|
||||
<span className="wfr-subtitle">
|
||||
{doneCount}/{wf.work_items.length} work items done, {failedCount} interrupted
|
||||
{wf.profile && <> · {wf.profile}</>}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="wfr-work-items">
|
||||
{wf.work_items.map(item => (
|
||||
<div key={item.work_item_projection_id} className={`wfr-work-item wfr-work-item--${item.status}`}>
|
||||
<span className="wfr-work-item-icon" style={{ color: STATUS_COLOR[item.status] || STATUS_COLOR.pending }}>
|
||||
{STATUS_ICON[item.status] || STATUS_ICON.pending}
|
||||
</span>
|
||||
<span className="wfr-work-item-title">{item.title}</span>
|
||||
{item.interrupted && <span className="wfr-work-item-badge">interrupted</span>}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<div className="wfr-actions">
|
||||
{isRecovering ? (
|
||||
<span className="wfr-recovering">
|
||||
<span className="spinner-inline" /> Recovering...
|
||||
</span>
|
||||
) : (
|
||||
<>
|
||||
<button className="wfr-btn wfr-btn--resume" onClick={() => onResume(wf.parent_task_id)}>
|
||||
Resume
|
||||
</button>
|
||||
<button className="wfr-btn wfr-btn--cancel" onClick={() => onCancel(wf.parent_task_id)}>
|
||||
Cancel
|
||||
</button>
|
||||
<button className="wfr-btn wfr-btn--dismiss" onClick={() => setDismissed(prev => new Set(prev).add(wf.parent_task_id))}>
|
||||
Dismiss
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react'
|
||||
import type { AgentInfo, OrgInfoPayload, SavedOrgSummary } from '../types/visual'
|
||||
import { WorkItemRecoveryPanel } from './WorkItemRecoveryPanel'
|
||||
import type { ChatMessage, CheckpointReplyMetadata, OutgoingAttachmentPayload } from '../types/chat'
|
||||
import type { KanbanTask, Session, TaskPreferredAgent } from '../types/kanban'
|
||||
import type { BoardStoreState } from '../kanban/BoardStore'
|
||||
@@ -253,7 +252,7 @@ interface WorkspacePageProps {
|
||||
*/
|
||||
onContinueInNewChat?: (mode: 'task' | 'company' | 'org' | 'custom', companyProfile?: 'corporate' | 'custom', orgId?: string) => void
|
||||
onSessionStop?: (taskId: string) => void
|
||||
onSessionResume?: (taskId: string) => void
|
||||
onSessionResume?: (taskId: string, runtimeSessionId?: string, checkpointId?: string) => void
|
||||
onSessionComplete?: (taskId: string) => void
|
||||
onLoadSessionDetail?: (
|
||||
taskId: string,
|
||||
@@ -263,9 +262,6 @@ interface WorkspacePageProps {
|
||||
onCollabSync?: () => void
|
||||
orgInfoData?: OrgInfoPayload | null
|
||||
onNavigateToOrg?: () => void
|
||||
recoveryStatus?: any
|
||||
onRecoveryResume?: (parentTaskId: string) => void
|
||||
onRecoveryCancel?: (parentTaskId: string) => void
|
||||
commsState?: import('../lib/wsClient').CommsStatePayload | null
|
||||
commsMessage?: import('../lib/wsClient').CommsMessagePayload | null
|
||||
onCommsRefresh?: (opts?: { task_id?: string; session_id?: string; project_id?: string }) => void
|
||||
@@ -305,9 +301,6 @@ export function WorkspacePage({
|
||||
onCollabSync,
|
||||
orgInfoData,
|
||||
onNavigateToOrg,
|
||||
recoveryStatus,
|
||||
onRecoveryResume,
|
||||
onRecoveryCancel,
|
||||
commsState,
|
||||
commsMessage,
|
||||
onCommsRefresh,
|
||||
@@ -988,13 +981,21 @@ export function WorkspacePage({
|
||||
|
||||
const handleResume = useCallback(() => {
|
||||
const targetSession = activeConversation.runtimeSession ?? activeConversation.displaySession ?? activeSession
|
||||
const targetTaskId = targetSession?.resumeParentTaskId ?? targetSession?.taskId ?? activeSessionId
|
||||
if (targetTaskId) onSessionResume?.(targetTaskId)
|
||||
const uiTaskId = activeSessionId ?? targetSession?.taskId
|
||||
const runtimeSessionId = targetSession?.resumeParentSessionId
|
||||
?? targetSession?.parentSessionId
|
||||
?? targetSession?.sessionId
|
||||
if (uiTaskId) {
|
||||
onSessionResume?.(uiTaskId, runtimeSessionId, targetSession?.pendingRuntimeCheckpointId)
|
||||
}
|
||||
}, [activeConversation.runtimeSession, activeConversation.displaySession, activeSession, activeSessionId, onSessionResume])
|
||||
|
||||
const handleResumeTask = useCallback((taskId: string) => {
|
||||
const session = sessions.find(s => s.taskId === taskId)
|
||||
onSessionResume?.(session?.resumeParentTaskId ?? taskId)
|
||||
const runtimeSessionId = session?.resumeParentSessionId
|
||||
?? session?.parentSessionId
|
||||
?? session?.sessionId
|
||||
onSessionResume?.(taskId, runtimeSessionId, session?.pendingRuntimeCheckpointId)
|
||||
}, [sessions, onSessionResume])
|
||||
|
||||
const handleCompleteTask = useCallback((taskId: string) => {
|
||||
@@ -1034,8 +1035,11 @@ export function WorkspacePage({
|
||||
}
|
||||
const targetTaskId = activeSessionId
|
||||
if (!targetTaskId) return
|
||||
const checkpointReplyId = String(latestPendingCheckpointReply?.response_to_checkpoint_id ?? '').trim()
|
||||
const runtimeSession = activeConversation.runtimeSession ?? activeConversation.displaySession ?? activeSession
|
||||
const runtimeCheckpointId = String(runtimeSession?.pendingRuntimeCheckpointId ?? '').trim()
|
||||
let outgoingMetadata = latestPendingCheckpointReply
|
||||
?? (runtimeCheckpointId ? { response_to_checkpoint_id: runtimeCheckpointId } : undefined)
|
||||
const checkpointReplyId = String(outgoingMetadata?.response_to_checkpoint_id ?? '').trim()
|
||||
if (!checkpointReplyId) {
|
||||
const uiMessageId = makeOptimisticUserMessageId()
|
||||
outgoingMetadata = { ...(latestPendingCheckpointReply ?? {}), ui_message_id: uiMessageId }
|
||||
@@ -1050,7 +1054,7 @@ export function WorkspacePage({
|
||||
}
|
||||
dispatchSessionSend(targetTaskId, content, attachments, outgoingMetadata)
|
||||
},
|
||||
[effectiveView.kind, activeSessionId, activeConversation.displaySession, activeSession, latestPendingCheckpointReply, chatStore, dispatchSessionSend, onSecretarySend],
|
||||
[effectiveView.kind, activeSessionId, activeConversation.runtimeSession, activeConversation.displaySession, activeSession, latestPendingCheckpointReply, chatStore, dispatchSessionSend, onSecretarySend],
|
||||
)
|
||||
|
||||
// ── MessageList send (checkpoint replies) ──
|
||||
@@ -1114,13 +1118,6 @@ export function WorkspacePage({
|
||||
{/* Middle column: Kanban Board (hidden when panel maximized) */}
|
||||
{panelState !== 'maximized' && (
|
||||
<div className="workspace-board">
|
||||
{recoveryStatus && onRecoveryResume && onRecoveryCancel && (
|
||||
<WorkItemRecoveryPanel
|
||||
data={recoveryStatus}
|
||||
onResume={onRecoveryResume}
|
||||
onCancel={onRecoveryCancel}
|
||||
/>
|
||||
)}
|
||||
{agents.length > 0 && <AgentStatusBar agents={agents} tasks={boardStore.tasks} />}
|
||||
{isCompanyMode ? (
|
||||
boardStore.activeBoard && activeSession && (
|
||||
@@ -1206,7 +1203,6 @@ export function WorkspacePage({
|
||||
onCommsRefresh={onCommsRefresh ? () => onCommsRefresh({ session_id: activeSession?.sessionId || undefined, project_id: projectId || undefined }) : undefined}
|
||||
onCommsReadMessage={onCommsReadMessage}
|
||||
orgInfoData={orgInfoData ?? null}
|
||||
recoveryStatus={recoveryStatus ?? null}
|
||||
canShowTeamTab={canShowTeamTab}
|
||||
onTeamStopRun={activeSessionId ? () => onSessionStop?.(activeSessionId) : undefined}
|
||||
onTitleChange={onTitleChange}
|
||||
|
||||
@@ -1605,132 +1605,6 @@
|
||||
}
|
||||
}
|
||||
|
||||
/* ── Work Item Recovery Panel ────────────────────────────────────────── */
|
||||
|
||||
.wfr-panel {
|
||||
padding: 0 8px;
|
||||
}
|
||||
|
||||
.wfr-card {
|
||||
background: color-mix(in srgb, var(--yellow, #f39c12) 8%, var(--bg-secondary));
|
||||
border: 1px solid color-mix(in srgb, var(--yellow, #f39c12) 25%, transparent);
|
||||
border-radius: 8px;
|
||||
padding: 12px 14px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.wfr-header {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 8px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.wfr-icon {
|
||||
font-size: 16px;
|
||||
line-height: 1;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.wfr-header-text {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2px;
|
||||
min-width: 0;
|
||||
}
|
||||
|
||||
.wfr-title {
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
.wfr-subtitle {
|
||||
font-size: 11px;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
|
||||
.wfr-work-items {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 4px;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
.wfr-work-item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
font-size: 12px;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
|
||||
.wfr-work-item-icon {
|
||||
font-size: 12px;
|
||||
width: 14px;
|
||||
text-align: center;
|
||||
flex-shrink: 0;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.wfr-work-item-title {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.wfr-work-item-badge {
|
||||
font-size: 9px;
|
||||
padding: 1px 5px;
|
||||
border-radius: 3px;
|
||||
background: color-mix(in srgb, var(--red, #e74c3c) 15%, transparent);
|
||||
color: var(--red, #e74c3c);
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
.wfr-actions {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
align-items: center;
|
||||
}
|
||||
|
||||
.wfr-btn {
|
||||
padding: 5px 14px;
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
border: none;
|
||||
border-radius: 6px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.wfr-btn--resume {
|
||||
background: var(--green, #27ae60);
|
||||
color: #fff;
|
||||
}
|
||||
.wfr-btn--resume:hover { opacity: 0.85; }
|
||||
|
||||
.wfr-btn--cancel {
|
||||
background: var(--red, #e74c3c);
|
||||
color: #fff;
|
||||
}
|
||||
.wfr-btn--cancel:hover { opacity: 0.85; }
|
||||
|
||||
.wfr-btn--dismiss {
|
||||
background: transparent;
|
||||
color: var(--text-secondary);
|
||||
}
|
||||
.wfr-btn--dismiss:hover { color: var(--text); }
|
||||
|
||||
.wfr-recovering {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
font-size: 12px;
|
||||
color: var(--accent);
|
||||
}
|
||||
|
||||
/* ═══════════════════════════════════════════════════════
|
||||
Kanban empty-state
|
||||
═══════════════════════════════════════════════════════ */
|
||||
|
||||
@@ -1,552 +0,0 @@
|
||||
"""Pluggable work-item crash-recovery manager.
|
||||
|
||||
Detects interrupted company-mode work items after server restart and provides
|
||||
deterministic Resume / Cancel operations — no LLM needed.
|
||||
|
||||
Hooks into existing engine capabilities without modifying core code:
|
||||
- engine.store → task queries + persistence
|
||||
- engine._load_company_runtime_snapshot() → reconstruct work-item plan + tasks
|
||||
- engine.company_executor.execute() → resume execution
|
||||
- ws_handler.broadcast() → push status to all clients
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Any, Awaitable, Callable
|
||||
|
||||
from opc.layer2_organization.work_item_identity import work_item_projection_id_from_metadata
|
||||
from opc.layer2_organization.work_item_transition import apply_task_status_transition
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data types
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@dataclass
|
||||
class RecoverableWorkItem:
|
||||
work_item_projection_id: str
|
||||
title: str
|
||||
task_id: str
|
||||
status: str # "done" | "failed" | "pending" | "blocked" | "cancelled"
|
||||
interrupted: bool # has interrupted_recovery metadata
|
||||
previous_status: str # what it was before reconciliation
|
||||
|
||||
|
||||
@dataclass
|
||||
class InterruptedWorkItemRun:
|
||||
parent_session_id: str
|
||||
parent_task_id: str
|
||||
project_id: str
|
||||
title: str
|
||||
profile: str
|
||||
interrupted_at: str
|
||||
work_items: list[RecoverableWorkItem] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RecoveryStatus:
|
||||
interrupted: list[InterruptedWorkItemRun] = field(default_factory=list)
|
||||
active_recoveries: list[str] = field(default_factory=list)
|
||||
scanned_at: float = 0.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Manager
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class RuntimeRecoveryManager:
|
||||
"""Scans for interrupted work-item runs and provides deterministic recovery."""
|
||||
|
||||
_CACHE_TTL = 10.0 # seconds
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine: Any,
|
||||
broadcast_fn: Callable[[dict[str, Any]], Awaitable[None]],
|
||||
) -> None:
|
||||
self._engine = engine
|
||||
self._broadcast = broadcast_fn
|
||||
self._lock = asyncio.Lock()
|
||||
self._active_recoveries: dict[str, asyncio.Task[Any]] = {}
|
||||
self._cached: RecoveryStatus | None = None
|
||||
self._cache_until: float = 0.0
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Public API
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def get_recovery_status(self) -> RecoveryStatus:
|
||||
"""Return cached scan results, re-scanning if stale."""
|
||||
now = time.time()
|
||||
if self._cached is not None and now < self._cache_until:
|
||||
# Keep active_recoveries up to date even from cache
|
||||
self._cached.active_recoveries = list(self._active_recoveries.keys())
|
||||
return self._cached
|
||||
status = await self.scan()
|
||||
self._cached = status
|
||||
self._cache_until = now + self._CACHE_TTL
|
||||
return status
|
||||
|
||||
async def scan(self) -> RecoveryStatus:
|
||||
"""Scan the database for recoverable interrupted work-item runs."""
|
||||
store = self._engine.store
|
||||
if not store:
|
||||
return RecoveryStatus()
|
||||
|
||||
project_id = self._engine.project_id or "default"
|
||||
try:
|
||||
all_tasks = await store.get_tasks(project_id=project_id)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Recovery scan failed: {exc}")
|
||||
return RecoveryStatus()
|
||||
|
||||
# Group projected work-item tasks by parent_session_id.
|
||||
groups: dict[str, list[Any]] = {}
|
||||
all_tasks_by_session: dict[str, Any] = {}
|
||||
for task in all_tasks:
|
||||
session_id = str(getattr(task, "session_id", "") or "").strip()
|
||||
if session_id:
|
||||
all_tasks_by_session[session_id] = task
|
||||
|
||||
parent_sid = str(getattr(task, "parent_session_id", "") or "").strip()
|
||||
projection_id = work_item_projection_id_from_metadata(getattr(task, "metadata", {}) or {})
|
||||
if parent_sid and projection_id:
|
||||
groups.setdefault(parent_sid, []).append(task)
|
||||
|
||||
interrupted: list[InterruptedWorkItemRun] = []
|
||||
|
||||
for parent_sid, tasks in groups.items():
|
||||
# Check if any task has interrupted_recovery metadata
|
||||
has_interrupted = any(
|
||||
_is_interrupted(t) for t in tasks
|
||||
)
|
||||
if not has_interrupted:
|
||||
continue
|
||||
|
||||
# Skip if all tasks are terminal (DONE or CANCELLED)
|
||||
from opc.core.models import TaskStatus
|
||||
non_terminal = [
|
||||
t for t in tasks
|
||||
if t.status not in (TaskStatus.DONE, TaskStatus.CANCELLED)
|
||||
]
|
||||
if not non_terminal:
|
||||
continue
|
||||
|
||||
# Find the parent (primary) task
|
||||
parent_task = all_tasks_by_session.get(parent_sid)
|
||||
parent_task_id = parent_task.id if parent_task else parent_sid
|
||||
title = parent_task.title if parent_task else "Unknown work-item run"
|
||||
|
||||
# Build work-item list.
|
||||
work_items: list[RecoverableWorkItem] = []
|
||||
earliest_interrupt = ""
|
||||
for t in sorted(tasks, key=lambda x: (x.created_at, x.id)):
|
||||
meta = dict(getattr(t, "metadata", {}) or {})
|
||||
recovery_meta = meta.get("interrupted_recovery", {})
|
||||
is_int = _is_interrupted(t)
|
||||
if is_int and recovery_meta.get("detected_at", ""):
|
||||
detected = recovery_meta["detected_at"]
|
||||
if not earliest_interrupt or detected < earliest_interrupt:
|
||||
earliest_interrupt = detected
|
||||
|
||||
work_items.append(RecoverableWorkItem(
|
||||
work_item_projection_id=work_item_projection_id_from_metadata(meta, fallback=t.id),
|
||||
title=t.title,
|
||||
task_id=t.id,
|
||||
status=t.status.value if hasattr(t.status, "value") else str(t.status),
|
||||
interrupted=is_int,
|
||||
previous_status=recovery_meta.get("previous_status", ""),
|
||||
))
|
||||
|
||||
profile = ""
|
||||
for t in tasks:
|
||||
p = (getattr(t, "metadata", {}) or {}).get("company_profile", "")
|
||||
if p:
|
||||
profile = p
|
||||
break
|
||||
|
||||
interrupted.append(InterruptedWorkItemRun(
|
||||
parent_session_id=parent_sid,
|
||||
parent_task_id=parent_task_id,
|
||||
project_id=project_id,
|
||||
title=title,
|
||||
profile=profile,
|
||||
interrupted_at=earliest_interrupt or datetime.now().isoformat(),
|
||||
work_items=work_items,
|
||||
))
|
||||
|
||||
return RecoveryStatus(
|
||||
interrupted=interrupted,
|
||||
active_recoveries=list(self._active_recoveries.keys()),
|
||||
scanned_at=time.time(),
|
||||
)
|
||||
|
||||
async def resume(self, parent_task_id: str) -> dict[str, Any]:
|
||||
"""Deterministically resume an interrupted work-item run."""
|
||||
async with self._lock:
|
||||
if parent_task_id in self._active_recoveries:
|
||||
return {"ok": False, "error": "already_in_progress"}
|
||||
|
||||
# Find the interrupted work-item run.
|
||||
status = await self.scan()
|
||||
wf = next((w for w in status.interrupted if w.parent_task_id == parent_task_id), None)
|
||||
if not wf:
|
||||
return {"ok": False, "error": "not_found"}
|
||||
|
||||
# Load snapshot
|
||||
snapshot = await self._engine._load_company_runtime_snapshot(wf.parent_session_id)
|
||||
if not snapshot:
|
||||
return {"ok": False, "error": "snapshot_unavailable"}
|
||||
|
||||
plan, tasks = snapshot
|
||||
|
||||
# Clean orphaned checkpoints
|
||||
await self._clean_orphaned_checkpoints(wf, tasks)
|
||||
|
||||
# Reset interrupted/failed/blocked tasks → PENDING
|
||||
from opc.core.models import TaskStatus
|
||||
resumed_ids: list[str] = []
|
||||
failed_ids: list[str] = []
|
||||
for task in tasks:
|
||||
if task.status == TaskStatus.DONE:
|
||||
continue
|
||||
if task.status in (TaskStatus.FAILED, TaskStatus.BLOCKED):
|
||||
task.result = None
|
||||
task.execution_lock = False
|
||||
task.execution_locked_at = None
|
||||
meta = dict(task.metadata)
|
||||
meta.pop("interrupted_recovery", None)
|
||||
progress = list(meta.get("progress_log", []))
|
||||
progress.append(f"[Recovery] Resumed at {datetime.now().isoformat()}")
|
||||
meta["progress_log"] = progress[-20:]
|
||||
task.metadata = meta
|
||||
projection_id = work_item_projection_id_from_metadata(meta, fallback=task.id)
|
||||
try:
|
||||
await apply_task_status_transition(
|
||||
self._engine.store,
|
||||
task,
|
||||
target_status_or_phase=TaskStatus.PENDING,
|
||||
reason="office_recovery_resume",
|
||||
release_claim=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Recovery resume skipped %s: %s", task.id, exc)
|
||||
failed_ids.append(projection_id)
|
||||
continue
|
||||
if task.status != TaskStatus.PENDING:
|
||||
logger.warning("Recovery resume preserved non-runnable phase for %s", task.id)
|
||||
failed_ids.append(projection_id)
|
||||
continue
|
||||
await self._engine.store.save_task(task)
|
||||
resumed_ids.append(projection_id)
|
||||
|
||||
if not resumed_ids:
|
||||
return {
|
||||
"ok": False,
|
||||
"error": "no_work_items_to_resume",
|
||||
"failed_work_item_projection_ids": failed_ids,
|
||||
}
|
||||
|
||||
# Invalidate cache
|
||||
self._cache_until = 0.0
|
||||
|
||||
# Launch execution in background
|
||||
await self._set_run_recovery_state(
|
||||
wf.parent_session_id,
|
||||
status="resuming",
|
||||
lifecycle_status="active",
|
||||
)
|
||||
bg_task = asyncio.create_task(
|
||||
self._execute_recovery(parent_task_id, wf.parent_session_id, plan, tasks)
|
||||
)
|
||||
self._active_recoveries[parent_task_id] = bg_task
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"resumed_work_item_projection_ids": resumed_ids,
|
||||
"failed_work_item_projection_ids": failed_ids,
|
||||
}
|
||||
|
||||
async def cancel(self, parent_task_id: str) -> dict[str, Any]:
|
||||
"""Cancel an interrupted work-item run and clean up."""
|
||||
async with self._lock:
|
||||
# Cancel active recovery if running
|
||||
bg = self._active_recoveries.pop(parent_task_id, None)
|
||||
if bg and not bg.done():
|
||||
bg.cancel()
|
||||
|
||||
# Find the interrupted work-item run.
|
||||
status = await self.scan()
|
||||
wf = next((w for w in status.interrupted if w.parent_task_id == parent_task_id), None)
|
||||
if not wf:
|
||||
return {"ok": False, "error": "not_found"}
|
||||
|
||||
# Load tasks and cancel non-terminal ones
|
||||
snapshot = await self._engine._load_company_runtime_snapshot(wf.parent_session_id)
|
||||
if not snapshot:
|
||||
return {"ok": False, "error": "snapshot_unavailable"}
|
||||
|
||||
_, tasks = snapshot
|
||||
from opc.core.models import TaskStatus
|
||||
cancelled_count = 0
|
||||
failed_ids: list[str] = []
|
||||
for task in tasks:
|
||||
if task.status not in (TaskStatus.DONE, TaskStatus.CANCELLED):
|
||||
projection_id = work_item_projection_id_from_metadata(
|
||||
getattr(task, "metadata", {}) or {},
|
||||
fallback=task.id,
|
||||
)
|
||||
try:
|
||||
await apply_task_status_transition(
|
||||
self._engine.store,
|
||||
task,
|
||||
target_status_or_phase=TaskStatus.CANCELLED,
|
||||
reason="office_recovery_cancel",
|
||||
release_claim=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Recovery cancel skipped %s: %s", task.id, exc)
|
||||
failed_ids.append(projection_id)
|
||||
continue
|
||||
if task.status != TaskStatus.CANCELLED:
|
||||
logger.warning("Recovery cancel preserved non-cancelled phase for %s", task.id)
|
||||
failed_ids.append(projection_id)
|
||||
continue
|
||||
cancelled_count += 1
|
||||
|
||||
# Clean orphaned checkpoints
|
||||
await self._clean_orphaned_checkpoints(wf, tasks)
|
||||
|
||||
# Invalidate cache
|
||||
self._cache_until = 0.0
|
||||
|
||||
await self._set_run_recovery_state(
|
||||
wf.parent_session_id,
|
||||
status="cancelled",
|
||||
lifecycle_status="cancelled",
|
||||
)
|
||||
await self._broadcast_status()
|
||||
return {
|
||||
"ok": True,
|
||||
"cancelled_count": cancelled_count,
|
||||
"failed_work_item_projection_ids": failed_ids,
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _execute_recovery(
|
||||
self,
|
||||
parent_task_id: str,
|
||||
parent_session_id: str,
|
||||
plan: Any,
|
||||
tasks: list[Any],
|
||||
) -> None:
|
||||
"""Run the work-item executor in the background."""
|
||||
project_id = self._engine.project_id or "default"
|
||||
try:
|
||||
await self._set_run_recovery_state(
|
||||
parent_session_id,
|
||||
status="started",
|
||||
lifecycle_status="active",
|
||||
)
|
||||
await self._broadcast({"type": "recovery_result", "payload": {
|
||||
"project_id": project_id,
|
||||
"parent_task_id": parent_task_id, "status": "started",
|
||||
}})
|
||||
|
||||
executor = self._engine.company_executor
|
||||
if not executor:
|
||||
raise RuntimeError("company_executor not available")
|
||||
|
||||
result = await executor.execute(plan, tasks)
|
||||
|
||||
await self._broadcast({"type": "recovery_result", "payload": {
|
||||
"project_id": project_id,
|
||||
"parent_task_id": parent_task_id, "status": "completed",
|
||||
"summary": result[:500] if result else "",
|
||||
}})
|
||||
await self._set_run_recovery_state(
|
||||
parent_session_id,
|
||||
status="completed",
|
||||
lifecycle_status="active",
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
await self._broadcast({"type": "recovery_result", "payload": {
|
||||
"project_id": project_id,
|
||||
"parent_task_id": parent_task_id, "status": "cancelled",
|
||||
}})
|
||||
await self._set_run_recovery_state(
|
||||
parent_session_id,
|
||||
status="cancelled",
|
||||
lifecycle_status="cancelled",
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(f"Recovery execution failed for {parent_task_id}: {exc}")
|
||||
await self._broadcast({"type": "recovery_result", "payload": {
|
||||
"project_id": project_id,
|
||||
"parent_task_id": parent_task_id, "status": "failed",
|
||||
"error": str(exc),
|
||||
}})
|
||||
await self._set_run_recovery_state(
|
||||
parent_session_id,
|
||||
status="failed",
|
||||
lifecycle_status="blocked",
|
||||
extra={"error": str(exc)},
|
||||
)
|
||||
finally:
|
||||
self._active_recoveries.pop(parent_task_id, None)
|
||||
self._cache_until = 0.0
|
||||
await self._broadcast_status()
|
||||
|
||||
async def _clean_orphaned_checkpoints(
|
||||
self,
|
||||
wf: InterruptedWorkItemRun,
|
||||
tasks: list[Any],
|
||||
) -> int:
|
||||
"""Resolve pending checkpoints whose tasks are no longer active."""
|
||||
store = self._engine.store
|
||||
if not store:
|
||||
return 0
|
||||
|
||||
session_ids = {
|
||||
str(getattr(t, "session_id", "") or "").strip()
|
||||
for t in tasks
|
||||
}
|
||||
session_ids.add(wf.parent_session_id)
|
||||
session_ids.discard("")
|
||||
|
||||
cleaned = 0
|
||||
try:
|
||||
pending = await store.get_pending_checkpoints(
|
||||
project_id=wf.project_id,
|
||||
)
|
||||
for cp in pending:
|
||||
cp_session = str(cp.session_id or "").strip()
|
||||
if cp_session in session_ids:
|
||||
await store.resolve_execution_checkpoint(
|
||||
cp.checkpoint_id, status="cancelled"
|
||||
)
|
||||
cleaned += 1
|
||||
except Exception as exc:
|
||||
logger.debug(f"Checkpoint cleanup error: {exc}")
|
||||
|
||||
return cleaned
|
||||
|
||||
async def _broadcast_status(self) -> None:
|
||||
"""Push updated recovery status to all connected clients."""
|
||||
try:
|
||||
status = await self.get_recovery_status()
|
||||
await self._broadcast({"type": "recovery_status", "payload":
|
||||
_serialize_status(status, project_id=self._engine.project_id or "default")
|
||||
})
|
||||
except Exception as exc:
|
||||
logger.debug(f"Recovery status broadcast failed: {exc}")
|
||||
|
||||
def _invalidate_cache(self) -> None:
|
||||
"""Force next get_recovery_status to re-scan."""
|
||||
self._cache_until = 0.0
|
||||
self._cached = None
|
||||
|
||||
async def _set_run_recovery_state(
|
||||
self,
|
||||
key: str,
|
||||
*,
|
||||
status: str,
|
||||
lifecycle_status: str | None = None,
|
||||
match_task_id: bool = False,
|
||||
extra: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
store = getattr(self._engine, "store", None)
|
||||
if not store or not hasattr(store, "list_delegation_runs") or not hasattr(store, "save_delegation_run"):
|
||||
return
|
||||
runs = await store.list_delegation_runs(project_id=self._engine.project_id or "default")
|
||||
target = None
|
||||
if match_task_id:
|
||||
for run in runs:
|
||||
metadata = dict(getattr(run, "metadata", {}) or {})
|
||||
if str(metadata.get("origin_task_id", "") or "").strip() == key:
|
||||
target = run
|
||||
break
|
||||
if target is None:
|
||||
for run in runs:
|
||||
if str(run.session_id or "").strip() == key:
|
||||
target = run
|
||||
break
|
||||
if target is None:
|
||||
return
|
||||
target.recovery_pointer = {
|
||||
**dict(getattr(target, "recovery_pointer", {}) or {}),
|
||||
"status": status,
|
||||
"updated_at": datetime.now().isoformat(),
|
||||
**dict(extra or {}),
|
||||
}
|
||||
if lifecycle_status:
|
||||
target.lifecycle_status = lifecycle_status
|
||||
target.updated_at = datetime.now()
|
||||
await store.save_delegation_run(target)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _is_interrupted(task: Any) -> bool:
|
||||
"""Check if a task was interrupted by crash."""
|
||||
from opc.core.models import TaskStatus
|
||||
if task.status != TaskStatus.FAILED:
|
||||
return False
|
||||
meta = getattr(task, "metadata", {}) or {}
|
||||
if meta.get("interrupted_recovery"):
|
||||
return True
|
||||
result = getattr(task, "result", {}) or {}
|
||||
artifacts = result.get("artifacts", {}) or {}
|
||||
return bool(artifacts.get("interrupted"))
|
||||
|
||||
|
||||
def _serialize_status(status: RecoveryStatus, *, project_id: str | None = None) -> dict[str, Any]:
|
||||
"""Convert RecoveryStatus to JSON-safe dict."""
|
||||
resolved_project_id = str(project_id or "").strip()
|
||||
if not resolved_project_id:
|
||||
for item in status.interrupted:
|
||||
if item.project_id:
|
||||
resolved_project_id = item.project_id
|
||||
break
|
||||
payload = {
|
||||
"interrupted": [
|
||||
{
|
||||
"parent_session_id": w.parent_session_id,
|
||||
"parent_task_id": w.parent_task_id,
|
||||
"project_id": w.project_id,
|
||||
"title": w.title,
|
||||
"profile": w.profile,
|
||||
"interrupted_at": w.interrupted_at,
|
||||
"work_items": [
|
||||
{
|
||||
"work_item_projection_id": s.work_item_projection_id,
|
||||
"title": s.title,
|
||||
"task_id": s.task_id,
|
||||
"status": s.status,
|
||||
"interrupted": s.interrupted,
|
||||
"previous_status": s.previous_status,
|
||||
}
|
||||
for s in w.work_items
|
||||
],
|
||||
}
|
||||
for w in status.interrupted
|
||||
],
|
||||
"active_recoveries": status.active_recoveries,
|
||||
"scanned_at": status.scanned_at,
|
||||
}
|
||||
if resolved_project_id:
|
||||
payload["project_id"] = resolved_project_id
|
||||
return payload
|
||||
@@ -144,17 +144,6 @@ async def create_app(
|
||||
|
||||
engine.event_bus.subscribe_all(_root_engine_event)
|
||||
|
||||
# ── Runtime crash recovery (pluggable, no engine modifications) ──
|
||||
from opc.plugins.office_ui.recovery_manager import RuntimeRecoveryManager
|
||||
recovery_manager = RuntimeRecoveryManager(engine, ws_handler.broadcast)
|
||||
ws_handler.recovery_manager = recovery_manager
|
||||
|
||||
# ── Startup self-heal for tasks abandoned by a prior process ─────
|
||||
# Must run before restoring persisted mode and before any WS client can
|
||||
# connect, so orphaned running/locked rows do not block new Continue /
|
||||
# session_send acquisitions.
|
||||
await ws_handler.heal_orphan_tasks_on_boot()
|
||||
|
||||
# ── Restore persisted mode and load matching agents on startup ───
|
||||
await ws_handler.restore_persisted_mode()
|
||||
startup_preset = ws_handler._resolve_preset_name()
|
||||
|
||||
@@ -32,4 +32,13 @@ class ServiceError(Exception):
|
||||
self.payload = dict(payload or {})
|
||||
|
||||
def to_payload(self) -> dict[str, Any]:
|
||||
return {"error": self.message, "code": self.code, **self.payload}
|
||||
# Transport envelope fields are authoritative. Business details may
|
||||
# add context, but must never turn an error acknowledgement into a
|
||||
# success or replace the exception's code/message.
|
||||
payload = {
|
||||
key: value
|
||||
for key, value in self.payload.items()
|
||||
if key not in {"ok", "error", "code"}
|
||||
}
|
||||
payload.update({"error": self.message, "code": self.code})
|
||||
return payload
|
||||
|
||||
@@ -200,48 +200,3 @@ class RuntimeService:
|
||||
payload["display_text"] = " | ".join(display_parts)
|
||||
payload["event_type"] = event_type
|
||||
return payload
|
||||
|
||||
async def recovery_scan(self, *, project_id: str) -> ServiceResult:
|
||||
manager = await self._recovery_manager(project_id)
|
||||
from opc.plugins.office_ui.recovery_manager import _serialize_status
|
||||
|
||||
status = await manager.get_recovery_status()
|
||||
return ServiceResult(_serialize_status(status, project_id=project_id))
|
||||
|
||||
async def recovery_action(self, *, project_id: str, action: str, parent_task_id: str) -> ServiceResult:
|
||||
manager = await self._recovery_manager(project_id)
|
||||
normalized = str(action or "").strip().lower()
|
||||
if normalized == "scan":
|
||||
return await self.recovery_scan(project_id=project_id)
|
||||
if not str(parent_task_id or "").strip():
|
||||
raise ServiceError("parent_task_id_required", "parent_task_id required")
|
||||
if normalized in {"resume", "retry"}:
|
||||
payload = await manager.resume(parent_task_id)
|
||||
elif normalized == "cancel":
|
||||
payload = await manager.cancel(parent_task_id)
|
||||
else:
|
||||
raise ServiceError("unknown_recovery_action", f"unknown action: {action}", {"action": action})
|
||||
payload = {**dict(payload), "project_id": project_id, "parent_task_id": parent_task_id, "action": normalized}
|
||||
if not payload.get("ok", False):
|
||||
raise ServiceError(str(payload.get("error") or "recovery_failed"), str(payload.get("error") or "recovery_failed"), payload)
|
||||
return ServiceResult(payload)
|
||||
|
||||
async def _recovery_manager(self, project_id: str) -> Any:
|
||||
engine = await self.context.engine_for_project(project_id)
|
||||
|
||||
async def _noop_broadcast(_event: dict[str, Any]) -> None:
|
||||
return None
|
||||
|
||||
managers = getattr(self.context, "recovery_managers", None)
|
||||
if managers is None:
|
||||
managers = {}
|
||||
setattr(self.context, "recovery_managers", managers)
|
||||
key = self.context.normalize_project_id(project_id)
|
||||
existing = managers.get(key)
|
||||
if existing is not None and getattr(existing, "_engine", None) is engine:
|
||||
return existing
|
||||
from opc.plugins.office_ui.recovery_manager import RuntimeRecoveryManager
|
||||
|
||||
manager = RuntimeRecoveryManager(engine, _noop_broadcast)
|
||||
managers[key] = manager
|
||||
return manager
|
||||
|
||||
@@ -12,6 +12,12 @@ from typing import Any
|
||||
|
||||
from loguru import logger
|
||||
from opc.core.models import Task, TaskStatus
|
||||
from opc.layer2_organization.company_runtime_identity import (
|
||||
ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES,
|
||||
COMPANY_RUNTIME_CHECKPOINT_TYPES,
|
||||
is_company_runtime_task,
|
||||
load_company_runtime_identity_index,
|
||||
)
|
||||
from opc.plugins.office_ui.execution_identity import (
|
||||
ExecutionIdentity,
|
||||
canonicalize_execution_identity,
|
||||
@@ -156,6 +162,32 @@ class SessionService:
|
||||
task_project = self.context.normalize_project_id(getattr(task, "project_id", None))
|
||||
if task_project != pid:
|
||||
raise ServiceError("target_wrong_project", "Target belongs to a different project", {"project_id": task_project})
|
||||
|
||||
# Company control is resolved from durable session/checkpoint scope
|
||||
# before any generic Task/session fallback. This prevents a shared
|
||||
# final-decider row from winning merely because it was loaded first.
|
||||
identity_index = await load_company_runtime_identity_index(store, pid)
|
||||
if task is not None:
|
||||
identity = identity_index.resolve(task_id=raw_target)
|
||||
else:
|
||||
identity = (
|
||||
identity_index.resolve(runtime_session_id=raw_target)
|
||||
or identity_index.resolve(task_session_id=raw_target)
|
||||
)
|
||||
if identity is not None:
|
||||
# A concrete Task id is the caller's UI/chat channel, never the
|
||||
# company execution identity. Keep that channel stable while the
|
||||
# shared identity supplies the durable runtime session/checkpoint.
|
||||
if task is not None:
|
||||
return task, identity.runtime_session_id
|
||||
resolved_task = (
|
||||
identity_index.task(identity.ui_anchor_task_id)
|
||||
or identity_index.task(identity.config_source_task_id)
|
||||
)
|
||||
if resolved_task is not None:
|
||||
return resolved_task, identity.runtime_session_id
|
||||
|
||||
if task is not None:
|
||||
return task, str(getattr(task, "session_id", "") or getattr(task, "parent_session_id", "") or "")
|
||||
|
||||
session = await store.get_session(raw_target) if hasattr(store, "get_session") else None
|
||||
@@ -165,139 +197,64 @@ class SessionService:
|
||||
if session_project != pid:
|
||||
raise ServiceError("target_wrong_project", "Target belongs to a different project", {"project_id": session_project})
|
||||
|
||||
tasks = await store.get_tasks(project_id=pid) if hasattr(store, "get_tasks") else []
|
||||
tasks = list(identity_index.tasks)
|
||||
session_tasks = [
|
||||
candidate for candidate in tasks
|
||||
if str(getattr(candidate, "session_id", "") or "") == raw_target
|
||||
]
|
||||
if not session_tasks:
|
||||
raise ServiceError("session_not_task_backed", "Session is not linked to a task-backed runtime", {"session_id": raw_target})
|
||||
session_tasks.sort(key=lambda item: bool(str(getattr(item, "parent_session_id", "") or "")))
|
||||
return session_tasks[0], raw_target
|
||||
task_mode_anchor = min(
|
||||
session_tasks,
|
||||
key=lambda item: (
|
||||
bool(str(getattr(item, "parent_session_id", "") or "")),
|
||||
str(getattr(item, "created_at", "") or ""),
|
||||
str(getattr(item, "id", "") or ""),
|
||||
),
|
||||
)
|
||||
return task_mode_anchor, raw_target
|
||||
|
||||
async def _resolve_company_runtime_target(self, engine: Any, task: Any) -> dict[str, Any]:
|
||||
store = getattr(engine, "store", None)
|
||||
parent_session_id = str(
|
||||
getattr(task, "parent_session_id", "")
|
||||
or getattr(task, "session_id", "")
|
||||
or ""
|
||||
).strip()
|
||||
parent_task_id = str(self.context.session_to_task.get(parent_session_id) or "").strip()
|
||||
project_id = self.context.normalize_project_id(getattr(task, "project_id", None) or getattr(engine, "project_id", None))
|
||||
try:
|
||||
project_tasks = await store.get_tasks(project_id=project_id) if hasattr(store, "get_tasks") else [task]
|
||||
except Exception:
|
||||
project_tasks = [task]
|
||||
for candidate in project_tasks:
|
||||
candidate_id = str(getattr(candidate, "id", "") or "").strip()
|
||||
candidate_session_id = str(getattr(candidate, "session_id", "") or "").strip()
|
||||
candidate_parent_session_id = str(getattr(candidate, "parent_session_id", "") or "").strip()
|
||||
if candidate_session_id == parent_session_id and not candidate_parent_session_id:
|
||||
parent_task_id = candidate_id
|
||||
break
|
||||
if not parent_task_id:
|
||||
parent_task_id = str(
|
||||
self.context.active_runtime_children.get(str(getattr(task, "id", "") or ""))
|
||||
or getattr(task, "id", "")
|
||||
or ""
|
||||
).strip()
|
||||
|
||||
affected_task_ids: list[str] = []
|
||||
for candidate in project_tasks:
|
||||
candidate_id = str(getattr(candidate, "id", "") or "").strip()
|
||||
if not candidate_id:
|
||||
continue
|
||||
candidate_session_id = str(getattr(candidate, "session_id", "") or "").strip()
|
||||
candidate_parent_session_id = str(getattr(candidate, "parent_session_id", "") or "").strip()
|
||||
if (
|
||||
candidate_id == str(getattr(task, "id", "") or "")
|
||||
or candidate_id == parent_task_id
|
||||
or candidate_session_id == parent_session_id
|
||||
or candidate_parent_session_id == parent_session_id
|
||||
):
|
||||
if candidate_id not in affected_task_ids:
|
||||
affected_task_ids.append(candidate_id)
|
||||
for child_id, origin_id in list(self.context.active_runtime_children.items()):
|
||||
if origin_id == parent_task_id or child_id == str(getattr(task, "id", "") or ""):
|
||||
if child_id not in affected_task_ids:
|
||||
affected_task_ids.append(child_id)
|
||||
if parent_task_id and parent_task_id not in affected_task_ids:
|
||||
affected_task_ids.insert(0, parent_task_id)
|
||||
return {
|
||||
"parent_session_id": parent_session_id,
|
||||
"parent_task_id": parent_task_id or str(getattr(task, "id", "") or ""),
|
||||
"origin_task_id": parent_task_id or str(getattr(task, "id", "") or ""),
|
||||
"affected_task_ids": affected_task_ids or [str(getattr(task, "id", "") or "")],
|
||||
}
|
||||
|
||||
async def _mark_company_runtime_stop_state(
|
||||
async def _resolve_company_runtime_target(
|
||||
self,
|
||||
*,
|
||||
engine: Any,
|
||||
task_ids: list[str],
|
||||
state: str,
|
||||
stop_intent_id: str,
|
||||
checkpoint_type: str = "",
|
||||
) -> None:
|
||||
task: Any,
|
||||
*,
|
||||
runtime_session_id: str = "",
|
||||
checkpoint_id: str = "",
|
||||
) -> dict[str, Any]:
|
||||
store = getattr(engine, "store", None)
|
||||
project_id = self.context.normalize_project_id(getattr(task, "project_id", None) or getattr(engine, "project_id", None))
|
||||
if not self.context.store_is_ready(store):
|
||||
return
|
||||
for task_id in task_ids:
|
||||
try:
|
||||
task = await store.get_task(str(task_id))
|
||||
except Exception:
|
||||
task = None
|
||||
if not task or self._is_terminal_status(task):
|
||||
continue
|
||||
metadata = dict(getattr(task, "metadata", {}) or {})
|
||||
metadata["company_runtime_stop_state"] = state
|
||||
metadata["company_runtime_stop_intent_id"] = stop_intent_id
|
||||
metadata["company_runtime_stop_marked_at"] = datetime.now().isoformat()
|
||||
metadata["dispatch_hold"] = "company_runtime_suspended"
|
||||
metadata["company_runtime_suspended_at"] = datetime.now().isoformat()
|
||||
if checkpoint_type:
|
||||
metadata["company_runtime_suspend_checkpoint_type"] = checkpoint_type
|
||||
metadata.setdefault("suspended_task_status", self._task_status_value(task))
|
||||
task.metadata = metadata
|
||||
task.status = TaskStatus.BLOCKED
|
||||
if hasattr(task, "execution_lock"):
|
||||
task.execution_lock = False
|
||||
if hasattr(task, "execution_locked_at"):
|
||||
task.execution_locked_at = None
|
||||
try:
|
||||
await store.save_task(task)
|
||||
except Exception:
|
||||
logger.opt(exception=True).debug("failed to mark company runtime stop state")
|
||||
|
||||
async def _clear_company_runtime_stop_state(self, *, engine: Any, task_ids: list[str]) -> None:
|
||||
store = getattr(engine, "store", None)
|
||||
if not self.context.store_is_ready(store):
|
||||
return
|
||||
for task_id in task_ids:
|
||||
try:
|
||||
task = await store.get_task(str(task_id))
|
||||
except Exception:
|
||||
task = None
|
||||
if not task:
|
||||
continue
|
||||
metadata = dict(getattr(task, "metadata", {}) or {})
|
||||
for key in (
|
||||
"dispatch_hold",
|
||||
"company_runtime_stop_state",
|
||||
"company_runtime_stop_intent_id",
|
||||
"company_runtime_stop_marked_at",
|
||||
"company_runtime_suspend_checkpoint_type",
|
||||
"company_runtime_suspended_at",
|
||||
"suspended_task_status",
|
||||
):
|
||||
metadata.pop(key, None)
|
||||
task.metadata = metadata
|
||||
if self._task_status_value(task) == "blocked":
|
||||
task.status = TaskStatus.IDLE
|
||||
try:
|
||||
await store.save_task(task)
|
||||
except Exception:
|
||||
logger.opt(exception=True).debug("failed to clear company runtime stop state")
|
||||
raise ServiceError("store_not_ready", "store_not_ready", {"project_id": project_id})
|
||||
index = await load_company_runtime_identity_index(store, project_id)
|
||||
identity = index.resolve(
|
||||
task_id=str(getattr(task, "id", "") or ""),
|
||||
runtime_session_id=runtime_session_id,
|
||||
checkpoint_id=checkpoint_id,
|
||||
)
|
||||
if identity is None:
|
||||
raise ServiceError(
|
||||
"company_runtime_identity_mismatch",
|
||||
"Company runtime identity does not match the requested task/session/checkpoint",
|
||||
{
|
||||
"task_id": str(getattr(task, "id", "") or ""),
|
||||
"runtime_session_id": str(runtime_session_id or ""),
|
||||
"checkpoint_id": str(checkpoint_id or ""),
|
||||
},
|
||||
)
|
||||
config_task = index.task(identity.config_source_task_id) or task
|
||||
ui_anchor_task_id = identity.ui_anchor_task_id
|
||||
return {
|
||||
"identity": identity,
|
||||
"runtime_session_id": identity.runtime_session_id,
|
||||
"ui_channel_task_id": str(getattr(task, "id", "") or ""),
|
||||
"ui_anchor_task_id": ui_anchor_task_id,
|
||||
"config_source_task_id": identity.config_source_task_id,
|
||||
"config_task": config_task,
|
||||
"origin_task_id": ui_anchor_task_id,
|
||||
"affected_task_ids": list(identity.runtime_task_ids),
|
||||
"checkpoint": identity.checkpoint,
|
||||
}
|
||||
|
||||
def _normalize_requested_config(
|
||||
self,
|
||||
@@ -743,8 +700,54 @@ class SessionService:
|
||||
if getattr(engine, "memory", None):
|
||||
await engine.memory.ensure_session(task.session_id, project_id=project_id, title=task.title, mode="primary", metadata={"source": "service"})
|
||||
await store.save_task(task)
|
||||
|
||||
# A Task selected by the UI/CLI is only the chat channel for company
|
||||
# mode. Resolve the durable runtime scope before choosing execution
|
||||
# configuration, session, origin, or checkpoint. In particular, a
|
||||
# role/work-item Task must never become the parent execution identity.
|
||||
company_target: dict[str, Any] | None = None
|
||||
task_is_company_runtime = is_company_runtime_task(task)
|
||||
try:
|
||||
company_target = await self._resolve_company_runtime_target(engine, task)
|
||||
except ServiceError as exc:
|
||||
if exc.code != "company_runtime_identity_mismatch" or task_is_company_runtime:
|
||||
raise
|
||||
|
||||
if task.status == TaskStatus.CANCELLED:
|
||||
company_identity = (
|
||||
company_target.get("identity")
|
||||
if company_target is not None
|
||||
else None
|
||||
)
|
||||
checkpoint = (
|
||||
company_target.get("checkpoint")
|
||||
if company_target is not None
|
||||
else None
|
||||
)
|
||||
active_cancelled_anchor = bool(
|
||||
company_identity is not None
|
||||
and str(getattr(company_identity, "ui_anchor_task_id", "") or "").strip()
|
||||
== str(getattr(task, "id", "") or "").strip()
|
||||
and checkpoint is not None
|
||||
and str(getattr(checkpoint, "checkpoint_type", "") or "").strip()
|
||||
in COMPANY_RUNTIME_CHECKPOINT_TYPES
|
||||
and str(getattr(checkpoint, "status", "") or "").strip().lower()
|
||||
in ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES
|
||||
)
|
||||
if not active_cancelled_anchor:
|
||||
raise ServiceError(
|
||||
"session_ended",
|
||||
"session_ended",
|
||||
{"project_id": project_id, "task_id": task.id},
|
||||
)
|
||||
|
||||
config_task = (
|
||||
company_target.get("config_task")
|
||||
if company_target is not None
|
||||
else task
|
||||
) or task
|
||||
identity = self.resolve_task_identity(
|
||||
task,
|
||||
config_task,
|
||||
default_exec_mode=mode,
|
||||
default_company_profile=company_profile if company_profile is not None else "corporate",
|
||||
default_preferred_agent=preferred_agent if preferred_agent is not None else "native",
|
||||
@@ -752,26 +755,85 @@ class SessionService:
|
||||
)
|
||||
if identity.is_custom_org and not identity.org_id:
|
||||
raise ServiceError("org_id_required", "org_id_required", {"project_id": project_id, "task_id": task.id})
|
||||
await self.persist_session_config(
|
||||
task,
|
||||
exec_mode=identity.exec_mode,
|
||||
company_profile=identity.company_profile,
|
||||
preferred_agent=identity.preferred_agent,
|
||||
org_id=identity.org_id,
|
||||
engine=engine,
|
||||
)
|
||||
|
||||
# Existing company scopes read configuration from the resolver's
|
||||
# config source. Only persist when that source is the selected Task;
|
||||
# this preserves normal session configuration while avoiding writes to
|
||||
# an internal work item merely because it was used as the UI channel.
|
||||
if (
|
||||
company_target is None
|
||||
or str(getattr(config_task, "id", "") or "").strip()
|
||||
== str(getattr(task, "id", "") or "").strip()
|
||||
):
|
||||
await self.persist_session_config(
|
||||
task,
|
||||
exec_mode=identity.exec_mode,
|
||||
company_profile=identity.company_profile,
|
||||
preferred_agent=identity.preferred_agent,
|
||||
org_id=identity.org_id,
|
||||
engine=engine,
|
||||
)
|
||||
|
||||
execution_session_id = str(getattr(task, "session_id", "") or "").strip()
|
||||
origin_task_id = str(getattr(task, "id", "") or "").strip() or None
|
||||
message_metadata: dict[str, Any] | None = None
|
||||
if company_target is not None:
|
||||
execution_session_id = str(
|
||||
company_target.get("runtime_session_id", "") or ""
|
||||
).strip()
|
||||
origin_task_id = str(
|
||||
company_target.get("ui_anchor_task_id", "") or ""
|
||||
).strip() or None
|
||||
if not execution_session_id:
|
||||
raise ServiceError(
|
||||
"company_runtime_identity_mismatch",
|
||||
"Company runtime has no canonical runtime session",
|
||||
{"project_id": project_id, "task_id": task.id},
|
||||
)
|
||||
checkpoint = company_target.get("checkpoint")
|
||||
if checkpoint is not None:
|
||||
checkpoint_id = str(
|
||||
getattr(checkpoint, "checkpoint_id", "") or ""
|
||||
).strip()
|
||||
checkpoint_status = str(
|
||||
getattr(checkpoint, "status", "") or ""
|
||||
).strip().lower()
|
||||
if checkpoint_status != "pending":
|
||||
raise ServiceError(
|
||||
"company_runtime_checkpoint_not_pending",
|
||||
"Company runtime checkpoint is not pending",
|
||||
{
|
||||
"project_id": project_id,
|
||||
"task_id": task.id,
|
||||
"checkpoint_id": checkpoint_id,
|
||||
"checkpoint_status": checkpoint_status,
|
||||
},
|
||||
)
|
||||
message_metadata = {
|
||||
"response_to_checkpoint_id": checkpoint_id,
|
||||
"response_to_checkpoint_type": str(
|
||||
getattr(checkpoint, "checkpoint_type", "") or ""
|
||||
).strip(),
|
||||
}
|
||||
|
||||
response = await engine.process_message(
|
||||
str(content or "").strip(),
|
||||
project_id=project_id,
|
||||
session_id=task.session_id,
|
||||
session_id=execution_session_id,
|
||||
mode=identity.exec_mode,
|
||||
org_id=identity.org_id or None,
|
||||
company_profile=identity.company_profile if identity.is_company_runtime else None,
|
||||
preferred_agent=identity.preferred_agent if identity.is_task else None,
|
||||
domains=list(domains or []),
|
||||
origin_task_id=task.id,
|
||||
origin_task_id=origin_task_id,
|
||||
message_metadata=message_metadata,
|
||||
)
|
||||
return ServiceResult({"project_id": project_id, "task_id": task.id, "session_id": task.session_id, "response": response})
|
||||
return ServiceResult({
|
||||
"project_id": project_id,
|
||||
"task_id": task.id,
|
||||
"session_id": execution_session_id,
|
||||
"response": response,
|
||||
})
|
||||
|
||||
async def rename(self, *, project_id: str, task_id: str = "", session_id: str = "", title: str) -> ServiceResult:
|
||||
pid = self.context.normalize_project_id(project_id)
|
||||
@@ -935,50 +997,44 @@ class SessionService:
|
||||
default_payload=default_payload,
|
||||
)
|
||||
engine = await self.context.engine_for_project(project_id)
|
||||
exec_mode, _company_profile = self.resolve_task_session_config(task)
|
||||
if exec_mode in {"company", "org", "custom"}:
|
||||
try:
|
||||
target_info = await self._resolve_company_runtime_target(engine, task)
|
||||
except ServiceError as exc:
|
||||
if exc.code != "company_runtime_identity_mismatch":
|
||||
raise
|
||||
target_info = None
|
||||
if target_info is None and is_company_runtime_task(task):
|
||||
raise ServiceError(
|
||||
"company_runtime_identity_mismatch",
|
||||
"Company runtime identity could not be resolved; refusing task-mode cancellation",
|
||||
{"task_id": resolved_task_id, "session_id": resolved_session_id},
|
||||
)
|
||||
if target_info is not None:
|
||||
stop_intent_id = str(uuid.uuid4())
|
||||
affected_task_ids = list(target_info.get("affected_task_ids", []) or [resolved_task_id])
|
||||
suspended: dict[str, Any] | None = None
|
||||
suspend = getattr(engine, "suspend_company_runtime", None)
|
||||
await self._mark_company_runtime_stop_state(
|
||||
engine=engine,
|
||||
task_ids=affected_task_ids,
|
||||
state="suspending",
|
||||
stop_intent_id=stop_intent_id,
|
||||
)
|
||||
if callable(suspend):
|
||||
try:
|
||||
suspended = await suspend(
|
||||
origin_task_id=str(target_info.get("origin_task_id", "") or resolved_task_id),
|
||||
session_id=(str(target_info.get("parent_session_id", "") or resolved_session_id).strip() or None),
|
||||
session_id=(str(target_info.get("runtime_session_id", "") or resolved_session_id).strip() or None),
|
||||
reason="user_stop",
|
||||
checkpoint_type="company_runtime_suspended",
|
||||
stop_intent_id=stop_intent_id,
|
||||
)
|
||||
except Exception:
|
||||
logger.opt(exception=True).warning("suspend_company_runtime failed during service stop")
|
||||
if suspended is not None:
|
||||
for candidate in list(suspended.get("task_ids", []) or []):
|
||||
candidate_id = str(candidate or "").strip()
|
||||
if candidate_id and candidate_id not in affected_task_ids:
|
||||
affected_task_ids.append(candidate_id)
|
||||
await self._mark_company_runtime_stop_state(
|
||||
engine=engine,
|
||||
task_ids=affected_task_ids,
|
||||
state="suspended",
|
||||
stop_intent_id=stop_intent_id,
|
||||
checkpoint_type=str(suspended.get("checkpoint_type", "") or "company_runtime_suspended"),
|
||||
)
|
||||
else:
|
||||
await self._mark_company_runtime_stop_state(
|
||||
engine=engine,
|
||||
task_ids=affected_task_ids,
|
||||
state="suspended",
|
||||
stop_intent_id=stop_intent_id,
|
||||
checkpoint_type="company_runtime_suspended",
|
||||
if suspended is None:
|
||||
raise ServiceError(
|
||||
"company_runtime_suspend_failed",
|
||||
"Company runtime could not be suspended",
|
||||
{"runtime_session_id": target_info.get("runtime_session_id", "")},
|
||||
)
|
||||
for candidate in list(suspended.get("task_ids", []) or []):
|
||||
candidate_id = str(candidate or "").strip()
|
||||
if candidate_id and candidate_id not in affected_task_ids:
|
||||
affected_task_ids.append(candidate_id)
|
||||
self.context.stop_requested_task_ids.update(affected_task_ids)
|
||||
if self.context.cancel_session_tasks is not None:
|
||||
for tid in affected_task_ids:
|
||||
@@ -1009,8 +1065,7 @@ class SessionService:
|
||||
"stop_intent_id": stop_intent_id,
|
||||
"checkpoint_id": str((suspended or {}).get("checkpoint_id", "") or ""),
|
||||
"task_ids": affected_task_ids,
|
||||
"resume_parent_task_id": str(target_info.get("parent_task_id", "") or resolved_task_id),
|
||||
"resume_parent_session_id": str(target_info.get("parent_session_id", "") or resolved_session_id),
|
||||
"resume_parent_session_id": str(target_info.get("runtime_session_id", "") or resolved_session_id),
|
||||
}
|
||||
return ServiceResult(payload, [ServiceEvent("session_runtime_control", payload), ServiceEvent("session_updated", payload)])
|
||||
|
||||
@@ -1039,6 +1094,8 @@ class SessionService:
|
||||
project_id: str,
|
||||
task_id: str = "",
|
||||
session_id: str = "",
|
||||
runtime_session_id: str = "",
|
||||
checkpoint_id: str = "",
|
||||
target: str = "",
|
||||
content: str = "",
|
||||
) -> ServiceResult:
|
||||
@@ -1066,45 +1123,111 @@ class SessionService:
|
||||
default_payload=default_payload,
|
||||
)
|
||||
engine = await self.context.engine_for_project(project_id)
|
||||
exec_mode, company_profile = self.resolve_task_session_config(task)
|
||||
target_info = await self._resolve_company_runtime_target(engine, task) if exec_mode in {"company", "org", "custom"} else {
|
||||
try:
|
||||
company_target = await self._resolve_company_runtime_target(
|
||||
engine,
|
||||
task,
|
||||
runtime_session_id=runtime_session_id,
|
||||
checkpoint_id=checkpoint_id,
|
||||
)
|
||||
except ServiceError as exc:
|
||||
if exc.code != "company_runtime_identity_mismatch" or runtime_session_id or checkpoint_id:
|
||||
raise
|
||||
if is_company_runtime_task(task):
|
||||
raise
|
||||
company_target = None
|
||||
target_info = company_target or {
|
||||
"affected_task_ids": [resolved_task_id],
|
||||
"parent_task_id": resolved_task_id,
|
||||
"parent_session_id": resolved_session_id,
|
||||
"ui_anchor_task_id": resolved_task_id,
|
||||
"runtime_session_id": resolved_session_id,
|
||||
"config_task": task,
|
||||
"checkpoint": None,
|
||||
}
|
||||
affected_task_ids = list(target_info.get("affected_task_ids", []) or [resolved_task_id])
|
||||
await self._clear_company_runtime_stop_state(engine=engine, task_ids=affected_task_ids)
|
||||
message = str(content or "").strip() or "Resume the existing runtime."
|
||||
engine_mode = "company" if exec_mode == "company" else ("org" if exec_mode in {"org", "custom"} else "task")
|
||||
org_id = self.resolve_task_org_id(task) if engine_mode == "org" else ""
|
||||
config_task = target_info.get("config_task") or task
|
||||
config_exec_mode, config_company_profile = self.resolve_task_session_config(config_task)
|
||||
engine_mode = "company" if config_exec_mode == "company" else (
|
||||
"org" if config_exec_mode in {"org", "custom"} else "task"
|
||||
)
|
||||
if engine_mode in {"company", "org"}:
|
||||
checkpoint = target_info.get("checkpoint")
|
||||
if checkpoint is None:
|
||||
raise ServiceError("company_runtime_checkpoint_not_found", "No active company runtime checkpoint")
|
||||
if str(getattr(checkpoint, "status", "") or "").strip().lower() != "pending":
|
||||
raise ServiceError(
|
||||
"company_runtime_checkpoint_not_pending",
|
||||
"Company runtime checkpoint is not pending",
|
||||
{"checkpoint_id": str(getattr(checkpoint, "checkpoint_id", "") or "")},
|
||||
)
|
||||
else:
|
||||
checkpoint = None
|
||||
org_id = self.resolve_task_org_id(config_task) if engine_mode == "org" else ""
|
||||
message_metadata: dict[str, Any] = {"ui_force_resume": True}
|
||||
if checkpoint is not None:
|
||||
message_metadata.update({
|
||||
"response_to_checkpoint_id": str(getattr(checkpoint, "checkpoint_id", "") or ""),
|
||||
"response_to_checkpoint_type": str(getattr(checkpoint, "checkpoint_type", "") or ""),
|
||||
})
|
||||
response = await engine.process_message(
|
||||
message,
|
||||
project_id=self.context.normalize_project_id(project_id),
|
||||
session_id=str(target_info.get("parent_session_id", "") or resolved_session_id),
|
||||
session_id=str(target_info.get("runtime_session_id", "") or resolved_session_id),
|
||||
mode=engine_mode,
|
||||
org_id=org_id or None,
|
||||
company_profile=company_profile if engine_mode == "company" else None,
|
||||
preferred_agent=self.resolve_task_preferred_agent(task) if engine_mode == "task" else None,
|
||||
origin_task_id=str(target_info.get("parent_task_id", "") or resolved_task_id),
|
||||
message_metadata={"ui_force_resume": True},
|
||||
company_profile=config_company_profile if engine_mode == "company" else None,
|
||||
preferred_agent=self.resolve_task_preferred_agent(config_task) if engine_mode == "task" else None,
|
||||
origin_task_id=(str(target_info.get("ui_anchor_task_id", "") or "").strip() or None),
|
||||
message_metadata=message_metadata,
|
||||
)
|
||||
runtime_control_state = "idle"
|
||||
pending_runtime_checkpoint_id = ""
|
||||
if engine_mode in {"company", "org"}:
|
||||
refreshed_index = await load_company_runtime_identity_index(
|
||||
engine.store,
|
||||
self.context.normalize_project_id(project_id),
|
||||
)
|
||||
refreshed_identity = refreshed_index.resolve(
|
||||
runtime_session_id=str(
|
||||
target_info.get("runtime_session_id", "") or resolved_session_id
|
||||
),
|
||||
)
|
||||
if refreshed_identity is not None and refreshed_identity.pending_checkpoint_id:
|
||||
pending_runtime_checkpoint_id = refreshed_identity.pending_checkpoint_id
|
||||
runtime_control_state = (
|
||||
"suspended"
|
||||
if refreshed_identity.pending_checkpoint_status == "pending"
|
||||
else "resuming"
|
||||
)
|
||||
payload = {
|
||||
**default_payload,
|
||||
"status": "resuming",
|
||||
"runtime_control_state": "resuming",
|
||||
"can_resume": False,
|
||||
"status": runtime_control_state,
|
||||
"runtime_control_state": runtime_control_state,
|
||||
"can_resume": runtime_control_state == "suspended",
|
||||
"response": response,
|
||||
"task_ids": affected_task_ids,
|
||||
"resume_parent_task_id": str(target_info.get("parent_task_id", "") or resolved_task_id),
|
||||
"resume_parent_session_id": str(target_info.get("parent_session_id", "") or resolved_session_id),
|
||||
"resume_parent_session_id": str(target_info.get("runtime_session_id", "") or resolved_session_id),
|
||||
"pending_runtime_checkpoint_id": pending_runtime_checkpoint_id,
|
||||
}
|
||||
return ServiceResult(payload, [ServiceEvent("session_runtime_control", payload), ServiceEvent("session_updated", payload)])
|
||||
|
||||
async def resume(self, *, project_id: str, task_id: str = "", session_id: str = "", target: str = "", content: str = "") -> ServiceResult:
|
||||
async def resume(
|
||||
self,
|
||||
*,
|
||||
project_id: str,
|
||||
task_id: str = "",
|
||||
session_id: str = "",
|
||||
runtime_session_id: str = "",
|
||||
checkpoint_id: str = "",
|
||||
target: str = "",
|
||||
content: str = "",
|
||||
) -> ServiceResult:
|
||||
return await self.continue_run(
|
||||
project_id=project_id,
|
||||
task_id=task_id,
|
||||
session_id=session_id,
|
||||
runtime_session_id=runtime_session_id,
|
||||
checkpoint_id=checkpoint_id,
|
||||
target=target,
|
||||
content=content,
|
||||
)
|
||||
|
||||
@@ -39,6 +39,11 @@ from opc.layer2_organization.phase import (
|
||||
should_hide_work_item_from_company_kanban,
|
||||
verdict,
|
||||
)
|
||||
from opc.layer2_organization.company_runtime_identity import (
|
||||
ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES,
|
||||
COMPANY_RUNTIME_CHECKPOINT_TYPES,
|
||||
build_company_runtime_identity_index,
|
||||
)
|
||||
from opc.layer2_organization.work_item_context_view import WorkItemContextView
|
||||
from opc.layer2_organization.work_item_identity import (
|
||||
WORK_ITEM_PROJECTION_ID_KEY,
|
||||
@@ -1503,6 +1508,11 @@ def _primary_session_tasks_by_session_id(
|
||||
*,
|
||||
task_meta_map: dict[str, dict[str, Any]] | None = None,
|
||||
) -> tuple[dict[str, Any], list[str]]:
|
||||
identity_index = build_company_runtime_identity_index(tasks)
|
||||
company_identities = {
|
||||
identity.runtime_session_id: identity
|
||||
for identity in identity_index.identities
|
||||
}
|
||||
primary_tasks_by_session_id: dict[str, Any] = {}
|
||||
ordered_session_ids: list[str] = []
|
||||
for task in tasks:
|
||||
@@ -1513,6 +1523,19 @@ def _primary_session_tasks_by_session_id(
|
||||
if bool(task_meta.get("review_task", False)):
|
||||
continue
|
||||
session_id = str(getattr(task, "session_id", "") or "").strip()
|
||||
company_identity = company_identities.get(session_id)
|
||||
if company_identity is not None:
|
||||
anchor = identity_index.task(company_identity.ui_anchor_task_id)
|
||||
if anchor is not None and session_id not in primary_tasks_by_session_id:
|
||||
primary_tasks_by_session_id[session_id] = anchor
|
||||
ordered_session_ids.append(session_id)
|
||||
if anchor is not None:
|
||||
# A shared final-decider/work-item Task must never replace a
|
||||
# pure UI anchor that owns the same session id.
|
||||
continue
|
||||
# Without a pure anchor this scope has no primary chat container.
|
||||
# Never synthesize one from a role/work-item Task.
|
||||
continue
|
||||
if not session_id or _task_parent_session_link(task, task_meta):
|
||||
continue
|
||||
current = primary_tasks_by_session_id.get(session_id)
|
||||
@@ -1530,31 +1553,16 @@ def _primary_session_tasks_by_session_id(
|
||||
return primary_tasks_by_session_id, ordered_session_ids
|
||||
|
||||
|
||||
def _shared_role_identity_tasks_by_session_id(
|
||||
def _company_config_source_tasks_by_session_id(
|
||||
tasks: list[Any],
|
||||
*,
|
||||
task_meta_map: dict[str, dict[str, Any]] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
identity_tasks_by_session_id: dict[str, Any] = {}
|
||||
for task in tasks:
|
||||
task_id = str(getattr(task, "id", "") or "").strip()
|
||||
task_meta = (
|
||||
task_meta_map.get(task_id, {}) if task_meta_map is not None and task_id else _task_metadata(task)
|
||||
)
|
||||
session_id = _shared_role_session_key(task, task_meta)
|
||||
if not session_id:
|
||||
continue
|
||||
current = identity_tasks_by_session_id.get(session_id)
|
||||
if current is None:
|
||||
identity_tasks_by_session_id[session_id] = task
|
||||
continue
|
||||
current_id = str(getattr(current, "id", "") or "").strip()
|
||||
current_meta = (
|
||||
task_meta_map.get(current_id, {}) if task_meta_map is not None and current_id else _task_metadata(current)
|
||||
)
|
||||
if _session_representative_rank(task, task_meta) > _session_representative_rank(current, current_meta):
|
||||
identity_tasks_by_session_id[session_id] = task
|
||||
return identity_tasks_by_session_id
|
||||
identity_index = build_company_runtime_identity_index(tasks)
|
||||
return {
|
||||
identity.runtime_session_id: task
|
||||
for identity in identity_index.identities
|
||||
if identity.config_source_task_id
|
||||
and (task := identity_index.task(identity.config_source_task_id)) is not None
|
||||
}
|
||||
|
||||
|
||||
async def build_company_kanban_projection(
|
||||
@@ -2594,31 +2602,7 @@ async def _build_company_runtime_control_by_task(
|
||||
if not store:
|
||||
return {}
|
||||
|
||||
parent_task_by_session: dict[str, str] = {}
|
||||
tasks_by_parent_session: dict[str, list[Any]] = {}
|
||||
for task in tasks:
|
||||
metadata = dict(getattr(task, "metadata", {}) or {})
|
||||
mode = str(metadata.get("mode", "") or metadata.get("exec_mode", "") or "").strip().lower()
|
||||
is_company_runtime_task = bool(
|
||||
mode in {"company", "org", "custom"}
|
||||
or str(getattr(task, "parent_session_id", "") or "").strip()
|
||||
or metadata.get("company_profile")
|
||||
or metadata.get("company_work_item_plan")
|
||||
or metadata.get("work_item_runtime")
|
||||
or metadata.get("work_item_projection_id")
|
||||
)
|
||||
if not is_company_runtime_task:
|
||||
continue
|
||||
session_id = str(getattr(task, "session_id", "") or "").strip()
|
||||
parent_session_id = str(getattr(task, "parent_session_id", "") or "").strip()
|
||||
task_id = str(getattr(task, "id", "") or "").strip()
|
||||
if session_id and not parent_session_id:
|
||||
parent_task_by_session[session_id] = task_id
|
||||
runtime_parent_session_id = parent_session_id or session_id
|
||||
if runtime_parent_session_id:
|
||||
tasks_by_parent_session.setdefault(runtime_parent_session_id, []).append(task)
|
||||
|
||||
checkpoints_by_session: dict[str, Any] = {}
|
||||
checkpoints: list[Any] = []
|
||||
getter = getattr(store, "get_execution_checkpoints", None)
|
||||
if not callable(getter):
|
||||
getter = getattr(store, "get_pending_checkpoints", None)
|
||||
@@ -2626,29 +2610,25 @@ async def _build_company_runtime_control_by_task(
|
||||
try:
|
||||
kwargs = {
|
||||
"project_id": project_id,
|
||||
"checkpoint_types": ["company_runtime_suspended", "company_runtime_interrupted"],
|
||||
"checkpoint_types": sorted(COMPANY_RUNTIME_CHECKPOINT_TYPES),
|
||||
}
|
||||
if getattr(getter, "__name__", "") == "get_execution_checkpoints":
|
||||
kwargs["statuses"] = ["pending", "resuming"]
|
||||
kwargs["statuses"] = sorted(ACTIVE_COMPANY_RUNTIME_CHECKPOINT_STATUSES)
|
||||
checkpoints = await getter(**kwargs)
|
||||
for checkpoint in checkpoints:
|
||||
sid = str(getattr(checkpoint, "session_id", "") or "").strip()
|
||||
if sid and sid not in checkpoints_by_session:
|
||||
checkpoints_by_session[sid] = checkpoint
|
||||
except Exception:
|
||||
logger.opt(exception=True).debug("snapshot: failed to load company runtime checkpoints")
|
||||
checkpoints = []
|
||||
|
||||
identity_index = build_company_runtime_identity_index(tasks, checkpoints)
|
||||
|
||||
result: dict[str, dict[str, Any]] = {}
|
||||
for parent_session_id, group in tasks_by_parent_session.items():
|
||||
checkpoint = checkpoints_by_session.get(parent_session_id)
|
||||
parent_task_id = parent_task_by_session.get(parent_session_id, "")
|
||||
if not parent_task_id:
|
||||
for task in group:
|
||||
if not str(getattr(task, "parent_session_id", "") or "").strip():
|
||||
parent_task_id = str(getattr(task, "id", "") or "").strip()
|
||||
break
|
||||
if not parent_task_id and group:
|
||||
parent_task_id = str(getattr(group[0], "id", "") or "").strip()
|
||||
for identity in identity_index.identities:
|
||||
group = [
|
||||
task
|
||||
for task_id in identity.runtime_task_ids
|
||||
if (task := identity_index.task(task_id)) is not None
|
||||
]
|
||||
checkpoint = identity.checkpoint
|
||||
|
||||
def _task_status_value(task: Any) -> str:
|
||||
status = getattr(task, "status", "")
|
||||
@@ -2660,10 +2640,19 @@ async def _build_company_runtime_control_by_task(
|
||||
task for task in group
|
||||
if _task_status_value(task) not in {"done", "failed", "cancelled"}
|
||||
]
|
||||
has_running_task = any(
|
||||
_task_status_value(task) == "running"
|
||||
for task in non_terminal_group
|
||||
)
|
||||
# Persisted RUNNING is only a projection. The controller-local
|
||||
# execution registry is the sole proof that this process still owns a
|
||||
# coroutine capable of monitoring and persisting the run.
|
||||
runtime_is_live = getattr(engine, "_task_runtime_is_live", None)
|
||||
has_running_task = False
|
||||
if callable(runtime_is_live):
|
||||
for task in non_terminal_group:
|
||||
live_result = runtime_is_live(task)
|
||||
if inspect.isawaitable(live_result):
|
||||
live_result = await live_result
|
||||
if live_result is True:
|
||||
has_running_task = True
|
||||
break
|
||||
any_stop_in_progress = any(
|
||||
str((getattr(task, "metadata", {}) or {}).get("company_runtime_stop_state", "") or "").strip()
|
||||
in {"suspending", "suspended", "resuming_after_suspending"}
|
||||
@@ -2708,8 +2697,7 @@ async def _build_company_runtime_control_by_task(
|
||||
"runtime_control_state": state,
|
||||
"can_stop": state == "running",
|
||||
"can_resume": state == "suspended",
|
||||
"resume_parent_task_id": parent_task_id,
|
||||
"resume_parent_session_id": parent_session_id,
|
||||
"resume_parent_session_id": identity.runtime_session_id,
|
||||
"pending_runtime_checkpoint_id": pending_checkpoint_id,
|
||||
"stop_intent_id": str(checkpoint_payload.get("stop_intent_id", "") or ""),
|
||||
}
|
||||
@@ -3014,9 +3002,8 @@ async def build_project_index_sync(
|
||||
session_tasks,
|
||||
task_meta_map=task_meta_map,
|
||||
)
|
||||
shared_identity_tasks_by_session_id = _shared_role_identity_tasks_by_session_id(
|
||||
company_config_tasks_by_session_id = _company_config_source_tasks_by_session_id(
|
||||
session_tasks,
|
||||
task_meta_map=task_meta_map,
|
||||
)
|
||||
child_tasks_by_parent: dict[str, list[Any]] = {}
|
||||
for task in session_tasks:
|
||||
@@ -3087,11 +3074,12 @@ async def build_project_index_sync(
|
||||
representative_task = primary_tasks_by_session_id.get(session_id)
|
||||
representative_task_id = str(getattr(representative_task, "id", "") or "").strip()
|
||||
shared_session_id = _shared_role_session_key(t, t_meta)
|
||||
if shared_session_id and representative_task_id and representative_task_id != task_id:
|
||||
continue
|
||||
if shared_session_id:
|
||||
if not representative_task_id or representative_task_id != task_id:
|
||||
continue
|
||||
identity_task = t
|
||||
identity_meta = t_meta
|
||||
shared_identity_task = shared_identity_tasks_by_session_id.get(session_id)
|
||||
shared_identity_task = company_config_tasks_by_session_id.get(session_id)
|
||||
shared_identity_task_id = str(getattr(shared_identity_task, "id", "") or "").strip()
|
||||
if shared_identity_task_id and shared_identity_task_id != task_id:
|
||||
identity_task = shared_identity_task
|
||||
@@ -3569,9 +3557,8 @@ async def build_collab_sync(
|
||||
session_tasks,
|
||||
task_meta_map=task_meta_map,
|
||||
)
|
||||
shared_identity_tasks_by_session_id = _shared_role_identity_tasks_by_session_id(
|
||||
company_config_tasks_by_session_id = _company_config_source_tasks_by_session_id(
|
||||
session_tasks,
|
||||
task_meta_map=task_meta_map,
|
||||
)
|
||||
child_tasks_by_parent: dict[str, list[Any]] = {}
|
||||
for task in session_tasks:
|
||||
@@ -3616,11 +3603,15 @@ async def build_collab_sync(
|
||||
representative_task = primary_tasks_by_session_id.get(session_id)
|
||||
representative_task_id = str(getattr(representative_task, "id", "") or "").strip()
|
||||
shared_session_id = _shared_role_session_key(t, t_meta)
|
||||
if shared_session_id and representative_task_id and representative_task_id != str(getattr(t, "id", "") or "").strip():
|
||||
continue
|
||||
if shared_session_id:
|
||||
if (
|
||||
not representative_task_id
|
||||
or representative_task_id != str(getattr(t, "id", "") or "").strip()
|
||||
):
|
||||
continue
|
||||
identity_task = t
|
||||
identity_meta = t_meta
|
||||
shared_identity_task = shared_identity_tasks_by_session_id.get(session_id)
|
||||
shared_identity_task = company_config_tasks_by_session_id.get(session_id)
|
||||
shared_identity_task_id = str(getattr(shared_identity_task, "id", "") or "").strip()
|
||||
if shared_identity_task_id and shared_identity_task_id != str(getattr(t, "id", "") or "").strip():
|
||||
identity_task = shared_identity_task
|
||||
|
||||
@@ -51,8 +51,11 @@ class CompanyKanbanProjectionTests(unittest.IsolatedAsyncioTestCase):
|
||||
},
|
||||
)
|
||||
store = MagicMock()
|
||||
store.get_pending_checkpoints = AsyncMock(return_value=[])
|
||||
engine = SimpleNamespace(store=store)
|
||||
store.get_execution_checkpoints = AsyncMock(return_value=[])
|
||||
engine = SimpleNamespace(
|
||||
store=store,
|
||||
_task_runtime_is_live=AsyncMock(return_value=True),
|
||||
)
|
||||
|
||||
control = await _build_company_runtime_control_by_task(
|
||||
engine,
|
||||
@@ -95,8 +98,11 @@ class CompanyKanbanProjectionTests(unittest.IsolatedAsyncioTestCase):
|
||||
},
|
||||
)
|
||||
store = MagicMock()
|
||||
store.get_pending_checkpoints = AsyncMock(return_value=[])
|
||||
engine = SimpleNamespace(store=store)
|
||||
store.get_execution_checkpoints = AsyncMock(return_value=[])
|
||||
engine = SimpleNamespace(
|
||||
store=store,
|
||||
_task_runtime_is_live=AsyncMock(return_value=True),
|
||||
)
|
||||
|
||||
control = await _build_company_runtime_control_by_task(
|
||||
engine,
|
||||
@@ -106,8 +112,9 @@ class CompanyKanbanProjectionTests(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
self.assertEqual(control["child-task"]["runtime_control_state"], "running")
|
||||
self.assertTrue(control["child-task"]["can_stop"])
|
||||
engine._task_runtime_is_live.assert_awaited()
|
||||
|
||||
async def test_runtime_control_running_status_does_not_require_live_heartbeat(self) -> None:
|
||||
async def test_runtime_control_running_status_requires_controller_registry_ownership(self) -> None:
|
||||
created_at = datetime.now(timezone.utc)
|
||||
parent_task = SimpleNamespace(
|
||||
id="parent-task",
|
||||
@@ -122,7 +129,7 @@ class CompanyKanbanProjectionTests(unittest.IsolatedAsyncioTestCase):
|
||||
},
|
||||
)
|
||||
store = MagicMock()
|
||||
store.get_pending_checkpoints = AsyncMock(return_value=[])
|
||||
store.get_execution_checkpoints = AsyncMock(return_value=[])
|
||||
engine = SimpleNamespace(
|
||||
store=store,
|
||||
_task_runtime_is_live=AsyncMock(return_value=False),
|
||||
@@ -134,9 +141,9 @@ class CompanyKanbanProjectionTests(unittest.IsolatedAsyncioTestCase):
|
||||
"proj1",
|
||||
)
|
||||
|
||||
self.assertEqual(control["parent-task"]["runtime_control_state"], "running")
|
||||
self.assertTrue(control["parent-task"]["can_stop"])
|
||||
engine._task_runtime_is_live.assert_not_awaited()
|
||||
self.assertEqual(control["parent-task"]["runtime_control_state"], "idle")
|
||||
self.assertFalse(control["parent-task"]["can_stop"])
|
||||
engine._task_runtime_is_live.assert_awaited_once_with(parent_task)
|
||||
|
||||
async def test_runtime_control_treats_dispatch_hold_as_suspending_without_checkpoint(self) -> None:
|
||||
created_at = datetime.now(timezone.utc)
|
||||
@@ -1482,7 +1489,7 @@ class CollabSyncCompanyModeTests(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(session["company_profile"], "custom")
|
||||
self.assertEqual(session["preferred_agent"], "codex")
|
||||
|
||||
async def test_build_collab_sync_deduplicates_shared_role_sessions(self) -> None:
|
||||
async def test_build_collab_sync_does_not_promote_shared_role_session_without_ui_anchor(self) -> None:
|
||||
created_at = datetime.now(timezone.utc)
|
||||
shared_session_id = "root-session:role:cto"
|
||||
pending_task = SimpleNamespace(
|
||||
@@ -1519,6 +1526,7 @@ class CollabSyncCompanyModeTests(unittest.IsolatedAsyncioTestCase):
|
||||
engine = MagicMock()
|
||||
engine.store = MagicMock()
|
||||
engine.store.get_tasks = AsyncMock(return_value=[pending_task, running_task])
|
||||
engine.store.get_execution_checkpoints = AsyncMock(return_value=[])
|
||||
engine.project_id = "proj-shared"
|
||||
engine.llm = None
|
||||
|
||||
@@ -1586,10 +1594,7 @@ class CollabSyncCompanyModeTests(unittest.IsolatedAsyncioTestCase):
|
||||
)
|
||||
|
||||
sessions = result.get("sessions", [])
|
||||
self.assertEqual(len(sessions), 1)
|
||||
self.assertEqual(sessions[0]["project_id"], "proj-shared")
|
||||
self.assertEqual(sessions[0]["task_id"], "task-running")
|
||||
self.assertEqual(sessions[0]["session_id"], shared_session_id)
|
||||
self.assertEqual(sessions, [])
|
||||
|
||||
async def test_build_collab_sync_keeps_root_session_visible_when_final_decider_shares_session(self) -> None:
|
||||
created_at = datetime.now(timezone.utc)
|
||||
|
||||
+703
-565
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,549 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from functools import wraps
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from opc.core.active_task_runs import (
|
||||
ActiveTaskRunAdmissionClosed,
|
||||
ActiveTaskRunRegistry,
|
||||
)
|
||||
from opc.core.models import CompanyMemberSession, Task, TaskResult, TaskStatus
|
||||
from opc.engine import OPCEngine
|
||||
from opc.layer2_organization.company_mode import CompanyWorkItemExecutor
|
||||
from opc.layer2_organization.company_runtime_identity import is_company_runtime_task
|
||||
from opc.layer2_organization.org_work_item_planner import CompanyWorkItemRuntimePlan
|
||||
|
||||
|
||||
def _async_test(func):
|
||||
@wraps(func)
|
||||
def runner(*args, **kwargs):
|
||||
return asyncio.run(func(*args, **kwargs))
|
||||
|
||||
return runner
|
||||
|
||||
|
||||
def test_overlapping_attempts_remain_active_until_last_attempt_exits() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
first = registry.register("project-a", "task-1")
|
||||
second = registry.register("project-a", "task-1")
|
||||
|
||||
assert first != second
|
||||
assert registry.attempt_count("project-a", "task-1") == 2
|
||||
assert registry.is_active("project-a", "task-1")
|
||||
assert registry.unregister("project-a", "task-1", first)
|
||||
assert registry.is_active("project-a", "task-1")
|
||||
assert registry.unregister("project-a", "task-1", second)
|
||||
assert not registry.is_active("project-a", "task-1")
|
||||
|
||||
|
||||
def test_registry_isolates_projects_with_equal_task_ids() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
token = registry.register("project-a", "task-1")
|
||||
|
||||
assert registry.active_task_ids("project-a") == {"task-1"}
|
||||
assert registry.active_task_ids("project-b") == set()
|
||||
assert not registry.is_active("project-b", "task-1")
|
||||
assert registry.unregister("project-a", "task-1", token)
|
||||
|
||||
|
||||
def test_plain_child_task_is_not_classified_as_company_runtime_scope() -> None:
|
||||
task = Task(
|
||||
id="plain-task",
|
||||
title="Plain task",
|
||||
project_id="project-a",
|
||||
parent_session_id="parent-session",
|
||||
metadata={"mode": "task", "parent_session_id": "parent-session"},
|
||||
)
|
||||
|
||||
assert not is_company_runtime_task(task)
|
||||
|
||||
|
||||
def test_closing_admission_preserves_existing_attempts_and_rejects_new_ones() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
token = registry.register("project-a", "task-1")
|
||||
|
||||
registry.close_admission()
|
||||
|
||||
assert registry.admission_closed
|
||||
assert registry.active_task_ids("project-a") == {"task-1"}
|
||||
assert registry.is_active("project-a", "task-1")
|
||||
with pytest.raises(ActiveTaskRunAdmissionClosed):
|
||||
registry.register("project-a", "task-2")
|
||||
assert registry.unregister("project-a", "task-1", token)
|
||||
|
||||
|
||||
def test_closed_admission_allows_only_nested_live_driver_attempts() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
driver_token = registry.register("project-a", "driver-task")
|
||||
|
||||
with registry.bind_driver_attempt(driver_token):
|
||||
registry.close_admission()
|
||||
nested_token = registry.register("project-a", "claimed-child")
|
||||
assert registry.is_active("project-a", "claimed-child")
|
||||
registry.unregister("project-a", "claimed-child", nested_token)
|
||||
|
||||
with pytest.raises(ActiveTaskRunAdmissionClosed):
|
||||
registry.register("project-a", "new-ingress")
|
||||
registry.unregister("project-a", "driver-task", driver_token)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_shutdown_barrier_allows_only_reserved_handoff_to_register() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
handoff_token = registry.reserve_handoff()
|
||||
|
||||
with registry.bind_handoff(handoff_token):
|
||||
barrier = asyncio.create_task(
|
||||
registry.close_admission_and_wait_for_handoffs()
|
||||
)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert not barrier.done()
|
||||
attempt_token = registry.register("project-a", "task-1")
|
||||
|
||||
registry.release_handoff(handoff_token)
|
||||
await asyncio.wait_for(barrier, timeout=0.1)
|
||||
|
||||
# The handoff wait ends at real coroutine registration, not at the end of
|
||||
# that execution attempt.
|
||||
assert registry.is_active("project-a", "task-1")
|
||||
assert registry.pending_handoff_count == 0
|
||||
with pytest.raises(ActiveTaskRunAdmissionClosed):
|
||||
registry.register("project-a", "late-task")
|
||||
assert registry.unregister("project-a", "task-1", attempt_token)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_shutdown_barrier_drains_request_that_exits_before_registration() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
handoff_token = registry.reserve_handoff()
|
||||
barrier = asyncio.create_task(registry.close_admission_and_wait_for_handoffs())
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert not barrier.done()
|
||||
assert registry.release_handoff(handoff_token)
|
||||
await asyncio.wait_for(barrier, timeout=0.1)
|
||||
|
||||
assert registry.pending_handoff_count == 0
|
||||
assert registry.active_task_ids("project-a") == set()
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_revoked_handoff_cannot_block_shutdown_or_register_late() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
handoff_token = registry.reserve_handoff()
|
||||
|
||||
with registry.bind_handoff(handoff_token):
|
||||
assert registry.retain_current_handoff() == handoff_token
|
||||
registry.close_admission()
|
||||
assert registry.revoke_handoff(handoff_token)
|
||||
await asyncio.wait_for(
|
||||
registry.close_admission_and_wait_for_handoffs(),
|
||||
timeout=0.1,
|
||||
)
|
||||
with pytest.raises(ActiveTaskRunAdmissionClosed):
|
||||
registry.register("project-a", "late-task")
|
||||
|
||||
assert registry.pending_handoff_count == 0
|
||||
assert not registry.release_handoff(handoff_token)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_engine_turns_closed_admission_into_infrastructure_cancellation() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
registry.close_admission()
|
||||
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
|
||||
engine._run_task_once = AsyncMock()
|
||||
task = Task(
|
||||
id="late-task",
|
||||
title="Late task",
|
||||
project_id="project-a",
|
||||
status=TaskStatus.PENDING,
|
||||
)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await engine._execute_task(task)
|
||||
|
||||
engine._run_task_once.assert_not_awaited()
|
||||
assert registry.active_task_ids("project-a") == set()
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_task_liveness_uses_registry_only() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
|
||||
engine.store = SimpleNamespace(get_latest_external_session_for_task=AsyncMock())
|
||||
task = Task(
|
||||
id="task-1",
|
||||
title="Live task",
|
||||
project_id="project-a",
|
||||
status=TaskStatus.RUNNING,
|
||||
)
|
||||
|
||||
assert not await engine._task_runtime_is_live(task)
|
||||
engine.store.get_latest_external_session_for_task.assert_not_awaited()
|
||||
|
||||
token = registry.register("project-a", task.id)
|
||||
assert await engine._task_runtime_is_live(task)
|
||||
registry.unregister("project-a", task.id, token)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_project_delegate_receives_controller_registry() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
root = OPCEngine(project_id="project-a", active_task_run_registry=registry)
|
||||
root._initialized = True
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class FakeDelegate:
|
||||
def __init__(self, **kwargs: object) -> None:
|
||||
captured.update(kwargs)
|
||||
self.store = None
|
||||
|
||||
async def initialize(self) -> None:
|
||||
return None
|
||||
|
||||
with patch("opc.engine.OPCEngine", FakeDelegate):
|
||||
delegate = await root._get_project_delegate("project-b")
|
||||
|
||||
assert delegate is root._project_engine_delegates["project-b"]
|
||||
assert captured["active_task_run_registry"] is registry
|
||||
assert captured["owns_active_task_run_registry"] is False
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_shutdown_cancellation_does_not_write_business_cancelled() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
|
||||
engine._shutting_down = True
|
||||
engine.store = SimpleNamespace(
|
||||
is_ready=True,
|
||||
get_task=AsyncMock(),
|
||||
save_task=AsyncMock(),
|
||||
)
|
||||
engine._run_task_once = AsyncMock(side_effect=asyncio.CancelledError)
|
||||
task = Task(
|
||||
id="task-1",
|
||||
title="Interrupted task",
|
||||
project_id="project-a",
|
||||
status=TaskStatus.RUNNING,
|
||||
)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await engine._execute_task(task)
|
||||
|
||||
engine.store.get_task.assert_not_awaited()
|
||||
engine.store.save_task.assert_not_awaited()
|
||||
assert not registry.is_active("project-a", task.id)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_company_cancellation_never_synthesizes_hold_without_checkpoint() -> None:
|
||||
engine = OPCEngine(project_id="project-a")
|
||||
engine.store = SimpleNamespace(
|
||||
is_ready=True,
|
||||
get_task=AsyncMock(),
|
||||
save_task=AsyncMock(),
|
||||
)
|
||||
engine._run_task_once = AsyncMock(side_effect=asyncio.CancelledError)
|
||||
task = Task(
|
||||
id="company-task",
|
||||
title="Company task",
|
||||
project_id="project-a",
|
||||
status=TaskStatus.RUNNING,
|
||||
metadata={"work_item_runtime": True},
|
||||
)
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await engine._execute_task(task)
|
||||
|
||||
engine.store.get_task.assert_not_awaited()
|
||||
engine.store.save_task.assert_not_awaited()
|
||||
assert "company_runtime_suspended_at" not in task.metadata
|
||||
assert "last_stop_reason" not in task.metadata
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_suspended_checkpoint_discards_racing_task_completion() -> None:
|
||||
engine = OPCEngine(project_id="project-a")
|
||||
engine._shutting_down = False
|
||||
task = Task(
|
||||
id="company-task",
|
||||
title="Company task",
|
||||
project_id="project-a",
|
||||
status=TaskStatus.RUNNING,
|
||||
metadata={"work_item_runtime": True},
|
||||
)
|
||||
suspended = Task(
|
||||
id=task.id,
|
||||
title=task.title,
|
||||
project_id=task.project_id,
|
||||
status=TaskStatus.BLOCKED,
|
||||
metadata={
|
||||
"work_item_runtime": True,
|
||||
"dispatch_hold": "company_runtime_suspended",
|
||||
"company_runtime_stop_state": "suspended",
|
||||
},
|
||||
)
|
||||
engine.store = SimpleNamespace(
|
||||
get_task=AsyncMock(return_value=suspended),
|
||||
save_task=AsyncMock(),
|
||||
)
|
||||
engine._run_task_once = AsyncMock(
|
||||
return_value=TaskResult(status=TaskStatus.DONE, content="done", artifacts={})
|
||||
)
|
||||
engine._apply_runtime_state_to_task = MagicMock()
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await engine._execute_task(task)
|
||||
|
||||
engine.store.save_task.assert_not_awaited()
|
||||
engine._apply_runtime_state_to_task.assert_not_called()
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_attempt_stays_active_until_result_persistence_finishes() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
|
||||
save_started = asyncio.Event()
|
||||
allow_save = asyncio.Event()
|
||||
|
||||
async def blocked_save(_task: Task) -> None:
|
||||
save_started.set()
|
||||
await allow_save.wait()
|
||||
|
||||
engine.store = SimpleNamespace(
|
||||
get_task=AsyncMock(return_value=None),
|
||||
save_task=blocked_save,
|
||||
)
|
||||
engine._run_task_once = AsyncMock(
|
||||
return_value=TaskResult(status=TaskStatus.IDLE, content="done", artifacts={})
|
||||
)
|
||||
engine._apply_runtime_state_to_task = MagicMock()
|
||||
task = Task(
|
||||
id="persisting-task",
|
||||
title="Persisting task",
|
||||
project_id="project-a",
|
||||
status=TaskStatus.RUNNING,
|
||||
)
|
||||
|
||||
execution = asyncio.create_task(engine._execute_task(task))
|
||||
await save_started.wait()
|
||||
assert registry.is_active("project-a", task.id)
|
||||
allow_save.set()
|
||||
await execution
|
||||
assert not registry.is_active("project-a", task.id)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_claimed_work_item_ownership_covers_post_execution_finalize_gap() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
engine = OPCEngine(project_id="project-a", active_task_run_registry=registry)
|
||||
engine.store = SimpleNamespace(
|
||||
get_task=AsyncMock(return_value=None),
|
||||
save_task=AsyncMock(),
|
||||
)
|
||||
engine._run_task_once = AsyncMock(
|
||||
return_value=TaskResult(status=TaskStatus.IDLE, content="done", artifacts={})
|
||||
)
|
||||
engine._apply_runtime_state_to_task = MagicMock()
|
||||
task = Task(
|
||||
id="finalizing-work-item",
|
||||
title="Finalizing work item",
|
||||
project_id="project-a",
|
||||
parent_session_id="runtime-session",
|
||||
status=TaskStatus.RUNNING,
|
||||
metadata={"work_item_runtime": True},
|
||||
)
|
||||
inner_finished = asyncio.Event()
|
||||
allow_finalize = asyncio.Event()
|
||||
executor = object.__new__(CompanyWorkItemExecutor)
|
||||
executor.active_task_run_registry = registry
|
||||
|
||||
async def run_claimed(*_args: object, **_kwargs: object) -> TaskResult:
|
||||
result = await engine._execute_task(task)
|
||||
inner_finished.set()
|
||||
await allow_finalize.wait()
|
||||
return result
|
||||
|
||||
executor._run_claimed_work_item = run_claimed
|
||||
owned = executor._create_claimed_work_item_task(
|
||||
CompanyMemberSession(
|
||||
role_id="executor",
|
||||
seat_id="seat::executor",
|
||||
member_session_id="role-session",
|
||||
),
|
||||
task,
|
||||
{},
|
||||
)
|
||||
await inner_finished.wait()
|
||||
|
||||
assert registry.attempt_count("project-a", task.id) == 1
|
||||
allow_finalize.set()
|
||||
await owned
|
||||
assert not registry.is_active("project-a", task.id)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_work_item_claim_and_spawn_share_stop_scope_lock() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
executor = object.__new__(CompanyWorkItemExecutor)
|
||||
executor.active_task_run_registry = registry
|
||||
claim_entered = asyncio.Event()
|
||||
allow_claim = asyncio.Event()
|
||||
allow_child_exit = asyncio.Event()
|
||||
stop_acquired = asyncio.Event()
|
||||
task = Task(
|
||||
id="claimed-task",
|
||||
title="Claimed task",
|
||||
project_id="project-a",
|
||||
session_id="role-session",
|
||||
parent_session_id="runtime-session",
|
||||
metadata={"work_item_runtime": True},
|
||||
)
|
||||
member_session = CompanyMemberSession(
|
||||
role_id="executor",
|
||||
seat_id="seat::executor",
|
||||
member_session_id="role-session",
|
||||
)
|
||||
|
||||
async def claim_runnable_tasks(
|
||||
_tasks: list[Task],
|
||||
*,
|
||||
work_items: list[object],
|
||||
) -> list[tuple[CompanyMemberSession, Task]]:
|
||||
del work_items
|
||||
claim_entered.set()
|
||||
await allow_claim.wait()
|
||||
return [(member_session, task)]
|
||||
|
||||
async def run_claimed(*_args: object, **_kwargs: object) -> None:
|
||||
await allow_child_exit.wait()
|
||||
|
||||
executor.runtime = SimpleNamespace(
|
||||
claim_runnable_tasks=claim_runnable_tasks,
|
||||
)
|
||||
executor._run_claimed_work_item = run_claimed
|
||||
active: dict[asyncio.Task, tuple[CompanyMemberSession, Task]] = {}
|
||||
scheduled = asyncio.create_task(
|
||||
executor._claim_and_create_work_item_tasks([task], [], active)
|
||||
)
|
||||
await claim_entered.wait()
|
||||
|
||||
async def stop_scope() -> None:
|
||||
async with registry.scope_lock("project-a", "runtime-session"):
|
||||
assert registry.is_active("project-a", task.id)
|
||||
stop_acquired.set()
|
||||
|
||||
stopping = asyncio.create_task(stop_scope())
|
||||
await asyncio.sleep(0)
|
||||
assert not stop_acquired.is_set()
|
||||
|
||||
allow_claim.set()
|
||||
await scheduled
|
||||
await stopping
|
||||
assert len(active) == 1
|
||||
|
||||
allow_child_exit.set()
|
||||
await asyncio.gather(*active)
|
||||
assert not registry.is_active("project-a", task.id)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_company_executor_driver_ownership_covers_idle_scheduler_window() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
entered = asyncio.Event()
|
||||
allow_exit = asyncio.Event()
|
||||
executor = object.__new__(CompanyWorkItemExecutor)
|
||||
executor.active_task_run_registry = registry
|
||||
|
||||
async def idle_scheduler(
|
||||
_plan: CompanyWorkItemRuntimePlan,
|
||||
_tasks: list[Task],
|
||||
) -> str:
|
||||
entered.set()
|
||||
await allow_exit.wait()
|
||||
return "done"
|
||||
|
||||
executor._execute_multi_team_org = idle_scheduler
|
||||
task = Task(
|
||||
id="driver-task",
|
||||
title="Driver task",
|
||||
project_id="project-a",
|
||||
parent_session_id="runtime-session",
|
||||
metadata={"work_item_runtime": True},
|
||||
)
|
||||
execution = asyncio.create_task(
|
||||
executor.execute(CompanyWorkItemRuntimePlan(), [task])
|
||||
)
|
||||
await entered.wait()
|
||||
|
||||
assert registry.is_active("project-a", task.id)
|
||||
allow_exit.set()
|
||||
assert await execution == "done"
|
||||
assert not registry.is_active("project-a", task.id)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_borrowed_engine_shutdown_keeps_controller_registry_open() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
root_token = registry.register("project-a", "root-attempt")
|
||||
borrowed = OPCEngine(
|
||||
project_id="project-a",
|
||||
active_task_run_registry=registry,
|
||||
owns_active_task_run_registry=False,
|
||||
)
|
||||
|
||||
await borrowed.shutdown()
|
||||
|
||||
assert not registry.admission_closed
|
||||
assert registry.is_active("project-a", "root-attempt")
|
||||
next_token = registry.register("project-a", "next-attempt")
|
||||
registry.unregister("project-a", "next-attempt", next_token)
|
||||
registry.unregister("project-a", "root-attempt", root_token)
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_shutdown_preparation_includes_project_delegates() -> None:
|
||||
engine = OPCEngine(project_id="project-a")
|
||||
delegate_prepare = AsyncMock(
|
||||
return_value=[{"session_id": "delegate-session", "checkpoint_id": "checkpoint-1"}]
|
||||
)
|
||||
engine._project_engine_delegates["project-b"] = SimpleNamespace(
|
||||
prepare_active_company_runtimes_for_shutdown=delegate_prepare,
|
||||
)
|
||||
|
||||
prepared = await engine.prepare_active_company_runtimes_for_shutdown()
|
||||
|
||||
assert prepared == [{"session_id": "delegate-session", "checkpoint_id": "checkpoint-1"}]
|
||||
delegate_prepare.assert_awaited_once()
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_engine_shutdown_prepares_before_closing_subsystems() -> None:
|
||||
engine = OPCEngine(project_id="project-a")
|
||||
engine.prepare_active_company_runtimes_for_shutdown = AsyncMock(return_value=[])
|
||||
|
||||
await engine.shutdown()
|
||||
|
||||
engine.prepare_active_company_runtimes_for_shutdown.assert_awaited_once()
|
||||
|
||||
|
||||
@_async_test
|
||||
async def test_engine_shutdown_does_not_close_store_when_durable_prepare_fails() -> None:
|
||||
engine = OPCEngine(project_id="project-a")
|
||||
engine.prepare_active_company_runtimes_for_shutdown = AsyncMock(
|
||||
side_effect=RuntimeError("checkpoint failed")
|
||||
)
|
||||
engine.store = SimpleNamespace(close=AsyncMock())
|
||||
engine.message_bus.stop = MagicMock()
|
||||
|
||||
with pytest.raises(RuntimeError, match="checkpoint failed"):
|
||||
await engine.shutdown()
|
||||
|
||||
engine.message_bus.stop.assert_not_called()
|
||||
engine.store.close.assert_not_awaited()
|
||||
+264
-50
@@ -1116,6 +1116,34 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
self.assertEqual(calls, ["first", "second"])
|
||||
self.assertIn("Queued #1", console.export_text())
|
||||
|
||||
def test_chat_turn_controller_prepares_checkpoint_before_cancelling_active_turn(self) -> None:
|
||||
state, _engine = self._make_state()
|
||||
|
||||
async def _run() -> list[str]:
|
||||
order: list[str] = []
|
||||
started = asyncio.Event()
|
||||
|
||||
async def active_turn() -> None:
|
||||
started.set()
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
order.append("cancelled")
|
||||
|
||||
async def prepare() -> list[dict[str, Any]]:
|
||||
order.append("prepared")
|
||||
return []
|
||||
|
||||
state.engine.prepare_active_company_runtimes_for_shutdown = prepare
|
||||
controller = ChatTurnController(state)
|
||||
controller.active_task = asyncio.create_task(active_turn())
|
||||
await started.wait()
|
||||
|
||||
await controller.shutdown()
|
||||
return order
|
||||
|
||||
self.assertEqual(asyncio.run(_run()), ["prepared", "cancelled"])
|
||||
|
||||
def test_busy_slash_policy_allows_readonly_and_blocks_mutating_commands(self) -> None:
|
||||
self.assertEqual(_busy_slash_policy("kanban", []), BusyCommandPolicy.IMMEDIATE_READONLY)
|
||||
self.assertEqual(_busy_slash_policy("logs", ["task-1"]), BusyCommandPolicy.IMMEDIATE_READONLY)
|
||||
@@ -1168,6 +1196,17 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
"org_id": "quantum_harbor",
|
||||
"preferred_agent": "codex",
|
||||
})
|
||||
store.checkpoints = [
|
||||
SimpleNamespace(
|
||||
checkpoint_id="cp-org-interrupted",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id="task-1",
|
||||
session_id="sess-1",
|
||||
updated_at=datetime(2026, 5, 17, 12, 0),
|
||||
payload={},
|
||||
)
|
||||
]
|
||||
state, engine = self._make_state(store=store)
|
||||
state.mode = "company"
|
||||
state.company_profile = "corporate"
|
||||
@@ -1183,7 +1222,59 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
self.assertEqual(engine.calls[-1]["org_id"], "quantum_harbor")
|
||||
self.assertIsNone(engine.calls[-1]["company_profile"])
|
||||
self.assertEqual(engine.calls[-1]["preferred_agent"], "codex")
|
||||
self.assertEqual(engine.calls[-1]["message_metadata"], {"ui_force_resume": True})
|
||||
self.assertEqual(engine.calls[-1]["message_metadata"], {
|
||||
"ui_force_resume": True,
|
||||
"response_to_checkpoint_id": "cp-org-interrupted",
|
||||
"response_to_checkpoint_type": "company_runtime_interrupted",
|
||||
})
|
||||
|
||||
def test_company_continue_requires_a_durable_runtime_checkpoint(self) -> None:
|
||||
console = Console(record=True, force_terminal=False, width=120)
|
||||
store = self._Store()
|
||||
store.tasks[0].metadata.update({
|
||||
"exec_mode": "company",
|
||||
"company_profile": "corporate",
|
||||
})
|
||||
state, engine = self._make_state(store=store)
|
||||
|
||||
async def _run() -> None:
|
||||
await _handle_chat_slash_command(state, "/continue")
|
||||
|
||||
with patch("opc.cli.app.console", console):
|
||||
asyncio.run(_run())
|
||||
|
||||
self.assertEqual(engine.calls, [])
|
||||
self.assertIn("No suspended or interrupted company runtime", console.export_text())
|
||||
|
||||
def test_continue_slash_routes_to_durable_runtime_checkpoint(self) -> None:
|
||||
store = self._Store()
|
||||
store.checkpoints = [
|
||||
SimpleNamespace(
|
||||
checkpoint_id="cp-interrupted",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id="task-1",
|
||||
session_id="sess-1",
|
||||
updated_at=datetime(2026, 5, 17, 12, 0),
|
||||
payload={},
|
||||
)
|
||||
]
|
||||
state, engine = self._make_state(store=store)
|
||||
state.mode = "company"
|
||||
state.runtime_control_state = "suspended"
|
||||
|
||||
async def _run() -> None:
|
||||
await _handle_chat_slash_command(state, "/continue")
|
||||
|
||||
asyncio.run(_run())
|
||||
|
||||
self.assertEqual(engine.calls[-1]["session_id"], "sess-1")
|
||||
self.assertEqual(engine.calls[-1]["message_metadata"], {
|
||||
"ui_force_resume": True,
|
||||
"response_to_checkpoint_id": "cp-interrupted",
|
||||
"response_to_checkpoint_type": "company_runtime_interrupted",
|
||||
})
|
||||
self.assertEqual(state.runtime_control_checkpoint_id, "cp-interrupted")
|
||||
|
||||
def test_plain_message_after_stop_routes_to_suspend_checkpoint(self) -> None:
|
||||
store = self._Store()
|
||||
@@ -1209,7 +1300,127 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
|
||||
self.assertEqual(engine.calls[-1]["message_metadata"]["response_to_checkpoint_id"], "cp-suspend")
|
||||
self.assertEqual(engine.calls[-1]["message_metadata"]["response_to_checkpoint_type"], "company_runtime_suspended")
|
||||
self.assertEqual(state.runtime_control_state, "running")
|
||||
self.assertEqual(state.runtime_control_state, "suspended")
|
||||
|
||||
def test_plain_message_from_company_child_uses_root_runtime_checkpoint(self) -> None:
|
||||
store = self._Store()
|
||||
now = datetime(2026, 5, 17, 12, 0)
|
||||
store.tasks[0].metadata.update({
|
||||
"exec_mode": "company",
|
||||
"mode": "company",
|
||||
"company_profile": "corporate",
|
||||
})
|
||||
store.tasks[0].parent_session_id = ""
|
||||
store.tasks.append(SimpleNamespace(
|
||||
id="worker-child",
|
||||
title="Worker",
|
||||
description="Worker turn",
|
||||
status=TaskStatus.BLOCKED,
|
||||
priority=3,
|
||||
assigned_to="worker",
|
||||
session_id="sess-1:role:worker",
|
||||
parent_session_id="sess-1",
|
||||
project_id="demo",
|
||||
created_at=now,
|
||||
tags=[],
|
||||
result={},
|
||||
linked_work_item_id="worker-item",
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"work_item_runtime": True,
|
||||
"work_item_projection_id": "worker",
|
||||
"company_runtime_root_session_id": "sess-1",
|
||||
},
|
||||
context_snapshot={},
|
||||
))
|
||||
store.checkpoints = [SimpleNamespace(
|
||||
checkpoint_id="cp-child-interrupted",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id="task-1",
|
||||
session_id="sess-1",
|
||||
project_id="demo",
|
||||
updated_at=now,
|
||||
payload={"parent_session_id": "sess-1"},
|
||||
)]
|
||||
state, engine = self._make_state(store=store)
|
||||
state.session_id = "sess-1:role:worker"
|
||||
# The selected child channel may have left the CLI's ambient mode in
|
||||
# task mode. Runtime config, not ambient state, owns resumed execution.
|
||||
state.mode = "task"
|
||||
|
||||
asyncio.run(_process_interactive_chat_message(state, "revise and continue"))
|
||||
|
||||
call = engine.calls[-1]
|
||||
self.assertEqual(call["session_id"], "sess-1")
|
||||
self.assertEqual(call["origin_task_id"], "task-1")
|
||||
self.assertEqual(call["mode"], "company")
|
||||
self.assertEqual(call["company_profile"], "corporate")
|
||||
self.assertEqual(call["message_metadata"], {
|
||||
"response_to_checkpoint_id": "cp-child-interrupted",
|
||||
"response_to_checkpoint_type": "company_runtime_interrupted",
|
||||
})
|
||||
|
||||
def test_plain_message_during_resuming_checkpoint_fails_closed(self) -> None:
|
||||
console = Console(record=True, force_terminal=False, width=140)
|
||||
store = self._Store()
|
||||
store.tasks[0].metadata.update({
|
||||
"exec_mode": "company",
|
||||
"mode": "company",
|
||||
"company_profile": "corporate",
|
||||
})
|
||||
store.checkpoints = [SimpleNamespace(
|
||||
checkpoint_id="cp-resuming",
|
||||
checkpoint_type="company_runtime_suspended",
|
||||
status="resuming",
|
||||
task_id="task-1",
|
||||
session_id="sess-1",
|
||||
project_id="demo",
|
||||
updated_at=datetime(2026, 5, 17, 12, 0),
|
||||
payload={"parent_session_id": "sess-1"},
|
||||
)]
|
||||
state, engine = self._make_state(store=store)
|
||||
state.mode = "company"
|
||||
|
||||
with patch("opc.cli.app.console", console):
|
||||
asyncio.run(_process_interactive_chat_message(state, "continue again"))
|
||||
|
||||
self.assertEqual(engine.calls, [])
|
||||
self.assertIn("checkpoint is resuming", console.export_text())
|
||||
|
||||
def test_explicit_cli_runtime_checkpoint_mismatch_fails_closed(self) -> None:
|
||||
console = Console(record=True, force_terminal=False, width=140)
|
||||
store = self._Store()
|
||||
store.tasks[0].metadata.update({
|
||||
"exec_mode": "company",
|
||||
"mode": "company",
|
||||
"company_profile": "corporate",
|
||||
})
|
||||
store.checkpoints = [SimpleNamespace(
|
||||
checkpoint_id="cp-current",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id="task-1",
|
||||
session_id="sess-1",
|
||||
project_id="demo",
|
||||
updated_at=datetime(2026, 5, 17, 12, 0),
|
||||
payload={"parent_session_id": "sess-1"},
|
||||
)]
|
||||
state, engine = self._make_state(store=store)
|
||||
state.mode = "company"
|
||||
|
||||
with patch("opc.cli.app.console", console):
|
||||
asyncio.run(_process_interactive_chat_message(
|
||||
state,
|
||||
"continue stale",
|
||||
message_metadata={
|
||||
"response_to_checkpoint_id": "cp-stale",
|
||||
"response_to_checkpoint_type": "company_runtime_interrupted",
|
||||
},
|
||||
))
|
||||
|
||||
self.assertEqual(engine.calls, [])
|
||||
self.assertIn("checkpoint identity mismatch", console.export_text())
|
||||
|
||||
def test_session_resume_still_switches_session(self) -> None:
|
||||
state, engine = self._make_state(store=self._Store())
|
||||
@@ -1287,8 +1498,6 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
self.assertEqual(output["payload"]["checkpoint_id"], "cp-stop")
|
||||
self.assertEqual(output["cancelled"], ["task-company"])
|
||||
self.assertEqual(output["suspend_calls"][0]["session_id"], "sess-company")
|
||||
self.assertEqual(output["task"].metadata["dispatch_hold"], "company_runtime_suspended")
|
||||
self.assertEqual(output["task"].metadata["company_runtime_stop_state"], "suspended")
|
||||
|
||||
def test_session_service_continue_uses_force_resume_metadata(self) -> None:
|
||||
class Store:
|
||||
@@ -1305,10 +1514,17 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"company_profile": "corporate",
|
||||
"dispatch_hold": "company_runtime_suspended",
|
||||
"company_runtime_stop_state": "suspended",
|
||||
},
|
||||
)
|
||||
self.checkpoint = SimpleNamespace(
|
||||
checkpoint_id="cp-company",
|
||||
checkpoint_type="company_runtime_suspended",
|
||||
status="pending",
|
||||
project_id="demo",
|
||||
session_id="sess-company",
|
||||
task_id="task-company",
|
||||
payload={},
|
||||
)
|
||||
|
||||
async def get_task(self, task_id: str):
|
||||
return self.task if task_id == self.task.id else None
|
||||
@@ -1319,6 +1535,9 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
async def get_session(self, session_id: str):
|
||||
return SimpleNamespace(session_id=session_id, project_id="demo")
|
||||
|
||||
async def get_execution_checkpoints(self, **_kwargs):
|
||||
return [self.checkpoint]
|
||||
|
||||
async def save_task(self, task):
|
||||
self.task = task
|
||||
|
||||
@@ -1331,7 +1550,12 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
|
||||
engine = SimpleNamespace(project_id="demo", store=Store(), process_message=process_message)
|
||||
context = OfficeServiceContext(engine=engine, agent_store=None, chat_store=None, event_adapter=None)
|
||||
result = await SessionService(context).continue_run(project_id="demo", target="sess-company")
|
||||
result = await SessionService(context).continue_run(
|
||||
project_id="demo",
|
||||
target="sess-company",
|
||||
runtime_session_id="sess-company",
|
||||
checkpoint_id="cp-company",
|
||||
)
|
||||
self.assertEqual(result.payload["response"], "resumed")
|
||||
return calls
|
||||
|
||||
@@ -1340,7 +1564,11 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
self.assertEqual(calls[-1]["content"], "Resume the existing runtime.")
|
||||
self.assertEqual(calls[-1]["session_id"], "sess-company")
|
||||
self.assertEqual(calls[-1]["mode"], "company")
|
||||
self.assertEqual(calls[-1]["message_metadata"], {"ui_force_resume": True})
|
||||
self.assertEqual(calls[-1]["message_metadata"], {
|
||||
"ui_force_resume": True,
|
||||
"response_to_checkpoint_id": "cp-company",
|
||||
"response_to_checkpoint_type": "company_runtime_suspended",
|
||||
})
|
||||
|
||||
def test_session_service_continue_preserves_custom_org_id(self) -> None:
|
||||
class Store:
|
||||
@@ -1359,10 +1587,17 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
"company_profile": "custom",
|
||||
"org_id": "quantum_harbor",
|
||||
"preferred_agent": "codex",
|
||||
"dispatch_hold": "company_runtime_suspended",
|
||||
"company_runtime_stop_state": "suspended",
|
||||
},
|
||||
)
|
||||
self.checkpoint = SimpleNamespace(
|
||||
checkpoint_id="cp-org",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
project_id="demo",
|
||||
session_id="sess-org",
|
||||
task_id="task-org",
|
||||
payload={},
|
||||
)
|
||||
|
||||
async def get_task(self, task_id: str):
|
||||
return self.task if task_id == self.task.id else None
|
||||
@@ -1373,6 +1608,9 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
async def get_session(self, session_id: str):
|
||||
return SimpleNamespace(session_id=session_id, project_id="demo")
|
||||
|
||||
async def get_execution_checkpoints(self, **_kwargs):
|
||||
return [self.checkpoint]
|
||||
|
||||
async def save_task(self, task):
|
||||
self.task = task
|
||||
|
||||
@@ -1385,7 +1623,12 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
|
||||
engine = SimpleNamespace(project_id="demo", store=Store(), process_message=process_message)
|
||||
context = OfficeServiceContext(engine=engine, agent_store=None, chat_store=None, event_adapter=None)
|
||||
result = await SessionService(context).continue_run(project_id="demo", target="sess-org")
|
||||
result = await SessionService(context).continue_run(
|
||||
project_id="demo",
|
||||
target="sess-org",
|
||||
runtime_session_id="sess-org",
|
||||
checkpoint_id="cp-org",
|
||||
)
|
||||
self.assertEqual(result.payload["response"], "resumed")
|
||||
return calls
|
||||
|
||||
@@ -1394,7 +1637,11 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
self.assertEqual(calls[-1]["mode"], "org")
|
||||
self.assertEqual(calls[-1]["org_id"], "quantum_harbor")
|
||||
self.assertIsNone(calls[-1]["company_profile"])
|
||||
self.assertEqual(calls[-1]["message_metadata"], {"ui_force_resume": True})
|
||||
self.assertEqual(calls[-1]["message_metadata"], {
|
||||
"ui_force_resume": True,
|
||||
"response_to_checkpoint_id": "cp-org",
|
||||
"response_to_checkpoint_type": "company_runtime_interrupted",
|
||||
})
|
||||
|
||||
def test_queue_slash_lists_and_drops_queued_prompts(self) -> None:
|
||||
console = Console(record=True, force_terminal=False, width=160)
|
||||
@@ -2648,49 +2895,16 @@ class CliSlashCommandTests(unittest.TestCase):
|
||||
self.assertIn("Pending Checkpoints", rendered)
|
||||
self.assertIn("cp-1", rendered)
|
||||
|
||||
def test_recover_slash_lists_and_resumes_interrupted_runtime(self) -> None:
|
||||
def test_legacy_recover_slash_is_not_registered(self) -> None:
|
||||
console = Console(record=True, force_terminal=False, width=200)
|
||||
state, _engine = self._make_state(store=self._Store())
|
||||
resume_calls: list[str] = []
|
||||
|
||||
class _FakeRecoveryManager:
|
||||
async def get_status(self):
|
||||
return SimpleNamespace(
|
||||
interrupted=[
|
||||
SimpleNamespace(
|
||||
parent_task_id="parent-task",
|
||||
parent_session_id="parent-session",
|
||||
title="Interrupted Runtime",
|
||||
profile="corporate",
|
||||
interrupted_at="2026-05-03T12:00:00",
|
||||
work_items=[
|
||||
SimpleNamespace(projection_id="wi-1", interrupted=True),
|
||||
SimpleNamespace(projection_id="wi-2", interrupted=False),
|
||||
],
|
||||
)
|
||||
],
|
||||
active_recoveries=[],
|
||||
)
|
||||
|
||||
async def resume(self, parent_task_id: str):
|
||||
resume_calls.append(parent_task_id)
|
||||
if parent_task_id == "parent-task":
|
||||
return {"ok": True, "resumed_work_item_projection_ids": ["wi-1"]}
|
||||
return {"ok": False, "error": "not_found"}
|
||||
|
||||
async def _run() -> None:
|
||||
await _handle_chat_slash_command(state, "/recover")
|
||||
await _handle_chat_slash_command(state, "/recover resume parent-task")
|
||||
await _handle_chat_slash_command(state, "/recover resume cp-1")
|
||||
|
||||
with patch("opc.cli.app.console", console), patch("opc.cli.app._get_chat_recovery_manager", return_value=_FakeRecoveryManager()):
|
||||
asyncio.run(_run())
|
||||
with patch("opc.cli.app.console", console):
|
||||
asyncio.run(_handle_chat_slash_command(state, "/recover"))
|
||||
|
||||
rendered = console.export_text()
|
||||
self.assertIn("Interrupted Runtime", rendered)
|
||||
self.assertIn("Recovery started for parent-task", rendered)
|
||||
self.assertIn("Checkpoint cp-1 is not resumed directly", rendered)
|
||||
self.assertEqual(resume_calls, ["parent-task", "cp-1"])
|
||||
self.assertIn("Unknown command: /recover", rendered)
|
||||
self.assertNotIn("Interrupted Company Runtimes", rendered)
|
||||
|
||||
def test_logs_slash_renders_task_and_session_runtime_details(self) -> None:
|
||||
console = Console(record=True, force_terminal=False, width=220)
|
||||
|
||||
@@ -527,6 +527,25 @@ class CompanyCollaborationTests(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
await runtime.bootstrap([task])
|
||||
runtime.enqueue_runnable_work_items([work_item], task_by_work_item_id={"work-item-1": task})
|
||||
|
||||
async def claim_work_item(*_args: object, **kwargs: object) -> DelegationWorkItem:
|
||||
role_session_id = str(kwargs["role_runtime_session_id"])
|
||||
work_item.phase = Phase.RUNNING
|
||||
work_item.role_runtime_session_id = role_session_id
|
||||
work_item.claimed_by_role_runtime_session_id = role_session_id
|
||||
work_item.claimed_by_seat_id = str(kwargs.get("seat_id", ""))
|
||||
work_item.metadata = {
|
||||
**dict(work_item.metadata or {}),
|
||||
"claimed_by_role_session_id": role_session_id,
|
||||
"claimed_task_id": str(kwargs["task_id"]),
|
||||
}
|
||||
return work_item
|
||||
|
||||
runtime.store = SimpleNamespace(
|
||||
is_ready=True,
|
||||
claim_delegation_work_item_if_dispatchable=claim_work_item,
|
||||
save_delegation_role_session=AsyncMock(),
|
||||
)
|
||||
claims = await runtime.claim_runnable_tasks([task], work_items=[work_item])
|
||||
|
||||
self.assertEqual(len(claims), 1)
|
||||
@@ -534,6 +553,61 @@ class CompanyCollaborationTests(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(claimed_task.id, task.id)
|
||||
self.assertEqual(claimed_session.member_session_id, "role-session::proj1::root-a::executor::backend-architect")
|
||||
|
||||
async def test_company_runtime_does_not_spawn_after_atomic_claim_loses_to_hold(self) -> None:
|
||||
runtime = CompanyRuntime(
|
||||
org_engine=DummyOrgEngine(),
|
||||
communication=DummyRuntimeCommunication(),
|
||||
)
|
||||
task = Task(
|
||||
id="held-work-item-task",
|
||||
title="Held Work Item",
|
||||
session_id="root-a",
|
||||
parent_session_id="root-a",
|
||||
assigned_to="executor",
|
||||
status=TaskStatus.PENDING,
|
||||
project_id="proj1",
|
||||
metadata={
|
||||
"work_item_projection_id": "held_execution",
|
||||
"work_item_role_id": "executor",
|
||||
"employee_assignment": {
|
||||
"employee_id": "backend-architect",
|
||||
"role_id": "executor",
|
||||
},
|
||||
},
|
||||
)
|
||||
set_linked_work_item_id(task, "held-work-item")
|
||||
work_item = DelegationWorkItem(
|
||||
work_item_id="held-work-item",
|
||||
run_id="run-1",
|
||||
cell_id="cell-1",
|
||||
role_id="executor",
|
||||
projection_id="held_execution",
|
||||
phase=Phase.READY,
|
||||
)
|
||||
held_after_race = DelegationWorkItem(
|
||||
**{
|
||||
**work_item.__dict__,
|
||||
"metadata": {"dispatch_hold": "company_runtime_suspended"},
|
||||
}
|
||||
)
|
||||
claim = AsyncMock(return_value=None)
|
||||
runtime.store = SimpleNamespace(
|
||||
is_ready=True,
|
||||
claim_delegation_work_item_if_dispatchable=claim,
|
||||
get_delegation_work_item=AsyncMock(return_value=held_after_race),
|
||||
)
|
||||
await runtime.bootstrap([task])
|
||||
runtime.enqueue_runnable_work_items(
|
||||
[work_item],
|
||||
task_by_work_item_id={work_item.work_item_id: task},
|
||||
)
|
||||
|
||||
claims = await runtime.claim_runnable_tasks([task], work_items=[work_item])
|
||||
|
||||
self.assertEqual(claims, [])
|
||||
self.assertNotIn(work_item.work_item_id, runtime._claimed_work_item_ids)
|
||||
claim.assert_awaited_once()
|
||||
|
||||
async def test_company_runtime_does_not_double_claim_same_work_item(self) -> None:
|
||||
runtime = CompanyRuntime(
|
||||
org_engine=DummyOrgEngine(),
|
||||
@@ -2722,9 +2796,11 @@ class CompanyCollaborationTests(unittest.IsolatedAsyncioTestCase):
|
||||
engine.project_id = "proj1"
|
||||
engine.store = store
|
||||
engine.company_executor = DummyExecutor()
|
||||
attempt_token = engine._active_task_run_registry.register("proj1", running_task.id)
|
||||
|
||||
response = await engine._maybe_resume_existing_company_runtime("缁х画", "sess-parent-live")
|
||||
refreshed = await store.get_task(running_task.id)
|
||||
engine._active_task_run_registry.unregister("proj1", running_task.id, attempt_token)
|
||||
|
||||
self.assertIn("already in progress", response)
|
||||
self.assertEqual(refreshed.status, TaskStatus.RUNNING)
|
||||
|
||||
@@ -80,7 +80,6 @@ class SharedRoleSessionExecutionTests(unittest.IsolatedAsyncioTestCase):
|
||||
record_child_session_result=AsyncMock(),
|
||||
record_task_completion_async=AsyncMock(),
|
||||
)
|
||||
engine._active_task_runs = set()
|
||||
engine._run_task_once = AsyncMock(
|
||||
return_value=TaskResult(status=TaskStatus.DONE, content="done", artifacts={})
|
||||
)
|
||||
@@ -106,4 +105,3 @@ class SharedRoleSessionExecutionTests(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
engine.memory.record_assistant_turn.assert_awaited_once()
|
||||
engine.memory.record_child_session_result.assert_not_awaited()
|
||||
|
||||
|
||||
@@ -0,0 +1,410 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime, timedelta
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
from opc.core.models import ExecutionCheckpoint, Task, TaskStatus
|
||||
from opc.layer2_organization.company_runtime_identity import (
|
||||
build_company_runtime_identity_index,
|
||||
)
|
||||
from opc.plugins.office_ui.services.models import ServiceError
|
||||
from opc.plugins.office_ui.snapshot_builder import (
|
||||
_build_company_runtime_control_by_task,
|
||||
_primary_session_tasks_by_session_id,
|
||||
)
|
||||
from opc.plugins.office_ui.ws_handler import WSHandler
|
||||
|
||||
|
||||
def _runtime_records() -> tuple[list[Task], ExecutionCheckpoint]:
|
||||
runtime_session_id = "runtime-session"
|
||||
anchor = Task(
|
||||
id="ui-anchor",
|
||||
project_id="project-a",
|
||||
session_id=runtime_session_id,
|
||||
status=TaskStatus.CANCELLED,
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"mode": "company",
|
||||
"company_profile": "corporate",
|
||||
},
|
||||
created_at=datetime.now() - timedelta(minutes=3),
|
||||
)
|
||||
final_decider = Task(
|
||||
id="final-decider",
|
||||
project_id="project-a",
|
||||
session_id=runtime_session_id,
|
||||
parent_session_id=runtime_session_id,
|
||||
status=TaskStatus.CANCELLED,
|
||||
linked_work_item_id="work-item-root",
|
||||
metadata={
|
||||
"mode": "company",
|
||||
"work_item_runtime": True,
|
||||
"work_item_projection_id": "root",
|
||||
"shared_role_session": True,
|
||||
"shared_role_id": "ceo",
|
||||
"company_runtime_root_session_id": runtime_session_id,
|
||||
},
|
||||
created_at=datetime.now() - timedelta(minutes=2),
|
||||
)
|
||||
child = Task(
|
||||
id="worker",
|
||||
project_id="project-a",
|
||||
session_id=f"{runtime_session_id}:role:worker",
|
||||
parent_session_id=runtime_session_id,
|
||||
status=TaskStatus.BLOCKED,
|
||||
linked_work_item_id="work-item-worker",
|
||||
metadata={
|
||||
"mode": "company",
|
||||
"work_item_runtime": True,
|
||||
"work_item_projection_id": "worker",
|
||||
"shared_role_session": True,
|
||||
"company_runtime_root_session_id": runtime_session_id,
|
||||
},
|
||||
created_at=datetime.now() - timedelta(minutes=1),
|
||||
)
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-1",
|
||||
project_id="project-a",
|
||||
session_id=runtime_session_id,
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id=final_decider.id,
|
||||
payload={"parent_session_id": runtime_session_id},
|
||||
)
|
||||
return [anchor, final_decider, child], checkpoint
|
||||
|
||||
|
||||
def test_identity_is_session_first_and_never_selects_shared_final_decider_as_ui_anchor() -> None:
|
||||
tasks, checkpoint = _runtime_records()
|
||||
index = build_company_runtime_identity_index(tasks, [checkpoint])
|
||||
|
||||
identity = index.resolve(
|
||||
task_id="final-decider",
|
||||
runtime_session_id="runtime-session",
|
||||
checkpoint_id="checkpoint-1",
|
||||
)
|
||||
|
||||
assert identity is not None
|
||||
assert identity.runtime_session_id == "runtime-session"
|
||||
assert identity.ui_anchor_task_id == "ui-anchor"
|
||||
assert identity.config_source_task_id == "ui-anchor"
|
||||
assert identity.runtime_task_ids == ("ui-anchor", "final-decider", "worker")
|
||||
assert identity.pending_checkpoint_id == "checkpoint-1"
|
||||
assert identity.resumable is True
|
||||
assert index.resolve(task_session_id="runtime-session:role:worker") == identity
|
||||
assert index.resolve(task_id="worker", runtime_session_id="other-session") is None
|
||||
assert index.resolve(task_id="ui-anchor", checkpoint_id="other-checkpoint") is None
|
||||
|
||||
|
||||
def test_config_source_uses_configured_scope_task_when_ui_anchor_has_no_config() -> None:
|
||||
tasks, _checkpoint = _runtime_records()
|
||||
tasks[0].metadata = {}
|
||||
tasks[1].metadata.update({"exec_mode": "org", "company_profile": "custom", "org_id": "studio"})
|
||||
|
||||
identity = build_company_runtime_identity_index(tasks).resolve(
|
||||
task_id="ui-anchor",
|
||||
)
|
||||
|
||||
assert identity is not None
|
||||
assert identity.ui_anchor_task_id == "ui-anchor"
|
||||
assert identity.config_source_task_id == "final-decider"
|
||||
|
||||
|
||||
def test_snapshot_session_representative_uses_canonical_ui_anchor() -> None:
|
||||
tasks, _checkpoint = _runtime_records()
|
||||
primary, ordered = _primary_session_tasks_by_session_id(
|
||||
[tasks[1], tasks[0], tasks[2]],
|
||||
)
|
||||
|
||||
assert ordered[0] == "runtime-session"
|
||||
assert primary["runtime-session"].id == "ui-anchor"
|
||||
|
||||
|
||||
def test_runtime_without_ui_anchor_never_promotes_shared_work_item() -> None:
|
||||
tasks, checkpoint = _runtime_records()
|
||||
shared_final = tasks[1]
|
||||
shared_final.parent_session_id = None
|
||||
index = build_company_runtime_identity_index([shared_final, tasks[2]], [checkpoint])
|
||||
|
||||
identity = index.resolve(runtime_session_id="runtime-session")
|
||||
|
||||
assert identity is not None
|
||||
assert identity.ui_anchor_task_id == ""
|
||||
assert identity.config_source_task_id == "final-decider"
|
||||
primary, _ordered = _primary_session_tasks_by_session_id(
|
||||
[shared_final, tasks[2]],
|
||||
)
|
||||
assert "runtime-session" not in primary
|
||||
|
||||
|
||||
def test_snapshot_projects_checkpoint_control_to_cancelled_anchor_without_task_resume_identity() -> None:
|
||||
tasks, checkpoint = _runtime_records()
|
||||
|
||||
class Store:
|
||||
async def get_execution_checkpoints(self, **_kwargs):
|
||||
return [checkpoint]
|
||||
|
||||
engine = SimpleNamespace(store=Store())
|
||||
control = asyncio.run(_build_company_runtime_control_by_task(engine, tasks, "project-a"))
|
||||
|
||||
assert control["ui-anchor"]["runtime_control_state"] == "suspended"
|
||||
assert control["ui-anchor"]["can_resume"] is True
|
||||
assert control["ui-anchor"]["resume_parent_session_id"] == "runtime-session"
|
||||
assert control["ui-anchor"]["pending_runtime_checkpoint_id"] == "checkpoint-1"
|
||||
assert "resume_parent_task_id" not in control["ui-anchor"]
|
||||
|
||||
|
||||
def test_service_error_transport_fields_cannot_be_overridden() -> None:
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
sent: list[dict] = []
|
||||
|
||||
async def send_ack(_ws, ok=True, **payload):
|
||||
sent.append({"ok": ok, **payload})
|
||||
|
||||
handler._send_ack = send_ack
|
||||
error = ServiceError(
|
||||
"actual_code",
|
||||
"actual message",
|
||||
{"ok": True, "code": "wrong", "error": "wrong", "detail": "kept"},
|
||||
)
|
||||
|
||||
asyncio.run(handler._send_service_error(object(), error, action="test_action"))
|
||||
|
||||
assert sent == [{
|
||||
"ok": False,
|
||||
"detail": "kept",
|
||||
"error": "actual message",
|
||||
"code": "actual_code",
|
||||
"action": "test_action",
|
||||
}]
|
||||
|
||||
|
||||
def test_removed_recovery_action_receives_normal_unknown_message_ack() -> None:
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler._shutting_down = False
|
||||
handler._active_message_tasks = set()
|
||||
handler._send_ack = AsyncMock()
|
||||
ws = object()
|
||||
|
||||
asyncio.run(handler._route_message(ws, json.dumps({"type": "recovery_action"})))
|
||||
|
||||
handler._send_ack.assert_awaited_once_with(
|
||||
ws,
|
||||
ok=False,
|
||||
error="unknown_message_type",
|
||||
action="recovery_action",
|
||||
)
|
||||
|
||||
|
||||
def test_work_item_chat_resume_uses_canonical_ui_anchor_as_engine_origin() -> None:
|
||||
async def scenario() -> None:
|
||||
tasks, checkpoint = _runtime_records()
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler._exec_mode = "task"
|
||||
handler._company_profile = "corporate"
|
||||
handler._shutting_down = False
|
||||
handler._active_runtime_children = {}
|
||||
handler._session_to_task = {}
|
||||
handler._task_bg_context = {}
|
||||
handler._company_suspend_reply_locks = {"runtime-session": asyncio.Lock()}
|
||||
handler.chat_store = None
|
||||
handler._set_company_runtime_control = AsyncMock()
|
||||
handler._normalize_session_exec_mode = MagicMock(return_value="task")
|
||||
handler._normalize_session_company_profile = MagicMock(return_value="corporate")
|
||||
handler._resolve_task_session_config = MagicMock(
|
||||
return_value=("company", "corporate")
|
||||
)
|
||||
handler._resolve_task_org_id = MagicMock(return_value="")
|
||||
handler._extract_checkpoint_metadata = AsyncMock(return_value=None)
|
||||
handler._sync_task_transcript_messages = AsyncMock()
|
||||
handler.on_kanban_changed = AsyncMock()
|
||||
handler._flush_progress = AsyncMock()
|
||||
run_engine = SimpleNamespace(
|
||||
project_id="project-a",
|
||||
process_message=AsyncMock(return_value="resumed"),
|
||||
)
|
||||
target = {
|
||||
"ui_anchor_task_id": "ui-anchor",
|
||||
"config_task": tasks[1],
|
||||
}
|
||||
|
||||
await handler._process_company_suspend_reply(
|
||||
ui_task_id="final-decider",
|
||||
runtime_session_id="runtime-session",
|
||||
content="continue",
|
||||
attachment_refs=None,
|
||||
message_metadata={"ui_force_resume": True},
|
||||
user_message_id=None,
|
||||
user_message_created_at=None,
|
||||
run_engine=run_engine,
|
||||
run_project_id="project-a",
|
||||
target=target,
|
||||
checkpoint=checkpoint,
|
||||
lock=handler._company_suspend_reply_locks["runtime-session"],
|
||||
)
|
||||
|
||||
call = run_engine.process_message.await_args
|
||||
assert call.kwargs["session_id"] == "runtime-session"
|
||||
assert call.kwargs["origin_task_id"] == "ui-anchor"
|
||||
assert handler._session_to_task["runtime-session"] == "ui-anchor"
|
||||
handler.on_kanban_changed.assert_awaited_once_with(engine=run_engine)
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_delivery_feedback_rejects_missing_canonical_identity_without_first_task_fallback() -> None:
|
||||
async def scenario() -> None:
|
||||
tasks, checkpoint = _runtime_records()
|
||||
|
||||
class Store:
|
||||
is_ready = True
|
||||
|
||||
async def get_tasks(self, **_kwargs):
|
||||
# A shared final-decider deliberately precedes the UI anchor;
|
||||
# legacy first-match routing selected the wrong Task here.
|
||||
return [tasks[1], tasks[0], tasks[2]]
|
||||
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler._resolve_company_runtime_target = AsyncMock(return_value=None)
|
||||
engine = SimpleNamespace(store=Store())
|
||||
|
||||
target = await handler._company_delivery_feedback_parent_target(
|
||||
task_id="final-decider",
|
||||
waiting_task_id="worker",
|
||||
waiting_task=tasks[2],
|
||||
checkpoint=checkpoint,
|
||||
payload={"parent_session_id": "runtime-session"},
|
||||
engine=engine,
|
||||
)
|
||||
|
||||
assert target == {"parent_task_id": "", "parent_session_id": ""}
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_delivery_feedback_route_consumes_missing_identity_instead_of_running_work_item() -> None:
|
||||
async def scenario() -> None:
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.chat_store = None
|
||||
handler._company_delivery_feedback_reply_locks = {}
|
||||
handler._load_execution_checkpoint_for_reply = AsyncMock(return_value=SimpleNamespace(
|
||||
checkpoint_id="feedback-1",
|
||||
checkpoint_type="company_delivery_feedback",
|
||||
status="pending",
|
||||
task_id="worker",
|
||||
session_id="runtime-session",
|
||||
payload={"waiting_task_id": "worker", "parent_session_id": "runtime-session"},
|
||||
))
|
||||
handler._company_delivery_feedback_parent_target = AsyncMock(return_value={
|
||||
"parent_task_id": "",
|
||||
"parent_session_id": "",
|
||||
})
|
||||
handler._track_session = MagicMock()
|
||||
engine = SimpleNamespace(store=SimpleNamespace(is_ready=True))
|
||||
|
||||
handled = await handler._route_company_delivery_feedback_reply_if_pending(
|
||||
task_id="worker",
|
||||
content="looks good",
|
||||
session_id="runtime-session:worker",
|
||||
task=SimpleNamespace(id="worker"),
|
||||
attachment_refs=None,
|
||||
message_metadata={
|
||||
"response_to_checkpoint_id": "feedback-1",
|
||||
"response_to_checkpoint_type": "company_delivery_feedback",
|
||||
},
|
||||
user_message_id="message-1",
|
||||
user_message_created_at=None,
|
||||
run_engine=engine,
|
||||
run_project_id="project-a",
|
||||
reply_channel_id="session:worker",
|
||||
)
|
||||
|
||||
assert handled is True
|
||||
handler._track_session.assert_not_called()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_suspend_reply_identity_mismatch_and_resuming_checkpoint_fail_closed() -> None:
|
||||
async def scenario() -> None:
|
||||
tasks, checkpoint = _runtime_records()
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.chat_store = None
|
||||
handler._company_stop_finalize_tasks = {}
|
||||
handler._company_suspend_reply_locks = {}
|
||||
handler._track = MagicMock()
|
||||
engine = SimpleNamespace(project_id="project-a")
|
||||
target = {
|
||||
"runtime_session_id": "runtime-session",
|
||||
"checkpoint": checkpoint,
|
||||
}
|
||||
|
||||
handler._resolve_company_runtime_target = AsyncMock(
|
||||
side_effect=[target, None],
|
||||
)
|
||||
mismatched = await handler._route_company_suspend_reply_if_pending(
|
||||
task_id="worker",
|
||||
content="continue",
|
||||
session_id="runtime-session:role:worker",
|
||||
task=tasks[2],
|
||||
attachment_refs=None,
|
||||
message_metadata={
|
||||
"response_to_checkpoint_id": "wrong-checkpoint",
|
||||
"response_to_checkpoint_type": "company_runtime_interrupted",
|
||||
},
|
||||
user_message_id=None,
|
||||
user_message_created_at=None,
|
||||
run_engine=engine,
|
||||
run_project_id="project-a",
|
||||
)
|
||||
assert mismatched is True
|
||||
handler._track.assert_not_called()
|
||||
|
||||
checkpoint.status = "resuming"
|
||||
handler._resolve_company_runtime_target = AsyncMock(return_value=target)
|
||||
resuming = await handler._route_company_suspend_reply_if_pending(
|
||||
task_id="worker",
|
||||
content="continue again",
|
||||
session_id="runtime-session:role:worker",
|
||||
task=tasks[2],
|
||||
attachment_refs=None,
|
||||
message_metadata=None,
|
||||
user_message_id=None,
|
||||
user_message_created_at=None,
|
||||
run_engine=engine,
|
||||
run_project_id="project-a",
|
||||
)
|
||||
assert resuming is True
|
||||
handler._track.assert_not_called()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_escalation_control_uses_durable_anchor_not_transient_progress_maps() -> None:
|
||||
async def scenario() -> None:
|
||||
tasks, checkpoint = _runtime_records()
|
||||
|
||||
class Store:
|
||||
async def get_task(self, task_id: str):
|
||||
return next((task for task in tasks if task.id == task_id), None)
|
||||
|
||||
async def get_tasks(self, **_kwargs):
|
||||
return tasks
|
||||
|
||||
async def get_execution_checkpoints(self, **_kwargs):
|
||||
return [checkpoint]
|
||||
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.engine = SimpleNamespace(project_id="project-a", store=Store())
|
||||
handler._active_runtime_children = {"worker": "wrong-parent"}
|
||||
handler._session_to_task = {"runtime-session": "wrong-parent"}
|
||||
handler._ui_task_aliases = {}
|
||||
|
||||
resolved = await handler._resolve_escalation_session_task_id("worker")
|
||||
|
||||
assert resolved == "ui-anchor"
|
||||
|
||||
asyncio.run(scenario())
|
||||
@@ -74,6 +74,19 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
|
||||
external_provider_session_id: str = "provider-session-1",
|
||||
) -> tuple[CompanyWorkItemRuntimePlan, Task]:
|
||||
plan = self._plan(profile)
|
||||
await store.save_task(
|
||||
Task(
|
||||
id=f"ui-anchor-{parent_session_id}",
|
||||
title="Company chat",
|
||||
session_id=parent_session_id,
|
||||
project_id="proj1",
|
||||
status=TaskStatus.IDLE,
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"company_profile": profile,
|
||||
},
|
||||
)
|
||||
)
|
||||
await store.save_delegation_work_item(
|
||||
DelegationWorkItem(
|
||||
work_item_id=work_item_id,
|
||||
@@ -1445,7 +1458,7 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertNotIn("external_resume_session_id", resumed_task.metadata)
|
||||
self.assertEqual(resumed_task.metadata["external_resume_fallback"], "context_replay")
|
||||
|
||||
async def test_company_runtime_checkpoint_resolves_before_long_execute(self) -> None:
|
||||
async def test_company_runtime_checkpoint_stays_resuming_during_long_execute(self) -> None:
|
||||
store = await self._store()
|
||||
_, task = await self._seed_runtime(store)
|
||||
engine = self._engine(store)
|
||||
@@ -1472,7 +1485,7 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
|
||||
)
|
||||
captured["resuming_count_during_execute"] = len(resuming)
|
||||
captured["resolved_count_during_execute"] = len(resolved)
|
||||
captured["resume_state_during_execute"] = resolved[0].payload.get("resume_state") if resolved else ""
|
||||
captured["resume_state_during_execute"] = resuming[0].payload.get("resume_state") if resuming else ""
|
||||
return "runtime resumed"
|
||||
|
||||
engine.company_executor = DummyCompanyExecutor()
|
||||
@@ -1489,9 +1502,9 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
|
||||
statuses=["resolved"],
|
||||
)
|
||||
|
||||
self.assertEqual(captured["resuming_count_during_execute"], 0)
|
||||
self.assertEqual(captured["resolved_count_during_execute"], 1)
|
||||
self.assertEqual(captured["resume_state_during_execute"], "handoff_complete")
|
||||
self.assertEqual(captured["resuming_count_during_execute"], 1)
|
||||
self.assertEqual(captured["resolved_count_during_execute"], 0)
|
||||
self.assertEqual(captured["resume_state_during_execute"], "resuming")
|
||||
self.assertEqual(len(resolved), 1)
|
||||
self.assertEqual(resolved[0].payload.get("resume_state"), "handoff_complete")
|
||||
|
||||
@@ -1537,10 +1550,24 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
|
||||
checkpoint_types=["company_runtime_suspended"],
|
||||
statuses=["resuming"],
|
||||
)
|
||||
refreshed_task = await store.get_task(task.id)
|
||||
refreshed_item = await store.get_delegation_work_item("work-item-1")
|
||||
|
||||
self.assertEqual(len(pending), 1)
|
||||
self.assertEqual(resuming, [])
|
||||
self.assertEqual(pending[0].payload.get("resume_state"), "failed_before_handoff")
|
||||
assert refreshed_task is not None
|
||||
assert refreshed_item is not None
|
||||
self.assertEqual(
|
||||
refreshed_task.metadata.get("dispatch_hold"),
|
||||
"company_runtime_suspended",
|
||||
)
|
||||
self.assertEqual(
|
||||
refreshed_item.metadata.get("dispatch_hold"),
|
||||
"company_runtime_suspended",
|
||||
)
|
||||
self.assertEqual(refreshed_item.claimed_by_role_runtime_session_id, "")
|
||||
self.assertEqual(refreshed_item.claimed_by_seat_id, "")
|
||||
|
||||
async def test_suspend_is_parent_session_idempotent(self) -> None:
|
||||
store = await self._store()
|
||||
@@ -1831,22 +1858,16 @@ class CompanyRuntimeSuspendResumeTests(unittest.IsolatedAsyncioTestCase):
|
||||
async def test_continue_clears_parent_runtime_stop_marker(self) -> None:
|
||||
store = await self._store()
|
||||
_, task = await self._seed_runtime(store)
|
||||
parent = Task(
|
||||
id="parent-task",
|
||||
title="Parent company runtime",
|
||||
session_id="sess-parent",
|
||||
parent_session_id="",
|
||||
status=TaskStatus.RUNNING,
|
||||
project_id="proj1",
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"company_profile": "corporate",
|
||||
"company_runtime_stop_state": "suspended",
|
||||
"company_runtime_stop_intent_id": "intent-1",
|
||||
"company_runtime_stop_marked_at": "2026-04-29T11:02:40",
|
||||
"company_runtime_suspended_at": "2026-04-29T11:02:40",
|
||||
},
|
||||
)
|
||||
parent = await store.get_task("ui-anchor-sess-parent")
|
||||
assert parent is not None
|
||||
parent.status = TaskStatus.RUNNING
|
||||
parent.metadata = {
|
||||
**dict(parent.metadata or {}),
|
||||
"company_runtime_stop_state": "suspended",
|
||||
"company_runtime_stop_intent_id": "intent-1",
|
||||
"company_runtime_stop_marked_at": "2026-04-29T11:02:40",
|
||||
"company_runtime_suspended_at": "2026-04-29T11:02:40",
|
||||
}
|
||||
await store.save_task(parent)
|
||||
engine = self._engine(store)
|
||||
await engine.suspend_company_runtime(
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from opc import engine as engine_module
|
||||
from opc.engine import OPCEngine
|
||||
|
||||
|
||||
class EnginePidProbeTests(unittest.TestCase):
|
||||
def test_current_process_is_running(self) -> None:
|
||||
self.assertTrue(OPCEngine._pid_is_running(os.getpid()))
|
||||
|
||||
def test_posix_probe_treats_unexpected_oserror_as_not_running(self) -> None:
|
||||
with patch.object(engine_module.os, "name", "posix"), patch.object(
|
||||
engine_module.os,
|
||||
"kill",
|
||||
side_effect=OSError("platform probe failed"),
|
||||
):
|
||||
self.assertFalse(OPCEngine._pid_is_running(12345))
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,295 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import timedelta
|
||||
|
||||
from opc.core.models import ExecutionCheckpoint, Task, TaskStatus
|
||||
from opc.database.store import OPCStore
|
||||
|
||||
|
||||
def test_checkpoint_compare_and_set_has_one_winner_across_store_connections(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
async def scenario() -> None:
|
||||
db_path = tmp_path / "tasks.db"
|
||||
first_store = OPCStore(db_path)
|
||||
second_store = OPCStore(db_path)
|
||||
await first_store.initialize()
|
||||
await second_store.initialize()
|
||||
try:
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-1",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id="runtime-task",
|
||||
payload={"reason": "service_restart"},
|
||||
)
|
||||
await first_store.save_execution_checkpoint(checkpoint)
|
||||
|
||||
start = asyncio.Event()
|
||||
payloads = [
|
||||
{"reason": "service_restart", "claimed_by": "office"},
|
||||
{"reason": "service_restart", "claimed_by": "cli"},
|
||||
]
|
||||
|
||||
async def claim(store: OPCStore, payload: dict[str, str]) -> bool:
|
||||
await start.wait()
|
||||
return await store.compare_and_set_execution_checkpoint(
|
||||
checkpoint.checkpoint_id,
|
||||
expected_statuses={"pending"},
|
||||
status="resuming",
|
||||
payload=payload,
|
||||
)
|
||||
|
||||
claims = [
|
||||
asyncio.create_task(claim(first_store, payloads[0])),
|
||||
asyncio.create_task(claim(second_store, payloads[1])),
|
||||
]
|
||||
start.set()
|
||||
results = await asyncio.gather(*claims)
|
||||
|
||||
assert results.count(True) == 1
|
||||
assert results.count(False) == 1
|
||||
winner = results.index(True)
|
||||
rows = await first_store.get_execution_checkpoints(
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
)
|
||||
assert len(rows) == 1
|
||||
assert rows[0].status == "resuming"
|
||||
assert rows[0].payload == payloads[winner]
|
||||
finally:
|
||||
await second_store.close()
|
||||
await first_store.close()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_checkpoint_get_or_create_has_one_active_row_across_store_connections(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
async def scenario() -> None:
|
||||
db_path = tmp_path / "tasks.db"
|
||||
first_store = OPCStore(db_path)
|
||||
second_store = OPCStore(db_path)
|
||||
await first_store.initialize()
|
||||
await second_store.initialize()
|
||||
try:
|
||||
checkpoint_types = {
|
||||
"company_runtime_suspended",
|
||||
"company_runtime_interrupted",
|
||||
}
|
||||
candidates = [
|
||||
ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-office",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
task_id="runtime-task",
|
||||
payload={"creator": "office"},
|
||||
),
|
||||
ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-cli",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_suspended",
|
||||
task_id="runtime-task",
|
||||
payload={"creator": "cli"},
|
||||
),
|
||||
]
|
||||
start = asyncio.Event()
|
||||
|
||||
async def create(
|
||||
store: OPCStore,
|
||||
candidate: ExecutionCheckpoint,
|
||||
) -> tuple[ExecutionCheckpoint, bool]:
|
||||
await start.wait()
|
||||
return await store.get_or_create_active_execution_checkpoint(
|
||||
candidate,
|
||||
checkpoint_types=checkpoint_types,
|
||||
)
|
||||
|
||||
attempts = [
|
||||
asyncio.create_task(create(first_store, candidates[0])),
|
||||
asyncio.create_task(create(second_store, candidates[1])),
|
||||
]
|
||||
start.set()
|
||||
results = await asyncio.gather(*attempts)
|
||||
|
||||
assert [created for _, created in results].count(True) == 1
|
||||
assert [created for _, created in results].count(False) == 1
|
||||
assert len({row.checkpoint_id for row, _ in results}) == 1
|
||||
|
||||
active = await first_store.get_execution_checkpoints(
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_types=list(checkpoint_types),
|
||||
statuses=["pending", "resuming"],
|
||||
)
|
||||
assert len(active) == 1
|
||||
assert active[0].checkpoint_id == results[0][0].checkpoint_id
|
||||
finally:
|
||||
await second_store.close()
|
||||
await first_store.close()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_checkpoint_get_or_create_normalizes_historical_active_duplicates(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
async def scenario() -> None:
|
||||
store = OPCStore(tmp_path / "tasks.db")
|
||||
await store.initialize()
|
||||
try:
|
||||
older = ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-older",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="resuming",
|
||||
payload={"created": "older"},
|
||||
)
|
||||
newer = ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-newer",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_suspended",
|
||||
payload={"created": "newer"},
|
||||
)
|
||||
newer.updated_at = older.updated_at + timedelta(microseconds=1)
|
||||
await store.save_execution_checkpoint(older)
|
||||
await store.save_execution_checkpoint(newer)
|
||||
|
||||
winner, created = await store.get_or_create_active_execution_checkpoint(
|
||||
ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-unused",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
),
|
||||
checkpoint_types={
|
||||
"company_runtime_suspended",
|
||||
"company_runtime_interrupted",
|
||||
},
|
||||
)
|
||||
|
||||
assert created is False
|
||||
assert winner.checkpoint_id == "checkpoint-newer"
|
||||
active = await store.get_execution_checkpoints(
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
statuses=["pending", "resuming"],
|
||||
)
|
||||
assert [row.checkpoint_id for row in active] == ["checkpoint-newer"]
|
||||
all_rows = await store.get_execution_checkpoints(
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
)
|
||||
by_id = {row.checkpoint_id: row for row in all_rows}
|
||||
assert by_id["checkpoint-older"].status == "superseded"
|
||||
assert (
|
||||
by_id["checkpoint-older"].payload["superseded_by_checkpoint_id"]
|
||||
== "checkpoint-newer"
|
||||
)
|
||||
finally:
|
||||
await store.close()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_checkpoint_completion_and_cancelled_anchor_reopen_are_atomic(tmp_path) -> None:
|
||||
async def scenario() -> None:
|
||||
store = OPCStore(tmp_path / "tasks.db")
|
||||
await store.initialize()
|
||||
try:
|
||||
anchor = Task(
|
||||
id="ui-anchor",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
title="Company chat",
|
||||
status=TaskStatus.CANCELLED,
|
||||
execution_lock=True,
|
||||
)
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-1",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="resuming",
|
||||
payload={"ui_anchor_task_id": anchor.id},
|
||||
)
|
||||
await store.save_task(anchor)
|
||||
await store.save_execution_checkpoint(checkpoint)
|
||||
|
||||
completed = await store.complete_execution_checkpoint_and_reopen_ui_anchor(
|
||||
checkpoint.checkpoint_id,
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
expected_status="resuming",
|
||||
status="resolved",
|
||||
payload={"resume_state": "handoff_complete"},
|
||||
ui_anchor_task_id=anchor.id,
|
||||
)
|
||||
|
||||
assert completed is True
|
||||
assert (await store.get_task(anchor.id)).status == TaskStatus.IDLE
|
||||
rows = await store.get_execution_checkpoints(
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
)
|
||||
assert rows[0].status == "resolved"
|
||||
assert rows[0].payload == {"resume_state": "handoff_complete"}
|
||||
finally:
|
||||
await store.close()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_checkpoint_completion_does_not_reopen_anchor_after_stop_wins(tmp_path) -> None:
|
||||
async def scenario() -> None:
|
||||
store = OPCStore(tmp_path / "tasks.db")
|
||||
await store.initialize()
|
||||
try:
|
||||
anchor = Task(
|
||||
id="ui-anchor",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
title="Company chat",
|
||||
status=TaskStatus.CANCELLED,
|
||||
)
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-1",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
payload={"resume_state": "interrupted"},
|
||||
)
|
||||
await store.save_task(anchor)
|
||||
await store.save_execution_checkpoint(checkpoint)
|
||||
|
||||
completed = await store.complete_execution_checkpoint_and_reopen_ui_anchor(
|
||||
checkpoint.checkpoint_id,
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
expected_status="resuming",
|
||||
status="resolved",
|
||||
payload={"resume_state": "handoff_complete"},
|
||||
ui_anchor_task_id=anchor.id,
|
||||
)
|
||||
|
||||
assert completed is False
|
||||
assert (await store.get_task(anchor.id)).status == TaskStatus.CANCELLED
|
||||
rows = await store.get_execution_checkpoints(
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
)
|
||||
assert rows[0].status == "pending"
|
||||
assert rows[0].payload == {"resume_state": "interrupted"}
|
||||
finally:
|
||||
await store.close()
|
||||
|
||||
asyncio.run(scenario())
|
||||
@@ -0,0 +1,380 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from opc.core.active_task_runs import ActiveTaskRunRegistry
|
||||
from opc.core.models import ExecutionCheckpoint, Task
|
||||
from opc.plugins.office_ui.ws_handler import WSHandler
|
||||
|
||||
|
||||
def test_ws_shutdown_checkpoints_before_cancelling_and_awaiting_sessions() -> None:
|
||||
async def scenario() -> None:
|
||||
events: list[str] = []
|
||||
started = asyncio.Event()
|
||||
|
||||
async def execution() -> None:
|
||||
started.set()
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
events.append("execution_finally")
|
||||
|
||||
async def prepare() -> list[dict]:
|
||||
events.append("checkpoint")
|
||||
assert not execution_task.done()
|
||||
return []
|
||||
|
||||
execution_task = asyncio.create_task(execution())
|
||||
await started.wait()
|
||||
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.engine = SimpleNamespace()
|
||||
handler._root_engine = SimpleNamespace(
|
||||
prepare_active_company_runtimes_for_shutdown=prepare,
|
||||
)
|
||||
handler._shutting_down = False
|
||||
handler._progress_flush_task = None
|
||||
handler._clients = set()
|
||||
handler._active_message_tasks = set()
|
||||
handler._background_tasks = {execution_task}
|
||||
handler._task_bg_context = {execution_task: {"task_id": "runtime-task"}}
|
||||
handler._task_bg_map = {"runtime-task": {execution_task}}
|
||||
|
||||
await handler.shutdown(timeout=1.0)
|
||||
|
||||
assert events == ["checkpoint", "execution_finally"]
|
||||
assert execution_task.done()
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_ws_shutdown_checkpoint_failure_does_not_cancel_execution_or_close_the_gap() -> None:
|
||||
async def scenario() -> None:
|
||||
released = asyncio.Event()
|
||||
|
||||
async def execution() -> None:
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
released.set()
|
||||
|
||||
execution_task = asyncio.create_task(execution())
|
||||
await asyncio.sleep(0)
|
||||
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.engine = SimpleNamespace()
|
||||
handler._root_engine = SimpleNamespace(
|
||||
prepare_active_company_runtimes_for_shutdown=AsyncMock(
|
||||
side_effect=RuntimeError("checkpoint unavailable")
|
||||
),
|
||||
)
|
||||
handler._shutting_down = False
|
||||
handler._progress_flush_task = None
|
||||
handler._clients = set()
|
||||
handler._active_message_tasks = set()
|
||||
handler._background_tasks = {execution_task}
|
||||
handler._task_bg_context = {execution_task: {"task_id": "runtime-task"}}
|
||||
handler._task_bg_map = {"runtime-task": {execution_task}}
|
||||
|
||||
try:
|
||||
await handler.shutdown(timeout=1.0)
|
||||
except RuntimeError as exc:
|
||||
assert str(exc) == "checkpoint unavailable"
|
||||
else:
|
||||
raise AssertionError("shutdown must fail closed when checkpointing fails")
|
||||
|
||||
assert not released.is_set()
|
||||
assert not execution_task.done()
|
||||
execution_task.cancel()
|
||||
await asyncio.gather(execution_task, return_exceptions=True)
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_ws_shutdown_rejects_background_work_scheduled_by_late_ingress() -> None:
|
||||
async def scenario() -> None:
|
||||
entered = False
|
||||
|
||||
async def late_work() -> None:
|
||||
nonlocal entered
|
||||
entered = True
|
||||
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler._shutting_down = True
|
||||
handler._background_tasks = set()
|
||||
task = handler._track(late_work())
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
await asyncio.sleep(0)
|
||||
|
||||
assert task.cancelled()
|
||||
assert entered is False
|
||||
assert task not in handler._background_tasks
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_ws_shutdown_drains_queued_duplicate_handoff_before_checkpointing() -> None:
|
||||
async def scenario() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
runtime_lock = asyncio.Lock()
|
||||
execution_registered = asyncio.Event()
|
||||
prepare_called = asyncio.Event()
|
||||
execution_released = asyncio.Event()
|
||||
|
||||
async def prepare() -> list[dict]:
|
||||
assert registry.is_active("project-a", "runtime-task")
|
||||
assert registry.pending_handoff_count == 0
|
||||
prepare_called.set()
|
||||
return []
|
||||
|
||||
root_engine = SimpleNamespace(
|
||||
_active_task_run_registry=registry,
|
||||
prepare_active_company_runtimes_for_shutdown=prepare,
|
||||
)
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.engine = root_engine
|
||||
handler._root_engine = root_engine
|
||||
handler._shutting_down = False
|
||||
handler._progress_flush_task = None
|
||||
handler._clients = set()
|
||||
handler._active_message_tasks = set()
|
||||
handler._background_tasks = set()
|
||||
handler._task_bg_context = {}
|
||||
handler._task_bg_map = {}
|
||||
handler._handoff_route_tasks = {}
|
||||
|
||||
async def execution() -> None:
|
||||
async with runtime_lock:
|
||||
attempt_token = registry.register("project-a", "runtime-task")
|
||||
execution_registered.set()
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
registry.unregister("project-a", "runtime-task", attempt_token)
|
||||
execution_released.set()
|
||||
|
||||
async def queued_duplicate() -> None:
|
||||
async with runtime_lock:
|
||||
attempt_token = registry.register("project-a", "runtime-task")
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
registry.unregister("project-a", "runtime-task", attempt_token)
|
||||
|
||||
first_handoff = registry.reserve_handoff()
|
||||
with registry.bind_handoff(first_handoff):
|
||||
first = handler._track_session(
|
||||
"runtime-task",
|
||||
execution(),
|
||||
project_id="project-a",
|
||||
engine=root_engine,
|
||||
)
|
||||
registry.release_handoff(first_handoff)
|
||||
await execution_registered.wait()
|
||||
|
||||
second_handoff = registry.reserve_handoff()
|
||||
with registry.bind_handoff(second_handoff):
|
||||
second = handler._track_session(
|
||||
"runtime-task",
|
||||
queued_duplicate(),
|
||||
project_id="project-a",
|
||||
engine=root_engine,
|
||||
)
|
||||
registry.release_handoff(second_handoff)
|
||||
await asyncio.sleep(0)
|
||||
assert registry.pending_handoff_count == 1
|
||||
|
||||
await asyncio.wait_for(handler.shutdown(timeout=1.0), timeout=1.0)
|
||||
|
||||
assert prepare_called.is_set()
|
||||
assert execution_released.is_set()
|
||||
assert first.cancelled()
|
||||
assert second.cancelled()
|
||||
assert registry.pending_handoff_count == 0
|
||||
assert not registry.is_active("project-a", "runtime-task")
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_ws_shutdown_fails_closed_while_execution_cleanup_is_still_running() -> None:
|
||||
async def scenario() -> None:
|
||||
cancellation_started = asyncio.Event()
|
||||
allow_cleanup = asyncio.Event()
|
||||
|
||||
async def execution() -> None:
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
except asyncio.CancelledError:
|
||||
cancellation_started.set()
|
||||
await allow_cleanup.wait()
|
||||
|
||||
execution_task = asyncio.create_task(execution())
|
||||
await asyncio.sleep(0)
|
||||
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.engine = SimpleNamespace()
|
||||
handler._root_engine = SimpleNamespace(
|
||||
prepare_active_company_runtimes_for_shutdown=AsyncMock(return_value=[]),
|
||||
)
|
||||
handler._shutting_down = False
|
||||
handler._progress_flush_task = None
|
||||
handler._clients = set()
|
||||
handler._active_message_tasks = set()
|
||||
handler._background_tasks = {execution_task}
|
||||
handler._task_bg_context = {
|
||||
execution_task: {"task_id": "runtime-task", "execution_handoff": True}
|
||||
}
|
||||
handler._task_bg_map = {"runtime-task": {execution_task}}
|
||||
|
||||
try:
|
||||
await handler.shutdown(timeout=0.01)
|
||||
except RuntimeError as exc:
|
||||
assert "execution task(s)" in str(exc)
|
||||
else:
|
||||
raise AssertionError("shutdown must not close resources before execution cleanup")
|
||||
|
||||
assert cancellation_started.is_set()
|
||||
assert not execution_task.done()
|
||||
allow_cleanup.set()
|
||||
await execution_task
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_ws_shutdown_cancels_execution_before_waiting_for_client_close() -> None:
|
||||
async def scenario() -> None:
|
||||
execution_released = asyncio.Event()
|
||||
close_entered = asyncio.Event()
|
||||
allow_close = asyncio.Event()
|
||||
|
||||
async def execution() -> None:
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
execution_released.set()
|
||||
|
||||
class BlockingWebSocket:
|
||||
async def close(self, **_kwargs: object) -> None:
|
||||
close_entered.set()
|
||||
await allow_close.wait()
|
||||
|
||||
execution_task = asyncio.create_task(execution())
|
||||
await asyncio.sleep(0)
|
||||
client = BlockingWebSocket()
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.engine = SimpleNamespace()
|
||||
handler._root_engine = SimpleNamespace(
|
||||
prepare_active_company_runtimes_for_shutdown=AsyncMock(return_value=[]),
|
||||
)
|
||||
handler._shutting_down = False
|
||||
handler._progress_flush_task = None
|
||||
handler._clients = {client}
|
||||
handler._active_message_tasks = set()
|
||||
handler._background_tasks = {execution_task}
|
||||
handler._task_bg_context = {execution_task: {"task_id": "runtime-task"}}
|
||||
handler._task_bg_map = {"runtime-task": {execution_task}}
|
||||
|
||||
shutdown_task = asyncio.create_task(handler.shutdown(timeout=1.0))
|
||||
await close_entered.wait()
|
||||
|
||||
assert execution_released.is_set()
|
||||
assert execution_task.done()
|
||||
allow_close.set()
|
||||
await shutdown_task
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_duplicate_resume_does_not_leave_shutdown_handoff_barrier_queued() -> None:
|
||||
async def scenario() -> None:
|
||||
registry = ActiveTaskRunRegistry()
|
||||
execution_started = asyncio.Event()
|
||||
execution_released = asyncio.Event()
|
||||
|
||||
async def prepare() -> list[dict]:
|
||||
assert registry.is_active("project-a", "runtime-task")
|
||||
return []
|
||||
|
||||
root_engine = SimpleNamespace(
|
||||
project_id="project-a",
|
||||
_active_task_run_registry=registry,
|
||||
prepare_active_company_runtimes_for_shutdown=prepare,
|
||||
)
|
||||
handler = WSHandler.__new__(WSHandler)
|
||||
handler.engine = root_engine
|
||||
handler._root_engine = root_engine
|
||||
handler.chat_store = None
|
||||
handler._shutting_down = False
|
||||
handler._progress_flush_task = None
|
||||
handler._clients = set()
|
||||
handler._active_message_tasks = set()
|
||||
handler._background_tasks = set()
|
||||
handler._task_bg_context = {}
|
||||
handler._task_bg_map = {}
|
||||
handler._company_stop_finalize_tasks = {}
|
||||
handler._company_suspend_reply_locks = {}
|
||||
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="checkpoint-1",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
)
|
||||
task = Task(
|
||||
id="ui-task",
|
||||
title="Company chat",
|
||||
project_id="project-a",
|
||||
session_id="runtime-session",
|
||||
metadata={"exec_mode": "company"},
|
||||
)
|
||||
target = {
|
||||
"runtime_session_id": "runtime-session",
|
||||
"checkpoint": checkpoint,
|
||||
}
|
||||
handler._resolve_company_runtime_target = AsyncMock(return_value=target)
|
||||
|
||||
async def fake_resume(**_kwargs: object) -> None:
|
||||
attempt = registry.register("project-a", "runtime-task")
|
||||
execution_started.set()
|
||||
try:
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
registry.unregister("project-a", "runtime-task", attempt)
|
||||
execution_released.set()
|
||||
|
||||
handler._process_company_suspend_reply = fake_resume
|
||||
|
||||
async def route_once() -> bool:
|
||||
handoff = registry.reserve_handoff()
|
||||
try:
|
||||
with registry.bind_handoff(handoff):
|
||||
return await handler._route_company_suspend_reply_if_pending(
|
||||
task_id=task.id,
|
||||
content="continue",
|
||||
session_id=task.session_id,
|
||||
task=task,
|
||||
attachment_refs=None,
|
||||
message_metadata=None,
|
||||
user_message_id=None,
|
||||
user_message_created_at=None,
|
||||
run_engine=root_engine,
|
||||
run_project_id="project-a",
|
||||
)
|
||||
finally:
|
||||
registry.release_handoff(handoff)
|
||||
|
||||
assert await route_once() is True
|
||||
await execution_started.wait()
|
||||
assert await route_once() is True
|
||||
assert registry.pending_handoff_count == 0
|
||||
assert len(handler._background_tasks) == 1
|
||||
|
||||
await asyncio.wait_for(handler.shutdown(timeout=1.0), timeout=1.0)
|
||||
|
||||
assert execution_released.is_set()
|
||||
assert registry.pending_handoff_count == 0
|
||||
|
||||
asyncio.run(scenario())
|
||||
@@ -4,6 +4,7 @@ import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from opc.core.active_task_runs import ActiveTaskRunRegistry
|
||||
from opc.core.config import OPCConfig, RoleConfig
|
||||
from opc.core.org_config import (
|
||||
build_org_config_payload_from_config,
|
||||
@@ -137,6 +138,10 @@ def test_custom_runtime_initializes_with_parent_store(monkeypatch, tmp_path) ->
|
||||
self.store = kwargs.get("store") or object()
|
||||
self.owns_store = kwargs.get("owns_store")
|
||||
self.run_startup_reconcile = kwargs.get("run_startup_reconcile")
|
||||
self.active_task_run_registry = kwargs.get("active_task_run_registry")
|
||||
self.owns_active_task_run_registry = kwargs.get(
|
||||
"owns_active_task_run_registry"
|
||||
)
|
||||
self.bound_stores = []
|
||||
self.message_bus = FakeMessageBus(self)
|
||||
self.company_executor = SimpleNamespace(_signal_dispatcher_wake=lambda: None)
|
||||
@@ -164,6 +169,7 @@ def test_custom_runtime_initializes_with_parent_store(monkeypatch, tmp_path) ->
|
||||
parent.config = OPCConfig()
|
||||
parent.project_id = None
|
||||
parent.store = parent_store
|
||||
parent._active_task_run_registry = ActiveTaskRunRegistry()
|
||||
parent.on_progress = None
|
||||
parent.on_runtime_event = None
|
||||
parent.on_escalation = None
|
||||
@@ -198,6 +204,8 @@ def test_custom_runtime_initializes_with_parent_store(monkeypatch, tmp_path) ->
|
||||
assert runtime.store is parent_store
|
||||
assert runtime.owns_store is False
|
||||
assert runtime.run_startup_reconcile is False
|
||||
assert runtime.active_task_run_registry is parent._active_task_run_registry
|
||||
assert runtime.owns_active_task_run_registry is False
|
||||
assert runtime.bound_stores == []
|
||||
assert captured["kanban_callback_runtime"] is runtime
|
||||
assert callable(runtime.company_executor.on_kanban_changed)
|
||||
|
||||
@@ -188,10 +188,8 @@ class DirectStatusWriteLintTest(unittest.TestCase):
|
||||
PATTERN = re.compile(r"\.status\s*=\s*TaskStatus\.(CANCELLED|FAILED)")
|
||||
MIGRATED_COMPANY_AWARE_FILES = (
|
||||
"opc/plugins/office_ui/dispatcher.py",
|
||||
"opc/plugins/office_ui/recovery_manager.py",
|
||||
"opc/plugins/office_ui/ws_handler.py",
|
||||
"opc/plugins/cli_board/services/actions.py",
|
||||
"opc/plugins/cli_board/services/recovery.py",
|
||||
)
|
||||
MIGRATED_FIVE_STATUS_PATTERN = re.compile(
|
||||
r"\.status\s*=\s*TaskStatus\.(PENDING|RUNNING|DONE|FAILED|CANCELLED)"
|
||||
|
||||
@@ -27,6 +27,7 @@ from opc.database.store import _SQLiteConnectionAdapter
|
||||
from opc.layer2_organization import comms as file_comms
|
||||
from opc.plugins.office_ui.event_adapter import EventAdapter
|
||||
from opc.plugins.office_ui.chat_store import ChatStore
|
||||
from opc.plugins.office_ui.services.models import ServiceError
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
@@ -1823,6 +1824,126 @@ class TestWSHandlerSessionSend(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertEqual(len(ended), 1)
|
||||
self.handler._dispatch_session_message.assert_not_called()
|
||||
|
||||
async def _seed_cancelled_markerless_company_anchor(self) -> ExecutionCheckpoint:
|
||||
anchor = await self.store.get_task(self.task_id)
|
||||
assert anchor is not None
|
||||
anchor.status = TaskStatus.CANCELLED
|
||||
anchor.metadata = {}
|
||||
await self.store.save_task(anchor)
|
||||
await self.store.save_task(Task(
|
||||
id="shared-final-decider",
|
||||
title="Final decision",
|
||||
project_id="test-project",
|
||||
session_id=self.session_id,
|
||||
parent_session_id=self.session_id,
|
||||
linked_work_item_id="work-item-final",
|
||||
status=TaskStatus.BLOCKED,
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"company_profile": "corporate",
|
||||
"work_item_runtime": True,
|
||||
"work_item_projection_id": "final",
|
||||
"shared_role_session": True,
|
||||
},
|
||||
))
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="cp-markerless-anchor",
|
||||
project_id="test-project",
|
||||
session_id=self.session_id,
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id="shared-final-decider",
|
||||
payload={"parent_session_id": self.session_id},
|
||||
)
|
||||
await self.store.save_execution_checkpoint(checkpoint)
|
||||
return checkpoint
|
||||
|
||||
async def test_cancelled_markerless_company_anchor_text_routes_by_checkpoint_identity(self) -> None:
|
||||
checkpoint = await self._seed_cancelled_markerless_company_anchor()
|
||||
self.handler._process_company_suspend_reply = AsyncMock()
|
||||
fake_bg = object()
|
||||
|
||||
def _close_and_track(_task_id: str, coro: Any, **_kwargs: Any) -> object:
|
||||
coro.close()
|
||||
self.handler._task_bg_context[fake_bg] = {}
|
||||
return fake_bg
|
||||
|
||||
self.handler._track_session = MagicMock(side_effect=_close_and_track)
|
||||
ws = MagicMock()
|
||||
|
||||
await self.handler._handle_session_send(ws, {
|
||||
"project_id": "test-project",
|
||||
"task_id": self.task_id,
|
||||
"content": "继续恢复这个 runtime。",
|
||||
})
|
||||
|
||||
self.handler._process_company_suspend_reply.assert_called_once()
|
||||
routed = self.handler._process_company_suspend_reply.call_args.kwargs
|
||||
self.assertEqual(routed["ui_task_id"], self.task_id)
|
||||
self.assertEqual(routed["runtime_session_id"], self.session_id)
|
||||
self.assertEqual(routed["checkpoint"].checkpoint_id, checkpoint.checkpoint_id)
|
||||
self.handler._track_session.assert_called_once()
|
||||
anchor = await self.store.get_task(self.task_id)
|
||||
assert anchor is not None
|
||||
self.assertEqual(anchor.status, TaskStatus.CANCELLED)
|
||||
|
||||
async def test_cancelled_company_anchor_with_resuming_checkpoint_is_not_ended_early(self) -> None:
|
||||
checkpoint = await self._seed_cancelled_markerless_company_anchor()
|
||||
checkpoint.status = "resuming"
|
||||
await self.store.save_execution_checkpoint(checkpoint)
|
||||
self.handler._process_company_suspend_reply = AsyncMock()
|
||||
self.handler._process_session_message = AsyncMock()
|
||||
ws = MagicMock()
|
||||
|
||||
await self.handler._handle_session_send(ws, {
|
||||
"project_id": "test-project",
|
||||
"task_id": self.task_id,
|
||||
"content": "不要重复恢复。",
|
||||
})
|
||||
|
||||
ended = [
|
||||
call for call in self.handler._send_ack.await_args_list
|
||||
if call.kwargs.get("error") == "session_ended"
|
||||
]
|
||||
self.assertEqual(ended, [])
|
||||
self.handler._process_company_suspend_reply.assert_not_called()
|
||||
self.handler._process_session_message.assert_not_called()
|
||||
|
||||
async def test_cancelled_markerless_company_anchor_button_routes_by_checkpoint_identity(self) -> None:
|
||||
checkpoint = await self._seed_cancelled_markerless_company_anchor()
|
||||
self.handler._process_company_suspend_reply = AsyncMock()
|
||||
fake_bg = object()
|
||||
|
||||
def _close_and_track(_task_id: str, coro: Any, **_kwargs: Any) -> object:
|
||||
coro.close()
|
||||
self.handler._task_bg_context[fake_bg] = {}
|
||||
return fake_bg
|
||||
|
||||
self.handler._track_session = MagicMock(side_effect=_close_and_track)
|
||||
ws = MagicMock()
|
||||
|
||||
await self.handler._handle_session_resume(ws, {
|
||||
"project_id": "test-project",
|
||||
"task_id": self.task_id,
|
||||
"runtime_session_id": self.session_id,
|
||||
"checkpoint_id": checkpoint.checkpoint_id,
|
||||
})
|
||||
|
||||
self.handler._process_company_suspend_reply.assert_called_once()
|
||||
routed = self.handler._process_company_suspend_reply.call_args.kwargs
|
||||
self.assertEqual(routed["ui_task_id"], self.task_id)
|
||||
self.assertEqual(routed["runtime_session_id"], self.session_id)
|
||||
self.assertEqual(routed["checkpoint"].checkpoint_id, checkpoint.checkpoint_id)
|
||||
self.handler._send_ack.assert_awaited_with(
|
||||
ws,
|
||||
ok=True,
|
||||
runtime_session_id=self.session_id,
|
||||
checkpoint_id=checkpoint.checkpoint_id,
|
||||
)
|
||||
anchor = await self.store.get_task(self.task_id)
|
||||
assert anchor is not None
|
||||
self.assertEqual(anchor.status, TaskStatus.CANCELLED)
|
||||
|
||||
async def test_done_company_session_send_is_reopened_for_followup(self) -> None:
|
||||
"""Completed company chats can continue in the same CEO/company context."""
|
||||
ws = MagicMock()
|
||||
@@ -1980,23 +2101,23 @@ class TestWSHandlerSessionSend(unittest.IsolatedAsyncioTestCase):
|
||||
}
|
||||
await self.store.save_task(task)
|
||||
|
||||
self.engine.get_active_company_runtime_suspend_checkpoint = AsyncMock(
|
||||
return_value=SimpleNamespace(
|
||||
checkpoint_id="cp-suspended",
|
||||
checkpoint_type="company_runtime_suspended",
|
||||
status="pending",
|
||||
payload={},
|
||||
)
|
||||
)
|
||||
await self.store.save_execution_checkpoint(ExecutionCheckpoint(
|
||||
checkpoint_id="cp-suspended",
|
||||
project_id="test-project",
|
||||
session_id=self.session_id,
|
||||
checkpoint_type="company_runtime_suspended",
|
||||
status="pending",
|
||||
payload={"parent_session_id": self.session_id},
|
||||
))
|
||||
self.handler._process_company_suspend_reply = AsyncMock()
|
||||
fake_bg = object()
|
||||
|
||||
def _close_and_track(coro: Any) -> object:
|
||||
def _close_and_track(_task_id: str, coro: Any, **_kwargs: Any) -> object:
|
||||
coro.close()
|
||||
self.handler._task_bg_context[fake_bg] = {}
|
||||
return fake_bg
|
||||
|
||||
self.handler._track = MagicMock(side_effect=_close_and_track)
|
||||
self.handler._track_session = MagicMock()
|
||||
self.handler._track_session = MagicMock(side_effect=_close_and_track)
|
||||
|
||||
await self.handler._handle_session_send(ws, {
|
||||
"project_id": "test-project",
|
||||
@@ -2004,13 +2125,39 @@ class TestWSHandlerSessionSend(unittest.IsolatedAsyncioTestCase):
|
||||
"content": "改成 Sapphire Tide Runner,并让 CEO 自己修改/删除/新增 work item。",
|
||||
})
|
||||
|
||||
self.handler._track_session.assert_not_called()
|
||||
self.handler._track_session.assert_called_once()
|
||||
self.handler._process_company_suspend_reply.assert_called_once()
|
||||
call = self.handler._process_company_suspend_reply.call_args.kwargs
|
||||
self.assertEqual(call["parent_task_id"], self.task_id)
|
||||
self.assertEqual(call["parent_session_id"], self.session_id)
|
||||
self.assertEqual(call["ui_task_id"], self.task_id)
|
||||
self.assertEqual(call["runtime_session_id"], self.session_id)
|
||||
self.assertEqual(call["checkpoint"].checkpoint_id, "cp-suspended")
|
||||
self.assertEqual(call["content"], "改成 Sapphire Tide Runner,并让 CEO 自己修改/删除/新增 work item。")
|
||||
|
||||
async def test_rejected_company_resume_refreshes_optimistic_runtime_control(self) -> None:
|
||||
task = await self.store.get_task(self.task_id)
|
||||
assert task is not None
|
||||
task.metadata = {"exec_mode": "company", "company_profile": "corporate"}
|
||||
await self.store.save_task(task)
|
||||
self.handler._refresh_runtime_control_for_client = AsyncMock()
|
||||
ws = MagicMock()
|
||||
|
||||
await self.handler._handle_session_resume(ws, {
|
||||
"project_id": "test-project",
|
||||
"task_id": self.task_id,
|
||||
"runtime_session_id": self.session_id,
|
||||
})
|
||||
|
||||
self.handler._send_ack.assert_awaited_once_with(
|
||||
ws,
|
||||
ok=False,
|
||||
error="missing_checkpoint_id",
|
||||
)
|
||||
self.handler._refresh_runtime_control_for_client.assert_awaited_once_with(
|
||||
ws,
|
||||
engine=self.engine,
|
||||
project_id="test-project",
|
||||
)
|
||||
|
||||
async def test_session_send_persists_attachment_refs_and_dispatches_them(self) -> None:
|
||||
"""Uploaded session attachments should be stored and forwarded into engine execution."""
|
||||
ws = MagicMock()
|
||||
@@ -3264,6 +3411,37 @@ class TestWSHandlerSessionStop(unittest.IsolatedAsyncioTestCase):
|
||||
self.assertTrue(any(msg["payload"]["task_id"] == "stop-parent" and msg["payload"]["status"] == "idle" for msg in status_updates))
|
||||
self.assertTrue(any(msg["payload"]["task_id"] == "stop-child" and msg["payload"]["status"] == "cancelled" for msg in status_updates))
|
||||
|
||||
async def test_company_identity_failure_is_rejected_without_task_tree_cancel(self) -> None:
|
||||
ws = MagicMock()
|
||||
task = Task(
|
||||
id="company-stop-mismatch",
|
||||
title="Company runtime",
|
||||
session_id="company-stop-session",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.RUNNING,
|
||||
metadata={"exec_mode": "company", "company_profile": "corporate"},
|
||||
)
|
||||
await self.store.save_task(task)
|
||||
self.handler._resolve_company_runtime_target = AsyncMock(return_value=None)
|
||||
self.handler._cancel_task_tree = AsyncMock()
|
||||
|
||||
await self.handler._handle_session_stop(
|
||||
ws,
|
||||
{"project_id": "test-project", "task_id": task.id},
|
||||
)
|
||||
|
||||
self.handler._cancel_task_tree.assert_not_awaited()
|
||||
self.handler._send_ack.assert_awaited_once_with(
|
||||
ws,
|
||||
ok=False,
|
||||
error="company_runtime_identity_mismatch",
|
||||
project_id="test-project",
|
||||
task_id=task.id,
|
||||
)
|
||||
persisted = await self.store.get_task(task.id)
|
||||
assert persisted is not None
|
||||
self.assertEqual(persisted.status, TaskStatus.RUNNING)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Test 6: WSHandler — on_opc_event child_session_created
|
||||
@@ -3711,7 +3889,7 @@ class TestWSHandlerSessionDetail(unittest.IsolatedAsyncioTestCase):
|
||||
"execution_checkpoint_lifecycle",
|
||||
)
|
||||
|
||||
async def test_session_detail_includes_runtime_control_state(self) -> None:
|
||||
async def test_session_detail_uses_controller_registry_for_runtime_control_state(self) -> None:
|
||||
ws = MagicMock()
|
||||
ws.send_json = AsyncMock()
|
||||
task = Task(
|
||||
@@ -3735,8 +3913,17 @@ class TestWSHandlerSessionDetail(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
payload = ws.send_json.await_args.args[0]["payload"]
|
||||
self.assertTrue(payload["ok"])
|
||||
self.assertEqual(payload["session_state"].get("runtime_control_state"), "running")
|
||||
self.assertTrue(payload["session_state"].get("can_stop"))
|
||||
self.assertEqual(payload["session_state"].get("runtime_control_state"), "idle")
|
||||
self.assertFalse(payload["session_state"].get("can_stop"))
|
||||
|
||||
self.engine._task_runtime_is_live = AsyncMock(return_value=True)
|
||||
await self.handler._handle_session_detail(
|
||||
ws,
|
||||
{"project_id": "test-project", "task_id": "custom-running-task"},
|
||||
)
|
||||
live_payload = ws.send_json.await_args.args[0]["payload"]
|
||||
self.assertEqual(live_payload["session_state"].get("runtime_control_state"), "running")
|
||||
self.assertTrue(live_payload["session_state"].get("can_stop"))
|
||||
|
||||
async def test_session_detail_prefers_task_description_for_role_prompt_context(self) -> None:
|
||||
ws = MagicMock()
|
||||
@@ -4582,10 +4769,13 @@ class TestWSHandlerShutdown(unittest.IsolatedAsyncioTestCase):
|
||||
|
||||
ws.send_json.assert_awaited_once()
|
||||
|
||||
async def test_shutdown_closes_clients_and_waits_for_active_messages(self) -> None:
|
||||
async def test_shutdown_closes_clients_and_cancels_non_handoff_messages(self) -> None:
|
||||
ws = MagicMock()
|
||||
ws.closed = False
|
||||
ws.closing = False
|
||||
self.handler._root_engine.prepare_active_company_runtimes_for_shutdown = AsyncMock(
|
||||
return_value=[]
|
||||
)
|
||||
|
||||
async def _close(*_args: Any, **_kwargs: Any) -> None:
|
||||
ws.closed = True
|
||||
@@ -4593,23 +4783,16 @@ class TestWSHandlerShutdown(unittest.IsolatedAsyncioTestCase):
|
||||
ws.close = AsyncMock(side_effect=_close)
|
||||
self.handler._clients.add(ws)
|
||||
|
||||
blocker = asyncio.Event()
|
||||
|
||||
async def _active_message() -> None:
|
||||
await blocker.wait()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
active_task = asyncio.create_task(_active_message())
|
||||
self.handler._active_message_tasks.add(active_task)
|
||||
|
||||
shutdown_task = asyncio.create_task(self.handler.shutdown(timeout=0.5))
|
||||
await asyncio.sleep(0.05)
|
||||
self.assertFalse(shutdown_task.done())
|
||||
|
||||
blocker.set()
|
||||
await shutdown_task
|
||||
await active_task
|
||||
await self.handler.shutdown(timeout=0.5)
|
||||
|
||||
ws.close.assert_awaited_once()
|
||||
self.assertTrue(active_task.cancelled())
|
||||
self.assertTrue(self.handler._shutting_down)
|
||||
|
||||
|
||||
@@ -4967,6 +5150,312 @@ class TestOfficeServiceExecutionIdentity(unittest.IsolatedAsyncioTestCase):
|
||||
async def asyncTearDown(self) -> None:
|
||||
await self.chat_store._db.close()
|
||||
|
||||
async def test_continue_uses_scope_config_for_markerless_cancelled_ui_anchor(self) -> None:
|
||||
anchor = Task(
|
||||
id="service-ui-anchor",
|
||||
title="Company chat",
|
||||
session_id="service-runtime-session",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.CANCELLED,
|
||||
metadata={},
|
||||
)
|
||||
final_decider = Task(
|
||||
id="service-final-decider",
|
||||
title="Final decision",
|
||||
session_id="service-runtime-session",
|
||||
parent_session_id="service-runtime-session",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.BLOCKED,
|
||||
linked_work_item_id="service-work-item",
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"company_profile": "corporate",
|
||||
"work_item_runtime": True,
|
||||
"work_item_projection_id": "final",
|
||||
"shared_role_session": True,
|
||||
},
|
||||
)
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="service-runtime-checkpoint",
|
||||
project_id="test-project",
|
||||
session_id="service-runtime-session",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id=final_decider.id,
|
||||
payload={"parent_session_id": "service-runtime-session"},
|
||||
)
|
||||
await self.store.save_task(anchor)
|
||||
await self.store.save_task(final_decider)
|
||||
await self.store.save_execution_checkpoint(checkpoint)
|
||||
|
||||
await self.session_service.continue_run(
|
||||
project_id="test-project",
|
||||
task_id=anchor.id,
|
||||
runtime_session_id="service-runtime-session",
|
||||
checkpoint_id=checkpoint.checkpoint_id,
|
||||
content="continue",
|
||||
)
|
||||
|
||||
call = self.engine.process_message.await_args
|
||||
self.assertEqual(call.kwargs["mode"], "company")
|
||||
self.assertEqual(call.kwargs["session_id"], "service-runtime-session")
|
||||
self.assertEqual(call.kwargs["origin_task_id"], anchor.id)
|
||||
self.assertEqual(
|
||||
call.kwargs["message_metadata"]["response_to_checkpoint_id"],
|
||||
checkpoint.checkpoint_id,
|
||||
)
|
||||
persisted_anchor = await self.store.get_task(anchor.id)
|
||||
assert persisted_anchor is not None
|
||||
self.assertEqual(persisted_anchor.status, TaskStatus.CANCELLED)
|
||||
|
||||
async def test_company_continue_preserves_requested_work_item_as_ui_channel(self) -> None:
|
||||
anchor = Task(
|
||||
id="service-channel-anchor",
|
||||
title="Company chat",
|
||||
session_id="service-channel-runtime",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.CANCELLED,
|
||||
metadata={"exec_mode": "company", "company_profile": "corporate"},
|
||||
)
|
||||
work_item = Task(
|
||||
id="service-channel-work-item",
|
||||
title="Shared final decision",
|
||||
session_id="service-channel-runtime",
|
||||
parent_session_id="service-channel-runtime",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.BLOCKED,
|
||||
linked_work_item_id="service-channel-wi",
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"company_profile": "corporate",
|
||||
"work_item_runtime": True,
|
||||
"work_item_projection_id": "final",
|
||||
"shared_role_session": True,
|
||||
},
|
||||
)
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="service-channel-checkpoint",
|
||||
project_id="test-project",
|
||||
session_id="service-channel-runtime",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id=work_item.id,
|
||||
payload={"parent_session_id": "service-channel-runtime"},
|
||||
)
|
||||
await self.store.save_task(anchor)
|
||||
await self.store.save_task(work_item)
|
||||
await self.store.save_execution_checkpoint(checkpoint)
|
||||
|
||||
result = await self.session_service.continue_run(
|
||||
project_id="test-project",
|
||||
task_id=work_item.id,
|
||||
runtime_session_id="service-channel-runtime",
|
||||
checkpoint_id=checkpoint.checkpoint_id,
|
||||
content="continue",
|
||||
)
|
||||
|
||||
call = self.engine.process_message.await_args
|
||||
self.assertEqual(result.payload["task_id"], work_item.id)
|
||||
self.assertEqual(call.kwargs["session_id"], "service-channel-runtime")
|
||||
self.assertEqual(call.kwargs["origin_task_id"], anchor.id)
|
||||
|
||||
async def test_company_identity_failure_never_falls_back_to_task_mode_control(self) -> None:
|
||||
from opc.plugins.office_ui.services.models import ServiceError
|
||||
|
||||
task = Task(
|
||||
id="service-company-control",
|
||||
title="Company control",
|
||||
session_id="service-company-session",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.RUNNING,
|
||||
metadata={"exec_mode": "company", "company_profile": "corporate"},
|
||||
)
|
||||
await self.store.save_task(task)
|
||||
mismatch = ServiceError(
|
||||
"company_runtime_identity_mismatch",
|
||||
"identity mismatch",
|
||||
)
|
||||
self.session_service._resolve_company_runtime_target = AsyncMock(
|
||||
side_effect=mismatch,
|
||||
)
|
||||
|
||||
with self.assertRaises(ServiceError) as stop_error:
|
||||
await self.session_service.stop(
|
||||
project_id="test-project",
|
||||
task_id=task.id,
|
||||
)
|
||||
self.assertEqual(stop_error.exception.code, "company_runtime_identity_mismatch")
|
||||
|
||||
with self.assertRaises(ServiceError) as continue_error:
|
||||
await self.session_service.continue_run(
|
||||
project_id="test-project",
|
||||
task_id=task.id,
|
||||
)
|
||||
self.assertEqual(continue_error.exception.code, "company_runtime_identity_mismatch")
|
||||
persisted = await self.store.get_task(task.id)
|
||||
assert persisted is not None
|
||||
self.assertEqual(persisted.status, TaskStatus.RUNNING)
|
||||
|
||||
async def test_session_send_from_work_item_uses_runtime_checkpoint_identity(self) -> None:
|
||||
anchor = Task(
|
||||
id="service-send-anchor",
|
||||
title="Company chat",
|
||||
session_id="service-send-runtime",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.CANCELLED,
|
||||
metadata={},
|
||||
)
|
||||
final_decider = Task(
|
||||
id="service-send-final",
|
||||
title="Final decider",
|
||||
session_id="service-send-runtime",
|
||||
parent_session_id="service-send-runtime",
|
||||
project_id="test-project",
|
||||
linked_work_item_id="service-send-final-wi",
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"company_profile": "corporate",
|
||||
"work_item_runtime": True,
|
||||
"work_item_projection_id": "final",
|
||||
"shared_role_session": True,
|
||||
},
|
||||
)
|
||||
worker = Task(
|
||||
id="service-send-worker",
|
||||
title="Worker",
|
||||
session_id="service-send-runtime:role:worker",
|
||||
parent_session_id="service-send-runtime",
|
||||
project_id="test-project",
|
||||
linked_work_item_id="service-send-worker-wi",
|
||||
metadata={
|
||||
"exec_mode": "company",
|
||||
"company_profile": "corporate",
|
||||
"work_item_runtime": True,
|
||||
"work_item_projection_id": "worker",
|
||||
},
|
||||
)
|
||||
original_worker_metadata = dict(worker.metadata)
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="service-send-checkpoint",
|
||||
project_id="test-project",
|
||||
session_id="service-send-runtime",
|
||||
checkpoint_type="company_runtime_interrupted",
|
||||
status="pending",
|
||||
task_id=final_decider.id,
|
||||
payload={"parent_session_id": "service-send-runtime"},
|
||||
)
|
||||
for task in (anchor, final_decider, worker):
|
||||
await self.store.save_task(task)
|
||||
await self.store.save_execution_checkpoint(checkpoint)
|
||||
|
||||
result = await self.session_service.send(
|
||||
project_id="test-project",
|
||||
task_id=worker.id,
|
||||
content="revise and continue",
|
||||
)
|
||||
|
||||
call = self.engine.process_message.await_args
|
||||
self.assertEqual(result.payload["task_id"], worker.id)
|
||||
self.assertEqual(result.payload["session_id"], "service-send-runtime")
|
||||
self.assertEqual(call.kwargs["session_id"], "service-send-runtime")
|
||||
self.assertEqual(call.kwargs["origin_task_id"], anchor.id)
|
||||
self.assertEqual(call.kwargs["mode"], "company")
|
||||
self.assertEqual(call.kwargs["message_metadata"], {
|
||||
"response_to_checkpoint_id": checkpoint.checkpoint_id,
|
||||
"response_to_checkpoint_type": checkpoint.checkpoint_type,
|
||||
})
|
||||
persisted_worker = await self.store.get_task(worker.id)
|
||||
assert persisted_worker is not None
|
||||
self.assertEqual(persisted_worker.metadata, original_worker_metadata)
|
||||
|
||||
async def test_session_send_rejects_resuming_checkpoint_without_engine_fallback(self) -> None:
|
||||
anchor = Task(
|
||||
id="service-resuming-anchor",
|
||||
title="Company chat",
|
||||
session_id="service-resuming-runtime",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.CANCELLED,
|
||||
metadata={"exec_mode": "company", "company_profile": "corporate"},
|
||||
)
|
||||
checkpoint = ExecutionCheckpoint(
|
||||
checkpoint_id="service-resuming-checkpoint",
|
||||
project_id="test-project",
|
||||
session_id="service-resuming-runtime",
|
||||
checkpoint_type="company_runtime_suspended",
|
||||
status="resuming",
|
||||
task_id=anchor.id,
|
||||
payload={"parent_session_id": "service-resuming-runtime"},
|
||||
)
|
||||
await self.store.save_task(anchor)
|
||||
await self.store.save_execution_checkpoint(checkpoint)
|
||||
|
||||
with self.assertRaises(ServiceError) as raised:
|
||||
await self.session_service.send(
|
||||
project_id="test-project",
|
||||
task_id=anchor.id,
|
||||
content="continue twice",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
raised.exception.code,
|
||||
"company_runtime_checkpoint_not_pending",
|
||||
)
|
||||
self.engine.process_message.assert_not_called()
|
||||
|
||||
async def test_session_send_rejects_cancelled_task_mode_session(self) -> None:
|
||||
task = Task(
|
||||
id="service-cancelled-task-mode",
|
||||
title="Cancelled task chat",
|
||||
session_id="service-cancelled-task-session",
|
||||
project_id="test-project",
|
||||
status=TaskStatus.CANCELLED,
|
||||
metadata={
|
||||
"mode": "task",
|
||||
"execution_mode": "task_mode",
|
||||
"origin_task_id": "service-cancelled-task-mode",
|
||||
},
|
||||
)
|
||||
await self.store.save_task(task)
|
||||
|
||||
with self.assertRaises(ServiceError) as raised:
|
||||
await self.session_service.send(
|
||||
project_id="test-project",
|
||||
task_id=task.id,
|
||||
content="must stay cancelled",
|
||||
)
|
||||
|
||||
self.assertEqual(raised.exception.code, "session_ended")
|
||||
self.engine.process_message.assert_not_called()
|
||||
|
||||
async def test_session_send_company_identity_mismatch_fails_closed(self) -> None:
|
||||
task = Task(
|
||||
id="service-send-mismatch",
|
||||
title="Company chat",
|
||||
session_id="service-send-mismatch-runtime",
|
||||
project_id="test-project",
|
||||
metadata={"exec_mode": "company", "company_profile": "corporate"},
|
||||
)
|
||||
await self.store.save_task(task)
|
||||
self.session_service._resolve_company_runtime_target = AsyncMock(
|
||||
side_effect=ServiceError(
|
||||
"company_runtime_identity_mismatch",
|
||||
"identity mismatch",
|
||||
),
|
||||
)
|
||||
|
||||
with self.assertRaises(ServiceError) as raised:
|
||||
await self.session_service.send(
|
||||
project_id="test-project",
|
||||
task_id=task.id,
|
||||
content="do not fall back",
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
raised.exception.code,
|
||||
"company_runtime_identity_mismatch",
|
||||
)
|
||||
self.engine.process_message.assert_not_called()
|
||||
|
||||
async def test_session_send_prefers_persisted_org_identity_over_call_defaults(self) -> None:
|
||||
task = Task(
|
||||
id="task-org-send",
|
||||
|
||||
@@ -352,7 +352,6 @@ class DeliveryCardPhaseSyncTests(unittest.IsolatedAsyncioTestCase):
|
||||
executor._kanban_broadcast_task = None
|
||||
executor._kanban_debounce_sec = 0.2
|
||||
executor.runtime = MagicMock()
|
||||
executor._active_task_runs = set()
|
||||
executor._runtime_invariant_issue_keys = set()
|
||||
return executor
|
||||
|
||||
|
||||
@@ -384,7 +384,6 @@ def test_office_ui_work_item_paths_do_not_use_legacy_projection_identity_names()
|
||||
[
|
||||
REPO_ROOT / "opc/plugins/office_ui/ws_handler.py",
|
||||
REPO_ROOT / "opc/plugins/office_ui/snapshot_builder.py",
|
||||
REPO_ROOT / "opc/plugins/office_ui/recovery_manager.py",
|
||||
],
|
||||
)
|
||||
assert not backend_matches, _format_matches(backend_matches)
|
||||
|
||||
Reference in New Issue
Block a user