fix(ui): reconcile runtime status so stale chips converge without a refresh (#11)
Live status deltas are one-shot best-effort broadcasts: a delta lost to a
disconnect window, a project-scope drop, or a not-ready store left the UI
stuck on a stale status ("thinking" vs stopped) until a hard refresh, which
rebuilds from the always-correct snapshot. Close the class, not the sites:
- Add a low-frequency runtime_status_sync reconciliation broadcast: every
12s (lazily started, cancelled on shutdown) re-broadcast the persisted
status + in-memory tracker state of every task with a live runtime, plus
one final tick for tasks that just ended. Candidates come purely from
in-memory registries (no table scans); idle system pays nothing.
- Frontend consumes it diff-before-dispatch: a tick where nothing drifted
triggers zero store updates and zero re-renders; clearing mirrors the
mergeLiveRuntimeField semantics already used by collab_sync.
- Fix the EventAdapter tracker state machine: tool_completed returns to
REFLECTING (the turn is still running), and turn_completed/turn_failed
now transition to IDLE and emit an authoritative idle runtime update.
- Guarantee the terminal board_task_status_changed in _run_session_task's
finally: a cancelled run previously skipped it, leaving the board on
"running". The fallback mirrors persisted state read-only.
Verified: 7 new tests in test_runtime_status_sync.py; 236 backend tests
pass with zero new failures; tsc clean; frontend structural tests pass;
dist rebuilt.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -207,6 +207,8 @@ class EventAdapter:
|
|||||||
results.append(_ve(agent_id, runtime_type, dict(p)))
|
results.append(_ve(agent_id, runtime_type, dict(p)))
|
||||||
if runtime_type in {
|
if runtime_type in {
|
||||||
"turn_started",
|
"turn_started",
|
||||||
|
"turn_completed",
|
||||||
|
"turn_failed",
|
||||||
"tool_started",
|
"tool_started",
|
||||||
"tool_progress",
|
"tool_progress",
|
||||||
"tool_completed",
|
"tool_completed",
|
||||||
@@ -233,6 +235,11 @@ class EventAdapter:
|
|||||||
elif runtime_type in {"turn_started", "permission_requested", "permission_resolved", "subagent_started", "subagent_updated", "verification_started"}:
|
elif runtime_type in {"turn_started", "permission_requested", "permission_resolved", "subagent_started", "subagent_updated", "verification_started"}:
|
||||||
tracker.state = AgentAnimState.REFLECTING
|
tracker.state = AgentAnimState.REFLECTING
|
||||||
elif runtime_type in {"tool_completed", "subagent_completed", "verification_completed"}:
|
elif runtime_type in {"tool_completed", "subagent_completed", "verification_completed"}:
|
||||||
|
# The turn is still running between tools — the agent is
|
||||||
|
# back to reasoning, not idle. IDLE is owned by turn end.
|
||||||
|
tracker.current_tool = None
|
||||||
|
tracker.state = AgentAnimState.REFLECTING
|
||||||
|
elif runtime_type in {"turn_completed", "turn_failed"}:
|
||||||
tracker.current_tool = None
|
tracker.current_tool = None
|
||||||
tracker.state = AgentAnimState.IDLE
|
tracker.state = AgentAnimState.IDLE
|
||||||
extras: dict[str, Any] = {}
|
extras: dict[str, Any] = {}
|
||||||
|
|||||||
+59
-59
File diff suppressed because one or more lines are too long
@@ -5,7 +5,7 @@
|
|||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||||
<link rel="icon" href="data:," />
|
<link rel="icon" href="data:," />
|
||||||
<title>OpenOPC Pixel Office</title>
|
<title>OpenOPC Pixel Office</title>
|
||||||
<script type="module" crossorigin src="./assets/index-CTK-c7_K.js"></script>
|
<script type="module" crossorigin src="./assets/index-BgyI65M_.js"></script>
|
||||||
<link rel="modulepreload" crossorigin href="./assets/phaser-DFK5Ua9d.js">
|
<link rel="modulepreload" crossorigin href="./assets/phaser-DFK5Ua9d.js">
|
||||||
<link rel="stylesheet" crossorigin href="./assets/index-DEvLDWDw.css">
|
<link rel="stylesheet" crossorigin href="./assets/index-DEvLDWDw.css">
|
||||||
</head>
|
</head>
|
||||||
|
|||||||
@@ -1449,6 +1449,56 @@ export default function App() {
|
|||||||
scheduleSessionDetailRefresh(payload.task_id)
|
scheduleSessionDetailRefresh(payload.task_id)
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
onRuntimeStatusSync: (payload) => {
|
||||||
|
if (!payloadMatchesActiveProject(payload as unknown as Record<string, unknown>, false)) return
|
||||||
|
const ss = sessionStoreRef.current
|
||||||
|
const bs = boardStoreRef.current
|
||||||
|
if (!ss) return
|
||||||
|
// Periodic reconciliation against the backend's authoritative status.
|
||||||
|
// Diff before dispatching: a tick where nothing drifted must not
|
||||||
|
// trigger a single store update (and therefore no re-render).
|
||||||
|
for (const entry of payload.sessions ?? []) {
|
||||||
|
const taskId = String(entry.task_id ?? '').trim()
|
||||||
|
if (!taskId) continue
|
||||||
|
const session = ss.sessions.find((s) => s.taskId === taskId)
|
||||||
|
if (!session) continue
|
||||||
|
const status = String(entry.status ?? '').trim()
|
||||||
|
const patch: Partial<import('./types/kanban').Session> = {}
|
||||||
|
if (status && status !== session.status) patch.status = status
|
||||||
|
const rawAgentStatus = typeof entry.agent_status === 'string' ? entry.agent_status.trim() : ''
|
||||||
|
if (rawAgentStatus === 'idle' || rawAgentStatus === 'reflecting' || rawAgentStatus === 'tool_active') {
|
||||||
|
if (rawAgentStatus !== session.agentStatus) patch.agentStatus = rawAgentStatus
|
||||||
|
const tool = typeof entry.current_tool === 'string' && entry.current_tool.trim()
|
||||||
|
? entry.current_tool
|
||||||
|
: undefined
|
||||||
|
if (tool !== session.currentTool) patch.currentTool = tool
|
||||||
|
if (runtimeStatusClearsDisplayTool(rawAgentStatus) && session.displayTool !== undefined) {
|
||||||
|
patch.displayTool = undefined
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// No live tracker for this task: only clear stale indicators when
|
||||||
|
// the backend says the task is no longer running (mirrors the
|
||||||
|
// mergeLiveRuntimeField semantics used by collab_sync).
|
||||||
|
const controlActive = session.runtimeControlState === 'running'
|
||||||
|
|| session.runtimeControlState === 'suspending'
|
||||||
|
|| session.runtimeControlState === 'resuming'
|
||||||
|
if (status && status !== 'running' && !controlActive) {
|
||||||
|
if (session.agentStatus !== undefined) patch.agentStatus = undefined
|
||||||
|
if (session.currentTool !== undefined) patch.currentTool = undefined
|
||||||
|
if (session.displayTool !== undefined) patch.displayTool = undefined
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (Object.keys(patch).length === 0) continue
|
||||||
|
ss.updateSession(taskId, patch)
|
||||||
|
if (bs && ('agentStatus' in patch || 'currentTool' in patch || 'displayTool' in patch)) {
|
||||||
|
const boardPatch: Partial<KanbanTask> = {}
|
||||||
|
if ('agentStatus' in patch) boardPatch.agentStatus = patch.agentStatus as KanbanTask['agentStatus']
|
||||||
|
if ('currentTool' in patch) boardPatch.currentTool = patch.currentTool
|
||||||
|
if ('displayTool' in patch) boardPatch.displayTool = patch.displayTool
|
||||||
|
bs.updateTask(taskId, boardPatch)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
onWorkerNotification: (payload) => {
|
onWorkerNotification: (payload) => {
|
||||||
if (!payloadMatchesActiveProject(payload as unknown as Record<string, unknown>, false)) return
|
if (!payloadMatchesActiveProject(payload as unknown as Record<string, unknown>, false)) return
|
||||||
const data = payload as Record<string, unknown>
|
const data = payload as Record<string, unknown>
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ interface SocketHandlers {
|
|||||||
onCrossOfficeCollab?: (payload: { agent_ids: string[]; task_id: string; action: string }) => void
|
onCrossOfficeCollab?: (payload: { agent_ids: string[]; task_id: string; action: string }) => void
|
||||||
onCollabMessage?: (type: string, payload: Record<string, unknown>) => void
|
onCollabMessage?: (type: string, payload: Record<string, unknown>) => void
|
||||||
onAgentRuntimeUpdate?: (payload: AgentRuntimePayload) => void
|
onAgentRuntimeUpdate?: (payload: AgentRuntimePayload) => void
|
||||||
|
onRuntimeStatusSync?: (payload: RuntimeStatusSyncPayload) => void
|
||||||
onWorkerNotification?: (payload: WorkerNotificationPayload) => void
|
onWorkerNotification?: (payload: WorkerNotificationPayload) => void
|
||||||
onKanbanViewData?: (payload: KanbanViewDataPayload) => void
|
onKanbanViewData?: (payload: KanbanViewDataPayload) => void
|
||||||
onSessionCreated?: (payload: { project_id: string; task_id: string; channel_id: string; session_id?: string; parent_session_id?: string; origin_task_id?: string; title: string; status: string; created_at: number; assignee_ids?: string[]; exec_mode?: string; company_profile?: string; org_id?: string; organization_id?: string; preferred_agent?: TaskPreferredAgent; selected_execution_agent?: TaskPreferredAgent }) => void
|
onSessionCreated?: (payload: { project_id: string; task_id: string; channel_id: string; session_id?: string; parent_session_id?: string; origin_task_id?: string; title: string; status: string; created_at: number; assignee_ids?: string[]; exec_mode?: string; company_profile?: string; org_id?: string; organization_id?: string; preferred_agent?: TaskPreferredAgent; selected_execution_agent?: TaskPreferredAgent }) => void
|
||||||
@@ -60,6 +61,16 @@ interface SocketHandlers {
|
|||||||
onCommsMessage?: (payload: CommsMessagePayload) => void
|
onCommsMessage?: (payload: CommsMessagePayload) => void
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface RuntimeStatusSyncPayload {
|
||||||
|
project_id: string
|
||||||
|
sessions: Array<{
|
||||||
|
task_id: string
|
||||||
|
status: string
|
||||||
|
agent_status?: string
|
||||||
|
current_tool?: string | null
|
||||||
|
}>
|
||||||
|
}
|
||||||
|
|
||||||
export interface CommsMessageItem {
|
export interface CommsMessageItem {
|
||||||
message_id: string
|
message_id: string
|
||||||
from: string
|
from: string
|
||||||
@@ -752,6 +763,9 @@ export class VisualSocketClient {
|
|||||||
case 'agent_runtime_update':
|
case 'agent_runtime_update':
|
||||||
this.handlers.onAgentRuntimeUpdate?.(parsed.payload)
|
this.handlers.onAgentRuntimeUpdate?.(parsed.payload)
|
||||||
break
|
break
|
||||||
|
case 'runtime_status_sync':
|
||||||
|
this.handlers.onRuntimeStatusSync?.(parsed.payload)
|
||||||
|
break
|
||||||
case 'worker_notification':
|
case 'worker_notification':
|
||||||
this.handlers.onWorkerNotification?.(parsed.payload as WorkerNotificationPayload)
|
this.handlers.onWorkerNotification?.(parsed.payload as WorkerNotificationPayload)
|
||||||
break
|
break
|
||||||
|
|||||||
@@ -119,6 +119,7 @@ export type SocketEnvelope =
|
|||||||
| { type: 'comms_state'; payload: Record<string, unknown> }
|
| { type: 'comms_state'; payload: Record<string, unknown> }
|
||||||
| { type: 'comms_message'; payload: Record<string, unknown> }
|
| { type: 'comms_message'; payload: Record<string, unknown> }
|
||||||
| { type: 'comms_state_dirty'; payload: { project_id: string; [key: string]: unknown } }
|
| { type: 'comms_state_dirty'; payload: { project_id: string; [key: string]: unknown } }
|
||||||
|
| { type: 'runtime_status_sync'; payload: { project_id: string; sessions: Array<{ task_id: string; status: string; agent_status?: string; current_tool?: string | null }> } }
|
||||||
|
|
||||||
export type SocketStatus = 'connecting' | 'connected' | 'disconnected' | 'error'
|
export type SocketStatus = 'connecting' | 'connected' | 'disconnected' | 'error'
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,168 @@
|
|||||||
|
"""Tests for the runtime-status reconciliation broadcast (issue #11).
|
||||||
|
|
||||||
|
The sync tick re-broadcasts the authoritative status of every task with a
|
||||||
|
live runtime so any dropped live delta converges within one interval.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import unittest
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from typing import Any
|
||||||
|
from unittest.mock import AsyncMock
|
||||||
|
|
||||||
|
from opc.plugins.office_ui.event_adapter import AgentAnimState, EventAdapter
|
||||||
|
from opc.plugins.office_ui.ws_handler import WSHandler
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class FakeEvent:
|
||||||
|
event_type: str
|
||||||
|
payload: dict[str, Any]
|
||||||
|
timestamp: float = 0.0
|
||||||
|
event_id: str = "test"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _FakeTask:
|
||||||
|
status: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class _FakeStore:
|
||||||
|
tasks: dict[str, _FakeTask] = field(default_factory=dict)
|
||||||
|
|
||||||
|
async def get_task(self, task_id: str) -> _FakeTask | None:
|
||||||
|
return self.tasks.get(task_id)
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeBgTask:
|
||||||
|
"""Hashable stand-in for an asyncio.Task in the bg-context registry."""
|
||||||
|
|
||||||
|
def __init__(self, done: bool = False) -> None:
|
||||||
|
self._done = done
|
||||||
|
|
||||||
|
def done(self) -> bool:
|
||||||
|
return self._done
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_bg_task(done: bool = False) -> _FakeBgTask:
|
||||||
|
return _FakeBgTask(done)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_handler(tasks: dict[str, _FakeTask]) -> WSHandler:
|
||||||
|
engine = SimpleNamespace(project_id="p1", store=_FakeStore(tasks))
|
||||||
|
handler = WSHandler(engine, SimpleNamespace(), SimpleNamespace(), EventAdapter())
|
||||||
|
handler._engine_for_project = AsyncMock(return_value=engine) # type: ignore[method-assign]
|
||||||
|
handler.broadcast = AsyncMock() # type: ignore[method-assign]
|
||||||
|
handler._clients = {object()} # type: ignore[assignment]
|
||||||
|
return handler
|
||||||
|
|
||||||
|
|
||||||
|
class RuntimeStatusSyncTickTests(unittest.IsolatedAsyncioTestCase):
|
||||||
|
async def test_tick_broadcasts_authoritative_status_with_tracker_state(self) -> None:
|
||||||
|
handler = _make_handler({"task-1": _FakeTask(status="running")})
|
||||||
|
handler._task_bg_context[_fake_bg_task()] = {"task_id": "task-1", "project_id": "p1"}
|
||||||
|
tracker = handler.event_adapter._get_tracker("agent-1")
|
||||||
|
tracker.task_id = "task-1"
|
||||||
|
tracker.state = AgentAnimState.REFLECTING
|
||||||
|
tracker.current_tool = None
|
||||||
|
|
||||||
|
await handler._runtime_status_sync_tick()
|
||||||
|
|
||||||
|
handler.broadcast.assert_awaited_once()
|
||||||
|
envelope = handler.broadcast.await_args.args[0]
|
||||||
|
self.assertEqual(envelope["type"], "runtime_status_sync")
|
||||||
|
self.assertEqual(envelope["payload"]["project_id"], "p1")
|
||||||
|
self.assertEqual(envelope["payload"]["sessions"], [{
|
||||||
|
"task_id": "task-1",
|
||||||
|
"status": "running",
|
||||||
|
"agent_status": "reflecting",
|
||||||
|
"current_tool": None,
|
||||||
|
}])
|
||||||
|
|
||||||
|
async def test_tick_without_clients_or_candidates_is_silent(self) -> None:
|
||||||
|
handler = _make_handler({"task-1": _FakeTask(status="running")})
|
||||||
|
|
||||||
|
# No candidates at all → nothing to broadcast.
|
||||||
|
await handler._runtime_status_sync_tick()
|
||||||
|
handler.broadcast.assert_not_awaited()
|
||||||
|
|
||||||
|
# Candidates but no clients → still silent (snapshot covers reconnect).
|
||||||
|
handler._task_bg_context[_fake_bg_task()] = {"task_id": "task-1", "project_id": "p1"}
|
||||||
|
handler._clients = set() # type: ignore[assignment]
|
||||||
|
await handler._runtime_status_sync_tick()
|
||||||
|
handler.broadcast.assert_not_awaited()
|
||||||
|
|
||||||
|
async def test_departed_task_gets_one_final_tick(self) -> None:
|
||||||
|
handler = _make_handler({"task-1": _FakeTask(status="done")})
|
||||||
|
bg = _fake_bg_task()
|
||||||
|
handler._task_bg_context[bg] = {"task_id": "task-1", "project_id": "p1"}
|
||||||
|
|
||||||
|
await handler._runtime_status_sync_tick()
|
||||||
|
self.assertEqual(handler.broadcast.await_count, 1)
|
||||||
|
|
||||||
|
# Runtime finished: the task leaves the registry, one final sync fires.
|
||||||
|
del handler._task_bg_context[bg]
|
||||||
|
await handler._runtime_status_sync_tick()
|
||||||
|
self.assertEqual(handler.broadcast.await_count, 2)
|
||||||
|
envelope = handler.broadcast.await_args.args[0]
|
||||||
|
self.assertEqual(envelope["payload"]["sessions"][0]["status"], "done")
|
||||||
|
|
||||||
|
# After the final tick the task is forgotten entirely.
|
||||||
|
await handler._runtime_status_sync_tick()
|
||||||
|
self.assertEqual(handler.broadcast.await_count, 2)
|
||||||
|
|
||||||
|
async def test_company_child_inherits_root_project(self) -> None:
|
||||||
|
handler = _make_handler({
|
||||||
|
"root-1": _FakeTask(status="running"),
|
||||||
|
"child-1": _FakeTask(status="running"),
|
||||||
|
})
|
||||||
|
handler._task_bg_context[_fake_bg_task()] = {"task_id": "root-1", "project_id": "p1"}
|
||||||
|
handler._active_runtime_children["root-1"] = "root-1"
|
||||||
|
handler._active_runtime_children["child-1"] = "root-1"
|
||||||
|
|
||||||
|
await handler._runtime_status_sync_tick()
|
||||||
|
|
||||||
|
envelope = handler.broadcast.await_args.args[0]
|
||||||
|
task_ids = [s["task_id"] for s in envelope["payload"]["sessions"]]
|
||||||
|
self.assertEqual(task_ids, ["child-1", "root-1"])
|
||||||
|
self.assertEqual(envelope["payload"]["project_id"], "p1")
|
||||||
|
|
||||||
|
async def test_done_bg_tasks_are_not_candidates(self) -> None:
|
||||||
|
handler = _make_handler({"task-1": _FakeTask(status="running")})
|
||||||
|
handler._task_bg_context[_fake_bg_task(done=True)] = {"task_id": "task-1", "project_id": "p1"}
|
||||||
|
|
||||||
|
await handler._runtime_status_sync_tick()
|
||||||
|
handler.broadcast.assert_not_awaited()
|
||||||
|
|
||||||
|
|
||||||
|
class TrackerTurnLifecycleTests(unittest.TestCase):
|
||||||
|
"""The tracker must treat tool boundaries as reasoning, and turn ends as idle."""
|
||||||
|
|
||||||
|
def _runtime_event(self, runtime_type: str, **extra: Any) -> FakeEvent:
|
||||||
|
return FakeEvent("runtime_event", {"type": runtime_type, "task_id": "task-1", **extra})
|
||||||
|
|
||||||
|
def test_tool_completed_returns_to_reflecting(self) -> None:
|
||||||
|
adapter = EventAdapter()
|
||||||
|
adapter.translate(self._runtime_event("tool_started", tool_name="file_read", agent_id="a1"))
|
||||||
|
adapter.translate(self._runtime_event("tool_completed", agent_id="a1"))
|
||||||
|
tracker = adapter._get_tracker("a1")
|
||||||
|
self.assertEqual(tracker.state, AgentAnimState.REFLECTING)
|
||||||
|
self.assertIsNone(tracker.current_tool)
|
||||||
|
|
||||||
|
def test_turn_end_goes_idle_and_emits_runtime_update(self) -> None:
|
||||||
|
adapter = EventAdapter()
|
||||||
|
adapter.translate(self._runtime_event("turn_started", agent_id="a1"))
|
||||||
|
for terminal in ("turn_completed", "turn_failed"):
|
||||||
|
adapter.translate(self._runtime_event("turn_started", agent_id="a1"))
|
||||||
|
events = adapter.translate(self._runtime_event(terminal, agent_id="a1"))
|
||||||
|
tracker = adapter._get_tracker("a1")
|
||||||
|
self.assertEqual(tracker.state, AgentAnimState.IDLE)
|
||||||
|
updates = [e for e in events if e["type"] == "agent_runtime_update"]
|
||||||
|
self.assertEqual(len(updates), 1)
|
||||||
|
self.assertEqual(updates[0]["data"]["status"], "idle")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -382,6 +382,7 @@ _PROJECT_SCOPED_ENVELOPE_TYPES = frozenset({
|
|||||||
"comms_state",
|
"comms_state",
|
||||||
"comms_message",
|
"comms_message",
|
||||||
"comms_state_dirty",
|
"comms_state_dirty",
|
||||||
|
"runtime_status_sync",
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
@@ -463,6 +464,8 @@ class WSHandler:
|
|||||||
self._pending_escalation_order: list[str] = []
|
self._pending_escalation_order: list[str] = []
|
||||||
self._progress_buffer: dict[str, list[dict[str, Any]]] = {}
|
self._progress_buffer: dict[str, list[dict[str, Any]]] = {}
|
||||||
self._progress_project_ids: dict[str, str] = {}
|
self._progress_project_ids: dict[str, str] = {}
|
||||||
|
self._runtime_status_sync_task: asyncio.Task[Any] | None = None
|
||||||
|
self._runtime_status_sync_prev: dict[str, str] = {}
|
||||||
self._assistant_delta_buffers: dict[tuple[str, str, str], dict[str, Any]] = {}
|
self._assistant_delta_buffers: dict[tuple[str, str, str], dict[str, Any]] = {}
|
||||||
self._assistant_delta_flush_tasks: dict[tuple[str, str, str], asyncio.Task[None]] = {}
|
self._assistant_delta_flush_tasks: dict[tuple[str, str, str], asyncio.Task[None]] = {}
|
||||||
self._assistant_delta_seq: int = 0
|
self._assistant_delta_seq: int = 0
|
||||||
@@ -482,6 +485,7 @@ class WSHandler:
|
|||||||
# in RAM before they're persisted and visible on page refresh.
|
# in RAM before they're persisted and visible on page refresh.
|
||||||
self._PROGRESS_FLUSH_THRESHOLD = 2
|
self._PROGRESS_FLUSH_THRESHOLD = 2
|
||||||
self._PROGRESS_FLUSH_INTERVAL_SEC = 3.0
|
self._PROGRESS_FLUSH_INTERVAL_SEC = 3.0
|
||||||
|
self._RUNTIME_STATUS_SYNC_INTERVAL_SEC = 12.0
|
||||||
self._progress_flush_task: asyncio.Task[None] | None = None
|
self._progress_flush_task: asyncio.Task[None] | None = None
|
||||||
self._shutting_down: bool = False
|
self._shutting_down: bool = False
|
||||||
self._active_message_tasks: set[asyncio.Task[Any]] = set()
|
self._active_message_tasks: set[asyncio.Task[Any]] = set()
|
||||||
@@ -1081,6 +1085,7 @@ class WSHandler:
|
|||||||
return ws
|
return ws
|
||||||
self._clients.add(ws)
|
self._clients.add(ws)
|
||||||
self._ensure_progress_flush_loop()
|
self._ensure_progress_flush_loop()
|
||||||
|
self._ensure_runtime_status_sync_loop()
|
||||||
logger.info(f"WS client connected ({len(self._clients)} total)")
|
logger.info(f"WS client connected ({len(self._clients)} total)")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -2328,6 +2333,109 @@ class WSHandler:
|
|||||||
"Periodic progress flush error for task %s", tid,
|
"Periodic progress flush error for task %s", tid,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def _ensure_runtime_status_sync_loop(self) -> None:
|
||||||
|
"""Start the runtime-status reconciliation coroutine if not running.
|
||||||
|
|
||||||
|
Live status deltas are one-shot, best-effort broadcasts: a delta lost
|
||||||
|
to a disconnect window, a project-scope drop, or a not-ready store
|
||||||
|
leaves the UI stuck on a stale status until a full refresh. This loop
|
||||||
|
periodically re-broadcasts the authoritative status of every task with
|
||||||
|
a live runtime (plus one final tick for tasks that just ended) so any
|
||||||
|
missed delta converges within one interval.
|
||||||
|
"""
|
||||||
|
if self._runtime_status_sync_task and not self._runtime_status_sync_task.done():
|
||||||
|
return
|
||||||
|
self._runtime_status_sync_task = asyncio.create_task(self._runtime_status_sync_loop())
|
||||||
|
|
||||||
|
async def _runtime_status_sync_loop(self) -> None:
|
||||||
|
while not self._shutting_down:
|
||||||
|
try:
|
||||||
|
await asyncio.sleep(self._RUNTIME_STATUS_SYNC_INTERVAL_SEC)
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
break
|
||||||
|
if self._shutting_down:
|
||||||
|
break
|
||||||
|
try:
|
||||||
|
await self._runtime_status_sync_tick()
|
||||||
|
except Exception:
|
||||||
|
logger.opt(exception=True).debug("runtime status sync tick failed")
|
||||||
|
|
||||||
|
def _collect_runtime_status_candidates(self) -> dict[str, str]:
|
||||||
|
"""Map task_id → project_id for every task with a live runtime.
|
||||||
|
|
||||||
|
Sources are purely in-memory (no store scans): the background-task
|
||||||
|
context registry covers every session run across modes; the company
|
||||||
|
runtime-children map adds work-item child tasks of a live tree.
|
||||||
|
"""
|
||||||
|
candidates: dict[str, str] = {}
|
||||||
|
for bg, ctx in list(self._task_bg_context.items()):
|
||||||
|
if bg.done():
|
||||||
|
continue
|
||||||
|
t_id = str(ctx.get("task_id") or "").strip()
|
||||||
|
if t_id:
|
||||||
|
candidates[t_id] = self._normalize_project_id(str(ctx.get("project_id") or ""))
|
||||||
|
for child_id, root_id in list(self._active_runtime_children.items()):
|
||||||
|
if child_id in candidates:
|
||||||
|
continue
|
||||||
|
pid = (
|
||||||
|
candidates.get(root_id)
|
||||||
|
or self._progress_project_ids.get(child_id)
|
||||||
|
or self._progress_project_ids.get(root_id)
|
||||||
|
)
|
||||||
|
if pid:
|
||||||
|
candidates[child_id] = self._normalize_project_id(pid)
|
||||||
|
return candidates
|
||||||
|
|
||||||
|
async def _runtime_status_sync_tick(self) -> None:
|
||||||
|
current = self._collect_runtime_status_candidates()
|
||||||
|
if not self._clients:
|
||||||
|
# Nobody to correct; a connecting client gets a full snapshot.
|
||||||
|
self._runtime_status_sync_prev = current
|
||||||
|
return
|
||||||
|
departed = {
|
||||||
|
tid: pid for tid, pid in self._runtime_status_sync_prev.items()
|
||||||
|
if tid not in current
|
||||||
|
}
|
||||||
|
self._runtime_status_sync_prev = current
|
||||||
|
to_sync = {**departed, **current}
|
||||||
|
if not to_sync:
|
||||||
|
return
|
||||||
|
tracker_by_task: dict[str, tuple[str, str | None]] = {}
|
||||||
|
for tracker in getattr(self.event_adapter, "_trackers", {}).values():
|
||||||
|
t_id = str(getattr(tracker, "task_id", "") or "").strip()
|
||||||
|
if t_id:
|
||||||
|
tracker_by_task[t_id] = (tracker.state.value, tracker.current_tool)
|
||||||
|
by_project: dict[str, list[str]] = {}
|
||||||
|
for tid, pid in to_sync.items():
|
||||||
|
by_project.setdefault(pid, []).append(tid)
|
||||||
|
for pid, task_ids in by_project.items():
|
||||||
|
try:
|
||||||
|
engine = await self._engine_for_project(pid)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
store = getattr(engine, "store", None)
|
||||||
|
if store is None or not self._store_is_ready(store):
|
||||||
|
continue
|
||||||
|
sessions: list[dict[str, Any]] = []
|
||||||
|
for task_id in sorted(task_ids):
|
||||||
|
try:
|
||||||
|
t = await store.get_task(task_id)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
if t is None:
|
||||||
|
continue
|
||||||
|
status_val = t.status.value if hasattr(t.status, "value") else str(t.status)
|
||||||
|
entry: dict[str, Any] = {"task_id": task_id, "status": status_val}
|
||||||
|
tracked = tracker_by_task.get(task_id)
|
||||||
|
if tracked is not None:
|
||||||
|
entry["agent_status"], entry["current_tool"] = tracked
|
||||||
|
sessions.append(entry)
|
||||||
|
if sessions:
|
||||||
|
await self.broadcast({
|
||||||
|
"type": "runtime_status_sync",
|
||||||
|
"payload": {"project_id": pid, "sessions": sessions},
|
||||||
|
})
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _parse_progress_entry(text: str) -> dict[str, Any] | None:
|
def _parse_progress_entry(text: str) -> dict[str, Any] | None:
|
||||||
"""Parse on_progress text into a ProgressEntry dict, or None to skip."""
|
"""Parse on_progress text into a ProgressEntry dict, or None to skip."""
|
||||||
@@ -4586,6 +4694,16 @@ class WSHandler:
|
|||||||
pass
|
pass
|
||||||
self._progress_flush_task = None
|
self._progress_flush_task = None
|
||||||
|
|
||||||
|
if self._runtime_status_sync_task:
|
||||||
|
# Cancel rather than await: the sync interval is long enough that
|
||||||
|
# waiting for a natural wake-up would stall shutdown.
|
||||||
|
self._runtime_status_sync_task.cancel()
|
||||||
|
try:
|
||||||
|
await self._runtime_status_sync_task
|
||||||
|
except (asyncio.CancelledError, Exception):
|
||||||
|
pass
|
||||||
|
self._runtime_status_sync_task = None
|
||||||
|
|
||||||
clients = list(self._clients)
|
clients = list(self._clients)
|
||||||
for ws in clients:
|
for ws in clients:
|
||||||
try:
|
try:
|
||||||
@@ -8474,6 +8592,7 @@ class WSHandler:
|
|||||||
await self.broadcast({"type": "board_task_status_changed", "payload": {
|
await self.broadcast({"type": "board_task_status_changed", "payload": {
|
||||||
"project_id": pid, "task_id": task_id, "column_id": "in-progress", "status": "running",
|
"project_id": pid, "task_id": task_id, "column_id": "in-progress", "status": "running",
|
||||||
}})
|
}})
|
||||||
|
terminal_board_broadcast_sent = False
|
||||||
# Register company runtime origin so child-task progress can dual-route
|
# Register company runtime origin so child-task progress can dual-route
|
||||||
company_runtime_target: dict[str, Any] | None = None
|
company_runtime_target: dict[str, Any] | None = None
|
||||||
if session_exec_mode in ("company", "org", "custom"):
|
if session_exec_mode in ("company", "org", "custom"):
|
||||||
@@ -8563,6 +8682,7 @@ class WSHandler:
|
|||||||
await self.broadcast({"type": "board_task_status_changed", "payload": {
|
await self.broadcast({"type": "board_task_status_changed", "payload": {
|
||||||
"project_id": pid, "task_id": task_id, "column_id": final_column_id, "status": final_status,
|
"project_id": pid, "task_id": task_id, "column_id": final_column_id, "status": final_status,
|
||||||
}})
|
}})
|
||||||
|
terminal_board_broadcast_sent = True
|
||||||
if session_exec_mode in ("company", "org", "custom"):
|
if session_exec_mode in ("company", "org", "custom"):
|
||||||
try:
|
try:
|
||||||
idle_target = await self._resolve_company_runtime_target(task_id, engine=engine)
|
idle_target = await self._resolve_company_runtime_target(task_id, engine=engine)
|
||||||
@@ -8593,6 +8713,7 @@ class WSHandler:
|
|||||||
await self.broadcast({"type": "board_task_status_changed", "payload": {
|
await self.broadcast({"type": "board_task_status_changed", "payload": {
|
||||||
"project_id": pid, "task_id": task_id, "column_id": "in-progress", "status": "failed",
|
"project_id": pid, "task_id": task_id, "column_id": "in-progress", "status": "failed",
|
||||||
}})
|
}})
|
||||||
|
terminal_board_broadcast_sent = True
|
||||||
if session_exec_mode in ("company", "org", "custom"):
|
if session_exec_mode in ("company", "org", "custom"):
|
||||||
try:
|
try:
|
||||||
failed_target = await self._resolve_company_runtime_target(task_id, engine=engine)
|
failed_target = await self._resolve_company_runtime_target(task_id, engine=engine)
|
||||||
@@ -8615,6 +8736,25 @@ class WSHandler:
|
|||||||
self._stop_requested_task_ids.discard(task_id)
|
self._stop_requested_task_ids.discard(task_id)
|
||||||
if session_id:
|
if session_id:
|
||||||
self._session_to_task.pop(session_id, None)
|
self._session_to_task.pop(session_id, None)
|
||||||
|
if not terminal_board_broadcast_sent:
|
||||||
|
# A cancelled run (or a failure inside the failure handler)
|
||||||
|
# never reaches the terminal broadcasts above, leaving the
|
||||||
|
# board stuck on "running" until a full refresh. Mirror the
|
||||||
|
# persisted status best-effort; never write engine state here.
|
||||||
|
try:
|
||||||
|
store = engine.store
|
||||||
|
if self._store_is_ready(store):
|
||||||
|
t = await store.get_task(task_id)
|
||||||
|
if t is not None:
|
||||||
|
status_val = t.status.value if hasattr(t.status, "value") else str(t.status)
|
||||||
|
await self.broadcast({"type": "board_task_status_changed", "payload": {
|
||||||
|
"project_id": pid,
|
||||||
|
"task_id": task_id,
|
||||||
|
"column_id": "done" if status_val in ("done", "cancelled") else "in-progress",
|
||||||
|
"status": status_val,
|
||||||
|
}})
|
||||||
|
except Exception:
|
||||||
|
logger.opt(exception=True).debug("failed to mirror terminal board status")
|
||||||
# Clear agent runtime indicator — include agent_id so the
|
# Clear agent runtime indicator — include agent_id so the
|
||||||
# frontend can also clear the swarm agent's reflecting/tool_active state.
|
# frontend can also clear the swarm agent's reflecting/tool_active state.
|
||||||
idle_payload: dict[str, Any] = {
|
idle_payload: dict[str, Any] = {
|
||||||
|
|||||||
Reference in New Issue
Block a user