fix(approval): deliver late approval clicks past the held session turn lock
A company goal turn can hold the per-task session lock for hours while its live dispatcher waits on AWAITING_HUMAN approval cards. The card answers are themselves session messages, so they queued behind that same lock — a three-way circular wait (dispatcher waits for the answer, the answer waits for the lock, the lock waits for the dispatcher) that left late approval clicks recorded but never delivered, and the parked branches wedged forever. Timely clicks were unaffected because the inline-wait reply path resolves a future without touching the lock, which is why only late approvals failed. Three legs, all verified live on a wedged production run: 1. Lock-free answer path (ws_handler): a reply that explicitly targets a pending task_user_input / company_work_item_gate checkpoint while the task lock is held by a live turn is delivered straight through the engine's checkpoint-resume channel. With a live dispatcher the engine only persists the input, applies the approval decision, releases the human wait, and wakes the loop — no second dispatcher, no re-entry. When the lock is free the serialized path is kept unchanged. Failures surface to the user instead of silently queueing behind the wedge. 2. Approval treadmill: company runtime parks persisted the blocked call without its arguments, so the OBS-7 decision bridge could not rebuild the allowlist context — a late approve resumed the task but recorded no grant, and the identical command re-blocked and re-parked on a fresh card every cycle. The runtime park artifact now persists tool_args, the decision bridge falls back to permission_requests when pause_request.permission_context is absent, and the legacy checkpoint migration preserves existing permission_requests entries instead of rebuilding them empty. 3. OPC_ESCALATION_TIMEOUT_SECONDS env override for the inline approval wait (default unchanged) so harnesses can exercise the expire/park/ late-click cycle in seconds. Live verification on the wedged run: both stranded cards resumed (the second through the lock-free path while the first held the lock), a fresh 10s-expiry card answered late resumed within one second, the decision bridge recorded the grant on reply, and the run converged to delivery. Regression: 6 new lock-free path tests + 2 decision-bridge tests; full suite 1932 passed. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+51
-4
@@ -7,6 +7,7 @@ import copy
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
@@ -639,9 +640,21 @@ class OPCEngine:
|
||||
self.org_engine = OrgEngine(self.config, self.opc_home, store=self.store)
|
||||
self.talent_market = TalentMarket(self.opc_home, self.config)
|
||||
self.task_scheduler = TaskGraphScheduler(self.store, self.event_bus)
|
||||
escalation_timeout_seconds = self.config.system.escalation_timeout_seconds
|
||||
# Test/ops override: lets a harness shrink the inline approval wait
|
||||
# (e.g. to seconds) so the late-click park/resume cycle can be
|
||||
# exercised without waiting out the production timeout.
|
||||
raw_escalation_timeout = str(os.environ.get("OPC_ESCALATION_TIMEOUT_SECONDS", "") or "").strip()
|
||||
if raw_escalation_timeout:
|
||||
try:
|
||||
escalation_timeout_seconds = max(1, int(raw_escalation_timeout))
|
||||
except ValueError:
|
||||
logger.warning(
|
||||
f"Ignoring invalid OPC_ESCALATION_TIMEOUT_SECONDS={raw_escalation_timeout!r}"
|
||||
)
|
||||
self.escalation = EscalationEngine(
|
||||
self.event_bus,
|
||||
timeout_seconds=self.config.system.escalation_timeout_seconds,
|
||||
timeout_seconds=escalation_timeout_seconds,
|
||||
user_reply_callback=self.on_escalation,
|
||||
)
|
||||
self.communication = CommunicationManager(self.store, self.event_bus, self.llm, self.org_engine)
|
||||
@@ -4303,9 +4316,17 @@ class OPCEngine:
|
||||
"created_at": latest_compaction.created_at.isoformat(),
|
||||
})
|
||||
|
||||
permission_requests: list[dict[str, Any]] = []
|
||||
# Preserve any permission_requests already recorded on the payload —
|
||||
# they carry the blocked call's tool_args, which are the only source
|
||||
# of the command text for a late allowlist grant. Rebuilding from the
|
||||
# legacy approval/pause_request keys is a fallback, not a replacement.
|
||||
permission_requests: list[dict[str, Any]] = [
|
||||
dict(item)
|
||||
for item in list(payload_data.get("permission_requests", []) or [])
|
||||
if isinstance(item, dict)
|
||||
]
|
||||
approval = dict(payload_data.get("approval", {}) or {})
|
||||
if approval:
|
||||
if approval and not permission_requests:
|
||||
permission_requests.append({
|
||||
"tool_name": str(payload_data.get("tool_name", "") or ""),
|
||||
"resolution": "ask",
|
||||
@@ -11357,10 +11378,36 @@ class OPCEngine:
|
||||
injected_reply = user_reply.strip()
|
||||
permission_context = dict(pause_request.get("permission_context", {}) or {})
|
||||
blocked_tool_name = str(permission_context.get("tool_name", "") or "").strip()
|
||||
blocked_tool_args: dict[str, Any] = {}
|
||||
if not blocked_tool_name:
|
||||
# Company runtime parks persist the blocked call as a
|
||||
# permission_requests entry (runtime_v2 artifacts), not as
|
||||
# pause_request.permission_context. Without this fallback a late
|
||||
# approval reply resumes the task but records no allowlist grant,
|
||||
# so the identical command re-blocks and re-parks on a fresh card
|
||||
# every cycle (the project-0012 approve treadmill).
|
||||
for request in reversed(list(payload.get("permission_requests", []) or [])):
|
||||
if not isinstance(request, dict):
|
||||
continue
|
||||
if str(request.get("resolution", "") or "").strip() != "ask":
|
||||
continue
|
||||
candidate_tool = str(request.get("tool_name", "") or "").strip()
|
||||
if not candidate_tool:
|
||||
continue
|
||||
blocked_tool_name = candidate_tool
|
||||
raw_request_args = request.get("tool_args")
|
||||
if isinstance(raw_request_args, dict):
|
||||
blocked_tool_args = dict(raw_request_args)
|
||||
break
|
||||
decision_token = normalize_escalation_reply(user_reply)
|
||||
if blocked_tool_name and decision_token and self.approval_engine is not None:
|
||||
arguments: dict[str, Any] = {}
|
||||
raw_args = payload.get("tool_args") or pause_request.get("tool_args") or {}
|
||||
raw_args = (
|
||||
payload.get("tool_args")
|
||||
or pause_request.get("tool_args")
|
||||
or blocked_tool_args
|
||||
or {}
|
||||
)
|
||||
if isinstance(raw_args, dict):
|
||||
arguments = dict(raw_args)
|
||||
candidate = str(permission_context.get("candidate", "") or "").strip()
|
||||
|
||||
@@ -4029,8 +4029,15 @@ class NativeRuntimeV2:
|
||||
resolution_value = getattr(resolution, "value", str(resolution))
|
||||
if resolution_value not in {"ask", "deny"}:
|
||||
continue
|
||||
raw_arguments = call.get("arguments")
|
||||
requests.append({
|
||||
"tool_name": str(call.get("function", "") or ""),
|
||||
# The blocked call's arguments must survive into the park
|
||||
# checkpoint: a late approval reply rebuilds the allowlist
|
||||
# context from them, and without the command text no grant can
|
||||
# be recorded — the task resumes, retries, re-blocks, and
|
||||
# re-parks on an identical card forever.
|
||||
"tool_args": dict(raw_arguments) if isinstance(raw_arguments, dict) else {},
|
||||
"resolution": resolution_value,
|
||||
"scope": getattr(getattr(decision, "scope", None), "value", str(getattr(decision, "scope", ""))),
|
||||
"risk_level": getattr(getattr(decision, "risk_level", None), "value", str(getattr(decision, "risk_level", ""))),
|
||||
|
||||
@@ -8525,6 +8525,143 @@ class WSHandler:
|
||||
logger.opt(exception=True).debug("failed to persist checkpoint card terminal state")
|
||||
return None
|
||||
|
||||
_LOCK_FREE_CHECKPOINT_ANSWER_TYPES = frozenset({
|
||||
"task_user_input",
|
||||
"company_work_item_gate",
|
||||
})
|
||||
|
||||
async def _try_lock_free_parked_checkpoint_answer(
|
||||
self,
|
||||
*,
|
||||
task_id: str,
|
||||
content: str,
|
||||
session_id: str | None,
|
||||
message_metadata: dict[str, Any] | None,
|
||||
user_message_id: str | None,
|
||||
user_message_created_at: float | None,
|
||||
engine: Any,
|
||||
pid: str,
|
||||
channel_id: str,
|
||||
session_exec_mode: str,
|
||||
session_company_profile: str | None,
|
||||
session_org_id: str,
|
||||
attachment_refs: list[dict] | None,
|
||||
) -> bool:
|
||||
"""Answer a pending park checkpoint without taking the per-task turn lock.
|
||||
|
||||
A company goal turn can hold the per-task lock for hours while its live
|
||||
dispatcher waits on AWAITING_HUMAN approval cards. The card answers are
|
||||
themselves session messages, so they queue behind that same lock — a
|
||||
circular wait: dispatcher waits for the answer, the answer waits for the
|
||||
lock, the lock waits for the dispatcher (project-0012 late-approval
|
||||
wedge). When the reply explicitly targets a pending park checkpoint and
|
||||
the lock is currently held, deliver it straight through the engine's
|
||||
checkpoint-resume channel instead: with a live dispatcher the engine
|
||||
only persists the input, applies the approval decision, releases the
|
||||
human wait, and wakes the loop — no second dispatcher, no re-entry.
|
||||
|
||||
Returns True when the reply was fully handled here.
|
||||
"""
|
||||
metadata = dict(message_metadata or {})
|
||||
checkpoint_id = str(metadata.get("response_to_checkpoint_id", "") or "").strip()
|
||||
checkpoint_type = str(metadata.get("response_to_checkpoint_type", "") or "").strip()
|
||||
if not checkpoint_id or checkpoint_type not in self._LOCK_FREE_CHECKPOINT_ANSWER_TYPES:
|
||||
return False
|
||||
lock = self._get_task_lock(task_id)
|
||||
if not lock.locked():
|
||||
# No turn in flight: the serialized path works and preserves
|
||||
# ordering, so keep the existing behavior.
|
||||
return False
|
||||
store = getattr(engine, "store", None)
|
||||
if not self._store_is_ready(store):
|
||||
return False
|
||||
getter = getattr(store, "get_pending_checkpoints", None)
|
||||
if not callable(getter):
|
||||
return False
|
||||
try:
|
||||
pending = await getter(project_id=pid)
|
||||
except Exception:
|
||||
logger.opt(exception=True).debug(
|
||||
"Lock-free checkpoint answer: failed to load pending checkpoints"
|
||||
)
|
||||
return False
|
||||
checkpoint = next(
|
||||
(
|
||||
item
|
||||
for item in pending or []
|
||||
if str(getattr(item, "checkpoint_id", "") or "").strip() == checkpoint_id
|
||||
and str(getattr(item, "checkpoint_type", "") or "").strip()
|
||||
in self._LOCK_FREE_CHECKPOINT_ANSWER_TYPES
|
||||
),
|
||||
None,
|
||||
)
|
||||
if checkpoint is None:
|
||||
return False
|
||||
logger.info(
|
||||
f"Lock-free checkpoint answer: task lock for {task_id} is held by a "
|
||||
f"live turn; delivering reply to pending checkpoint {checkpoint_id} "
|
||||
"through the engine resume channel"
|
||||
)
|
||||
try:
|
||||
engine_mode, company_profile = self._resolve_engine_mode(
|
||||
session_exec_mode,
|
||||
session_company_profile,
|
||||
)
|
||||
engine_message_metadata = dict(metadata)
|
||||
engine_message_metadata.update(_ui_message_identity_metadata(
|
||||
message_id=user_message_id,
|
||||
conversation_turn_id=_ui_conversation_turn_id(user_message_id),
|
||||
created_at=user_message_created_at,
|
||||
))
|
||||
response = await engine.process_message(
|
||||
content,
|
||||
project_id=pid,
|
||||
session_id=session_id,
|
||||
mode=engine_mode,
|
||||
org_id=session_org_id or None,
|
||||
company_profile=company_profile,
|
||||
origin_task_id=task_id,
|
||||
attachment_refs=attachment_refs,
|
||||
message_metadata=engine_message_metadata or None,
|
||||
)
|
||||
updated_checkpoint_msg = await self._mark_checkpoint_card_after_engine_response(
|
||||
channel_id=channel_id,
|
||||
project_id=pid,
|
||||
engine=engine,
|
||||
message_metadata=engine_message_metadata,
|
||||
response_message_id=user_message_id,
|
||||
)
|
||||
if updated_checkpoint_msg is not None:
|
||||
await self.broadcast({"type": "session_message", "payload": updated_checkpoint_msg})
|
||||
reply_text = str(response or "").strip() or "Input received."
|
||||
reply_msg = await self.chat_store.insert_message(
|
||||
channel_id=channel_id,
|
||||
sender="assistant",
|
||||
sender_name="OPC",
|
||||
content=reply_text,
|
||||
project_id=pid,
|
||||
metadata={"type": "system", "checkpoint_answer_lock_free": True},
|
||||
)
|
||||
await self.broadcast({"type": "session_message", "payload": reply_msg})
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
# Do NOT fall back to the locked path: it would silently queue
|
||||
# behind the in-flight turn — exactly the wedge this path exists
|
||||
# to break. Surface the failure so the user can retry.
|
||||
logger.opt(exception=True).warning(
|
||||
f"Lock-free checkpoint answer failed for {checkpoint_id}"
|
||||
)
|
||||
helper = await self.chat_store.insert_message(
|
||||
channel_id=channel_id,
|
||||
sender="system",
|
||||
sender_name="OPC",
|
||||
content=f"Failed to deliver the approval reply: {exc}. Please click the card again.",
|
||||
project_id=pid,
|
||||
)
|
||||
await self.broadcast({"type": "session_message", "payload": helper})
|
||||
return True
|
||||
|
||||
async def _process_session_message(
|
||||
self, task_id: str, content: str, *,
|
||||
session_id: str | None = None,
|
||||
@@ -8566,6 +8703,23 @@ class WSHandler:
|
||||
session_org_id = self._resolve_task_org_id(task)
|
||||
session_preferred_agent = self._resolve_task_preferred_agent(task)
|
||||
|
||||
if await self._try_lock_free_parked_checkpoint_answer(
|
||||
task_id=task_id,
|
||||
content=content,
|
||||
session_id=session_id,
|
||||
message_metadata=message_metadata,
|
||||
user_message_id=user_message_id,
|
||||
user_message_created_at=user_message_created_at,
|
||||
engine=engine,
|
||||
pid=pid,
|
||||
channel_id=channel_id,
|
||||
session_exec_mode=session_exec_mode,
|
||||
session_company_profile=session_company_profile,
|
||||
session_org_id=session_org_id,
|
||||
attachment_refs=attachment_refs,
|
||||
):
|
||||
return
|
||||
|
||||
# Per-task lock: same session serialized, different sessions concurrent
|
||||
async with self._get_task_lock(task_id):
|
||||
current_task = asyncio.current_task()
|
||||
|
||||
Reference in New Issue
Block a user