fix: route internal-turn approval cards to visible channels; feed provider errors back to the model
Approval cards raised by company-mode internal scheduling turns (review/report work items, session ids like `<root>:review::<wid>::vN`) were posted to the turn's own session channel, which the UI deliberately hides. The card silently timed out after 300s and the work item parked on AWAITING_HUMAN, so users saw only the gate card and never the approval prompt. ws_handler now detects these internal turns and routes their escalation cards to origin_task_id, the root session's primary task channel, or the activity channel — never the hidden one. Also unblocks the previously dead origin/session fallbacks in the resolver. Unclassified LLM stream failures (e.g. provider content-filter rejections like "input may contain sensitive information") used to hit a blind truncate-retry loop that replayed the identical payload for a dozen-plus consecutive failures. runtime_v2 now feeds the provider's verbatim error text back into the conversation as a "[runtime notice]" system message so the model can adapt (rephrase, drop quotes, change tack), bounded at 2 feedback retries (counter resets on any successful stream) plus one context-reset attempt, then fails honestly with the real error. The blind truncate path remains only for classified tool-protocol errors. Verified: new end-to-end tests for recover-after-notice and bounded-failure; runtime_v2 + ws_handler + escalation/approval + company-mode suites all green. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
@@ -270,6 +270,13 @@ class NativeRuntimeV2:
|
|||||||
1,
|
1,
|
||||||
int(self.config.system.native_runtime.reactive_compaction.max_overflow_retries or 1),
|
int(self.config.system.native_runtime.reactive_compaction.max_overflow_retries or 1),
|
||||||
) if self.config.system.native_runtime.reactive_compaction.enabled else 1
|
) if self.config.system.native_runtime.reactive_compaction.enabled else 1
|
||||||
|
# Unclassified provider failures (content filters, transient rejects)
|
||||||
|
# get bounded retries with the provider's error text fed back into the
|
||||||
|
# conversation so the model can adapt; the counter resets after every
|
||||||
|
# successful stream so long runs are not penalized for sporadic blips.
|
||||||
|
stream_error_feedback_retries = 0
|
||||||
|
max_stream_error_feedback_retries = 2
|
||||||
|
stream_error_context_reset_attempted = False
|
||||||
compaction_boundaries: list[dict[str, Any]] = []
|
compaction_boundaries: list[dict[str, Any]] = []
|
||||||
|
|
||||||
await self._save_runtime_session(
|
await self._save_runtime_session(
|
||||||
@@ -566,6 +573,7 @@ class NativeRuntimeV2:
|
|||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
await self._cancel_early_tool_runs(early_tool_runs)
|
await self._cancel_early_tool_runs(early_tool_runs)
|
||||||
|
if self.llm.is_tool_protocol_error(exc):
|
||||||
truncated = self._truncate_to_last_clean_user_turn(messages, base_prefix_len)
|
truncated = self._truncate_to_last_clean_user_turn(messages, base_prefix_len)
|
||||||
if truncated and len(truncated) > base_prefix_len:
|
if truncated and len(truncated) > base_prefix_len:
|
||||||
await self._emit_runtime_event(
|
await self._emit_runtime_event(
|
||||||
@@ -580,6 +588,40 @@ class NativeRuntimeV2:
|
|||||||
)
|
)
|
||||||
messages = truncated
|
messages = truncated
|
||||||
continue
|
continue
|
||||||
|
elif stream_error_feedback_retries < max_stream_error_feedback_retries:
|
||||||
|
stream_error_feedback_retries += 1
|
||||||
|
messages.append(self._provider_error_feedback_message(exc))
|
||||||
|
await self._emit_runtime_event(
|
||||||
|
runtime_session_id,
|
||||||
|
task,
|
||||||
|
"tool_protocol_retry",
|
||||||
|
{
|
||||||
|
"iteration": iteration + 1,
|
||||||
|
"strategy": "provider_error_feedback",
|
||||||
|
"attempt": stream_error_feedback_retries,
|
||||||
|
"message": str(exc),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
elif not stream_error_context_reset_attempted:
|
||||||
|
stream_error_context_reset_attempted = True
|
||||||
|
truncated = self._truncate_to_last_clean_user_turn(messages, base_prefix_len)
|
||||||
|
if truncated and base_prefix_len < len(truncated) < len(messages):
|
||||||
|
truncated.append(
|
||||||
|
self._provider_error_feedback_message(exc, context_reset=True)
|
||||||
|
)
|
||||||
|
await self._emit_runtime_event(
|
||||||
|
runtime_session_id,
|
||||||
|
task,
|
||||||
|
"tool_protocol_retry",
|
||||||
|
{
|
||||||
|
"iteration": iteration + 1,
|
||||||
|
"strategy": "provider_error_context_reset",
|
||||||
|
"message": str(exc),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
messages = truncated
|
||||||
|
continue
|
||||||
await self._emit_runtime_event(
|
await self._emit_runtime_event(
|
||||||
runtime_session_id,
|
runtime_session_id,
|
||||||
task,
|
task,
|
||||||
@@ -607,6 +649,7 @@ class NativeRuntimeV2:
|
|||||||
token_usage=total_usage,
|
token_usage=total_usage,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
stream_error_feedback_retries = 0
|
||||||
tool_calls = self._finalize_tool_calls(tool_call_chunks)
|
tool_calls = self._finalize_tool_calls(tool_call_chunks)
|
||||||
assistant_message = {"role": "assistant", "content": assistant_text}
|
assistant_message = {"role": "assistant", "content": assistant_text}
|
||||||
if tool_calls:
|
if tool_calls:
|
||||||
@@ -1697,6 +1740,41 @@ class NativeRuntimeV2:
|
|||||||
break
|
break
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _provider_error_feedback_message(
|
||||||
|
exc: Exception,
|
||||||
|
*,
|
||||||
|
context_reset: bool = False,
|
||||||
|
) -> dict[str, str]:
|
||||||
|
"""Conversation message telling the model why the last request failed.
|
||||||
|
|
||||||
|
Unclassified provider rejections (content filters, transient 4xx) never
|
||||||
|
produce model output, so without this the model has no way to know the
|
||||||
|
request failed or why. Feeding the provider's own error text back lets
|
||||||
|
the model decide how to proceed (rephrase, drop a quote, change tack)
|
||||||
|
instead of the runtime blindly replaying an identical payload.
|
||||||
|
"""
|
||||||
|
error_text = " ".join(str(exc).split())[:600]
|
||||||
|
if context_reset:
|
||||||
|
detail = (
|
||||||
|
"The previous LLM request kept failing at the model provider, so the "
|
||||||
|
"intermediate steps of the current turn were dropped from the request."
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
detail = (
|
||||||
|
"The previous LLM request failed at the model provider before any "
|
||||||
|
"output was produced."
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"role": "system",
|
||||||
|
"content": (
|
||||||
|
f"[runtime notice] {detail} Provider error: {error_text}. "
|
||||||
|
"This was not a user action. Adjust your next step accordingly — for "
|
||||||
|
"example rephrase sensitive wording, avoid quoting flagged content "
|
||||||
|
"verbatim, or choose another way to make progress — then continue the task."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
def _truncate_to_last_clean_user_turn(
|
def _truncate_to_last_clean_user_turn(
|
||||||
self,
|
self,
|
||||||
messages: list[dict[str, Any]],
|
messages: list[dict[str, Any]],
|
||||||
|
|||||||
@@ -1960,6 +1960,9 @@ class WSHandler:
|
|||||||
logger.warning(f"Failed to resolve escalation task mapping for {source_task_id}: {e}")
|
logger.warning(f"Failed to resolve escalation task mapping for {source_task_id}: {e}")
|
||||||
task = None
|
task = None
|
||||||
if task is not None:
|
if task is not None:
|
||||||
|
internal_turn_target = self._company_internal_turn_escalation_target(task)
|
||||||
|
if internal_turn_target is not None:
|
||||||
|
return internal_turn_target or None
|
||||||
ui_task_id = self._ui_task_id_for_task(task)
|
ui_task_id = self._ui_task_id_for_task(task)
|
||||||
if ui_task_id:
|
if ui_task_id:
|
||||||
return ui_task_id
|
return ui_task_id
|
||||||
@@ -1975,6 +1978,45 @@ class WSHandler:
|
|||||||
|
|
||||||
return source_task_id
|
return source_task_id
|
||||||
|
|
||||||
|
def _company_internal_turn_escalation_target(self, task: Any | None) -> str | None:
|
||||||
|
"""Visible routing target for escalations raised by internal
|
||||||
|
company-mode scheduling turns.
|
||||||
|
|
||||||
|
Review/report turn work items get composite ids (``review::<wid>::vN``),
|
||||||
|
so their runtime tasks carry session ids shaped like
|
||||||
|
``<root_session>:review::<wid>::vN``. The UI deliberately hides those
|
||||||
|
session channels, so an approval card posted to the turn's own channel
|
||||||
|
can never be seen or answered — it silently times out and the work item
|
||||||
|
parks on AWAITING_HUMAN.
|
||||||
|
|
||||||
|
Returns None when ``task`` is not such an internal turn (caller keeps
|
||||||
|
its normal resolution), the primary task id of the run's root session
|
||||||
|
when resolvable, or "" when the turn is internal but no visible session
|
||||||
|
is known (caller should fall back to the activity channel rather than
|
||||||
|
the hidden channel).
|
||||||
|
"""
|
||||||
|
if task is None:
|
||||||
|
return None
|
||||||
|
session_id = str(getattr(task, "session_id", "") or "").strip()
|
||||||
|
root_session_id, sep, suffix = session_id.partition(":")
|
||||||
|
if not sep or "::" not in suffix:
|
||||||
|
return None
|
||||||
|
metadata = dict(getattr(task, "metadata", {}) or {})
|
||||||
|
origin_task_id = str(metadata.get("origin_task_id") or "").strip()
|
||||||
|
task_id = str(getattr(task, "id", "") or "").strip()
|
||||||
|
if origin_task_id and origin_task_id != task_id:
|
||||||
|
return origin_task_id
|
||||||
|
for candidate_session_id in (
|
||||||
|
root_session_id,
|
||||||
|
str(getattr(task, "parent_session_id", "") or "").strip(),
|
||||||
|
):
|
||||||
|
if not candidate_session_id:
|
||||||
|
continue
|
||||||
|
mapped_task_id = str(self._session_to_task.get(candidate_session_id) or "").strip()
|
||||||
|
if mapped_task_id and mapped_task_id != task_id:
|
||||||
|
return mapped_task_id
|
||||||
|
return ""
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _pending_escalation_matches_task(record: dict[str, Any], task_id: str | None) -> bool:
|
def _pending_escalation_matches_task(record: dict[str, Any], task_id: str | None) -> bool:
|
||||||
task_key = str(task_id or "").strip()
|
task_key = str(task_id or "").strip()
|
||||||
|
|||||||
@@ -1227,6 +1227,120 @@ class NativeRuntimeV2Tests(unittest.IsolatedAsyncioTestCase):
|
|||||||
self.assertEqual(result.status, TaskStatus.DONE)
|
self.assertEqual(result.status, TaskStatus.DONE)
|
||||||
self.assertIn("Recovered after provider tool protocol fallback.", result.content)
|
self.assertIn("Recovered after provider tool protocol fallback.", result.content)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _make_provider_reject_llm(fail_times: int):
|
||||||
|
"""LLM stub whose stream fails ``fail_times`` times with an
|
||||||
|
unclassified provider rejection (content-filter style), then answers."""
|
||||||
|
|
||||||
|
class _ProviderRejectLLM:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.config = type("Cfg", (), {"max_tokens": 2048})()
|
||||||
|
self.stream_calls = 0
|
||||||
|
self.seen_notice_payloads: list[list[str]] = []
|
||||||
|
|
||||||
|
def prepare_user_message_content(self, content: str, attachment_refs=None):
|
||||||
|
_ = attachment_refs
|
||||||
|
return content
|
||||||
|
|
||||||
|
def get_tool_definitions(self, tools):
|
||||||
|
return tools
|
||||||
|
|
||||||
|
def is_context_overflow_error(self, error: Exception) -> bool:
|
||||||
|
_ = error
|
||||||
|
return False
|
||||||
|
|
||||||
|
def is_tool_protocol_error(self, error: Exception) -> bool:
|
||||||
|
_ = error
|
||||||
|
return False
|
||||||
|
|
||||||
|
def sanitize_tool_call_history(self, messages):
|
||||||
|
return list(messages)
|
||||||
|
|
||||||
|
async def chat_stream(self, messages, tools=None):
|
||||||
|
_ = tools
|
||||||
|
self.stream_calls += 1
|
||||||
|
self.seen_notice_payloads.append([
|
||||||
|
str(m.get("content", ""))
|
||||||
|
for m in messages
|
||||||
|
if m.get("role") == "system" and "[runtime notice]" in str(m.get("content", ""))
|
||||||
|
])
|
||||||
|
if self.stream_calls <= fail_times:
|
||||||
|
yield type("Evt", (), {"event_type": "message_start", "payload": {}, "model": "stub"})()
|
||||||
|
raise RuntimeError(
|
||||||
|
"litellm.BadRequestError: OpenAIException - The request failed "
|
||||||
|
"because the input may contain sensitive information."
|
||||||
|
)
|
||||||
|
yield type("Evt", (), {"event_type": "message_start", "payload": {}, "model": "stub"})()
|
||||||
|
yield type("Evt", (), {
|
||||||
|
"event_type": "assistant_delta",
|
||||||
|
"payload": {"text": "Rephrased and continued."},
|
||||||
|
"model": "stub",
|
||||||
|
})()
|
||||||
|
yield type("Evt", (), {"event_type": "message_stop", "payload": {}, "model": "stub"})()
|
||||||
|
|
||||||
|
async def chat(self, messages, tools=None):
|
||||||
|
raise AssertionError("non-stream fallback must not be used for unclassified errors")
|
||||||
|
|
||||||
|
return _ProviderRejectLLM()
|
||||||
|
|
||||||
|
async def test_unclassified_provider_error_feeds_error_back_and_recovers(self) -> None:
|
||||||
|
llm = self._make_provider_reject_llm(fail_times=1)
|
||||||
|
runtime = NativeRuntimeV2(
|
||||||
|
llm=llm,
|
||||||
|
tool_registry=ToolRegistry(),
|
||||||
|
memory_manager=_StubMemoryManager(_StubStore()),
|
||||||
|
config=OPCConfig(),
|
||||||
|
max_iterations=8,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await runtime.run(
|
||||||
|
system_prompt="You are a resilient runtime.",
|
||||||
|
user_message="Complete the task.",
|
||||||
|
task=Task(
|
||||||
|
title="provider-reject-recover",
|
||||||
|
description="provider-reject-recover",
|
||||||
|
session_id="sess-provider-reject-recover",
|
||||||
|
project_id="proj1",
|
||||||
|
metadata={"mode": "task"},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(result.status, TaskStatus.DONE)
|
||||||
|
self.assertIn("Rephrased and continued.", result.content)
|
||||||
|
self.assertEqual(llm.stream_calls, 2)
|
||||||
|
# The retry request must contain the provider's error text as a notice
|
||||||
|
retry_notices = llm.seen_notice_payloads[1]
|
||||||
|
self.assertEqual(len(retry_notices), 1)
|
||||||
|
self.assertIn("sensitive information", retry_notices[0])
|
||||||
|
self.assertIn("not a user action", retry_notices[0])
|
||||||
|
|
||||||
|
async def test_unclassified_provider_error_retries_are_bounded_then_fail(self) -> None:
|
||||||
|
llm = self._make_provider_reject_llm(fail_times=99)
|
||||||
|
runtime = NativeRuntimeV2(
|
||||||
|
llm=llm,
|
||||||
|
tool_registry=ToolRegistry(),
|
||||||
|
memory_manager=_StubMemoryManager(_StubStore()),
|
||||||
|
config=OPCConfig(),
|
||||||
|
max_iterations=20,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = await runtime.run(
|
||||||
|
system_prompt="You are a resilient runtime.",
|
||||||
|
user_message="Complete the task.",
|
||||||
|
task=Task(
|
||||||
|
title="provider-reject-bounded",
|
||||||
|
description="provider-reject-bounded",
|
||||||
|
session_id="sess-provider-reject-bounded",
|
||||||
|
project_id="proj1",
|
||||||
|
metadata={"mode": "task"},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertEqual(result.status, TaskStatus.FAILED)
|
||||||
|
self.assertIn("sensitive information", result.content)
|
||||||
|
# 1 initial + 2 feedback retries + 1 context-reset retry = 4, never 20
|
||||||
|
self.assertLessEqual(llm.stream_calls, 4)
|
||||||
|
|
||||||
async def test_todo_write_normalizes_openopc_task_ledger_shape(self) -> None:
|
async def test_todo_write_normalizes_openopc_task_ledger_shape(self) -> None:
|
||||||
runtime = NativeRuntimeV2(
|
runtime = NativeRuntimeV2(
|
||||||
llm=_StubLLM(),
|
llm=_StubLLM(),
|
||||||
|
|||||||
Reference in New Issue
Block a user