fix: route internal-turn approval cards to visible channels; feed provider errors back to the model

Approval cards raised by company-mode internal scheduling turns (review/report
work items, session ids like `<root>:review::<wid>::vN`) were posted to the
turn's own session channel, which the UI deliberately hides. The card silently
timed out after 300s and the work item parked on AWAITING_HUMAN, so users saw
only the gate card and never the approval prompt. ws_handler now detects these
internal turns and routes their escalation cards to origin_task_id, the root
session's primary task channel, or the activity channel — never the hidden one.
Also unblocks the previously dead origin/session fallbacks in the resolver.

Unclassified LLM stream failures (e.g. provider content-filter rejections like
"input may contain sensitive information") used to hit a blind truncate-retry
loop that replayed the identical payload for a dozen-plus consecutive failures.
runtime_v2 now feeds the provider's verbatim error text back into the
conversation as a "[runtime notice]" system message so the model can adapt
(rephrase, drop quotes, change tack), bounded at 2 feedback retries (counter
resets on any successful stream) plus one context-reset attempt, then fails
honestly with the real error. The blind truncate path remains only for
classified tool-protocol errors.

Verified: new end-to-end tests for recover-after-notice and bounded-failure;
runtime_v2 + ws_handler + escalation/approval + company-mode suites all green.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
LZH-YS1998
2026-07-07 18:22:49 +08:00
parent 12817a4e60
commit a30fa7588d
3 changed files with 238 additions and 4 deletions
+78
View File
@@ -270,6 +270,13 @@ class NativeRuntimeV2:
1,
int(self.config.system.native_runtime.reactive_compaction.max_overflow_retries or 1),
) if self.config.system.native_runtime.reactive_compaction.enabled else 1
# Unclassified provider failures (content filters, transient rejects)
# get bounded retries with the provider's error text fed back into the
# conversation so the model can adapt; the counter resets after every
# successful stream so long runs are not penalized for sporadic blips.
stream_error_feedback_retries = 0
max_stream_error_feedback_retries = 2
stream_error_context_reset_attempted = False
compaction_boundaries: list[dict[str, Any]] = []
await self._save_runtime_session(
@@ -566,6 +573,7 @@ class NativeRuntimeV2:
)
else:
await self._cancel_early_tool_runs(early_tool_runs)
if self.llm.is_tool_protocol_error(exc):
truncated = self._truncate_to_last_clean_user_turn(messages, base_prefix_len)
if truncated and len(truncated) > base_prefix_len:
await self._emit_runtime_event(
@@ -580,6 +588,40 @@ class NativeRuntimeV2:
)
messages = truncated
continue
elif stream_error_feedback_retries < max_stream_error_feedback_retries:
stream_error_feedback_retries += 1
messages.append(self._provider_error_feedback_message(exc))
await self._emit_runtime_event(
runtime_session_id,
task,
"tool_protocol_retry",
{
"iteration": iteration + 1,
"strategy": "provider_error_feedback",
"attempt": stream_error_feedback_retries,
"message": str(exc),
},
)
continue
elif not stream_error_context_reset_attempted:
stream_error_context_reset_attempted = True
truncated = self._truncate_to_last_clean_user_turn(messages, base_prefix_len)
if truncated and base_prefix_len < len(truncated) < len(messages):
truncated.append(
self._provider_error_feedback_message(exc, context_reset=True)
)
await self._emit_runtime_event(
runtime_session_id,
task,
"tool_protocol_retry",
{
"iteration": iteration + 1,
"strategy": "provider_error_context_reset",
"message": str(exc),
},
)
messages = truncated
continue
await self._emit_runtime_event(
runtime_session_id,
task,
@@ -607,6 +649,7 @@ class NativeRuntimeV2:
token_usage=total_usage,
)
stream_error_feedback_retries = 0
tool_calls = self._finalize_tool_calls(tool_call_chunks)
assistant_message = {"role": "assistant", "content": assistant_text}
if tool_calls:
@@ -1697,6 +1740,41 @@ class NativeRuntimeV2:
break
return None
@staticmethod
def _provider_error_feedback_message(
exc: Exception,
*,
context_reset: bool = False,
) -> dict[str, str]:
"""Conversation message telling the model why the last request failed.
Unclassified provider rejections (content filters, transient 4xx) never
produce model output, so without this the model has no way to know the
request failed or why. Feeding the provider's own error text back lets
the model decide how to proceed (rephrase, drop a quote, change tack)
instead of the runtime blindly replaying an identical payload.
"""
error_text = " ".join(str(exc).split())[:600]
if context_reset:
detail = (
"The previous LLM request kept failing at the model provider, so the "
"intermediate steps of the current turn were dropped from the request."
)
else:
detail = (
"The previous LLM request failed at the model provider before any "
"output was produced."
)
return {
"role": "system",
"content": (
f"[runtime notice] {detail} Provider error: {error_text}. "
"This was not a user action. Adjust your next step accordingly — for "
"example rephrase sensitive wording, avoid quoting flagged content "
"verbatim, or choose another way to make progress — then continue the task."
),
}
def _truncate_to_last_clean_user_turn(
self,
messages: list[dict[str, Any]],
+42
View File
@@ -1960,6 +1960,9 @@ class WSHandler:
logger.warning(f"Failed to resolve escalation task mapping for {source_task_id}: {e}")
task = None
if task is not None:
internal_turn_target = self._company_internal_turn_escalation_target(task)
if internal_turn_target is not None:
return internal_turn_target or None
ui_task_id = self._ui_task_id_for_task(task)
if ui_task_id:
return ui_task_id
@@ -1975,6 +1978,45 @@ class WSHandler:
return source_task_id
def _company_internal_turn_escalation_target(self, task: Any | None) -> str | None:
"""Visible routing target for escalations raised by internal
company-mode scheduling turns.
Review/report turn work items get composite ids (``review::<wid>::vN``),
so their runtime tasks carry session ids shaped like
``<root_session>:review::<wid>::vN``. The UI deliberately hides those
session channels, so an approval card posted to the turn's own channel
can never be seen or answered it silently times out and the work item
parks on AWAITING_HUMAN.
Returns None when ``task`` is not such an internal turn (caller keeps
its normal resolution), the primary task id of the run's root session
when resolvable, or "" when the turn is internal but no visible session
is known (caller should fall back to the activity channel rather than
the hidden channel).
"""
if task is None:
return None
session_id = str(getattr(task, "session_id", "") or "").strip()
root_session_id, sep, suffix = session_id.partition(":")
if not sep or "::" not in suffix:
return None
metadata = dict(getattr(task, "metadata", {}) or {})
origin_task_id = str(metadata.get("origin_task_id") or "").strip()
task_id = str(getattr(task, "id", "") or "").strip()
if origin_task_id and origin_task_id != task_id:
return origin_task_id
for candidate_session_id in (
root_session_id,
str(getattr(task, "parent_session_id", "") or "").strip(),
):
if not candidate_session_id:
continue
mapped_task_id = str(self._session_to_task.get(candidate_session_id) or "").strip()
if mapped_task_id and mapped_task_id != task_id:
return mapped_task_id
return ""
@staticmethod
def _pending_escalation_matches_task(record: dict[str, Any], task_id: str | None) -> bool:
task_key = str(task_id or "").strip()
+114
View File
@@ -1227,6 +1227,120 @@ class NativeRuntimeV2Tests(unittest.IsolatedAsyncioTestCase):
self.assertEqual(result.status, TaskStatus.DONE)
self.assertIn("Recovered after provider tool protocol fallback.", result.content)
@staticmethod
def _make_provider_reject_llm(fail_times: int):
"""LLM stub whose stream fails ``fail_times`` times with an
unclassified provider rejection (content-filter style), then answers."""
class _ProviderRejectLLM:
def __init__(self) -> None:
self.config = type("Cfg", (), {"max_tokens": 2048})()
self.stream_calls = 0
self.seen_notice_payloads: list[list[str]] = []
def prepare_user_message_content(self, content: str, attachment_refs=None):
_ = attachment_refs
return content
def get_tool_definitions(self, tools):
return tools
def is_context_overflow_error(self, error: Exception) -> bool:
_ = error
return False
def is_tool_protocol_error(self, error: Exception) -> bool:
_ = error
return False
def sanitize_tool_call_history(self, messages):
return list(messages)
async def chat_stream(self, messages, tools=None):
_ = tools
self.stream_calls += 1
self.seen_notice_payloads.append([
str(m.get("content", ""))
for m in messages
if m.get("role") == "system" and "[runtime notice]" in str(m.get("content", ""))
])
if self.stream_calls <= fail_times:
yield type("Evt", (), {"event_type": "message_start", "payload": {}, "model": "stub"})()
raise RuntimeError(
"litellm.BadRequestError: OpenAIException - The request failed "
"because the input may contain sensitive information."
)
yield type("Evt", (), {"event_type": "message_start", "payload": {}, "model": "stub"})()
yield type("Evt", (), {
"event_type": "assistant_delta",
"payload": {"text": "Rephrased and continued."},
"model": "stub",
})()
yield type("Evt", (), {"event_type": "message_stop", "payload": {}, "model": "stub"})()
async def chat(self, messages, tools=None):
raise AssertionError("non-stream fallback must not be used for unclassified errors")
return _ProviderRejectLLM()
async def test_unclassified_provider_error_feeds_error_back_and_recovers(self) -> None:
llm = self._make_provider_reject_llm(fail_times=1)
runtime = NativeRuntimeV2(
llm=llm,
tool_registry=ToolRegistry(),
memory_manager=_StubMemoryManager(_StubStore()),
config=OPCConfig(),
max_iterations=8,
)
result = await runtime.run(
system_prompt="You are a resilient runtime.",
user_message="Complete the task.",
task=Task(
title="provider-reject-recover",
description="provider-reject-recover",
session_id="sess-provider-reject-recover",
project_id="proj1",
metadata={"mode": "task"},
),
)
self.assertEqual(result.status, TaskStatus.DONE)
self.assertIn("Rephrased and continued.", result.content)
self.assertEqual(llm.stream_calls, 2)
# The retry request must contain the provider's error text as a notice
retry_notices = llm.seen_notice_payloads[1]
self.assertEqual(len(retry_notices), 1)
self.assertIn("sensitive information", retry_notices[0])
self.assertIn("not a user action", retry_notices[0])
async def test_unclassified_provider_error_retries_are_bounded_then_fail(self) -> None:
llm = self._make_provider_reject_llm(fail_times=99)
runtime = NativeRuntimeV2(
llm=llm,
tool_registry=ToolRegistry(),
memory_manager=_StubMemoryManager(_StubStore()),
config=OPCConfig(),
max_iterations=20,
)
result = await runtime.run(
system_prompt="You are a resilient runtime.",
user_message="Complete the task.",
task=Task(
title="provider-reject-bounded",
description="provider-reject-bounded",
session_id="sess-provider-reject-bounded",
project_id="proj1",
metadata={"mode": "task"},
),
)
self.assertEqual(result.status, TaskStatus.FAILED)
self.assertIn("sensitive information", result.content)
# 1 initial + 2 feedback retries + 1 context-reset retry = 4, never 20
self.assertLessEqual(llm.stream_calls, 4)
async def test_todo_write_normalizes_openopc_task_ledger_shape(self) -> None:
runtime = NativeRuntimeV2(
llm=_StubLLM(),