fix(llm): clamp max_tokens to the model's output limit before each call
A generous config default (32768) hard-fails on providers that reject an oversized max_tokens (e.g. DeepSeek caps output at 8192). Clamp to litellm's max_output_tokens when known, log once per model; unknown models pass through unchanged.
This commit is contained in:
+29
-2
@@ -86,6 +86,33 @@ def _normalized_model_name(model: str) -> str:
|
|||||||
# enough for modern models so compaction still has a real denominator.
|
# enough for modern models so compaction still has a real denominator.
|
||||||
_CONTEXT_WINDOW_FALLBACK = 128_000
|
_CONTEXT_WINDOW_FALLBACK = 128_000
|
||||||
_context_window_fallback_warned: set[str] = set()
|
_context_window_fallback_warned: set[str] = set()
|
||||||
|
_max_tokens_clamp_warned: set[str] = set()
|
||||||
|
|
||||||
|
|
||||||
|
def _clamp_max_tokens(model: str, requested: int) -> int:
|
||||||
|
"""Cap the requested output tokens at the model's known output limit.
|
||||||
|
|
||||||
|
Providers disagree on how to handle an oversized max_tokens: some clamp
|
||||||
|
silently, others (e.g. DeepSeek) reject the request outright. Clamping
|
||||||
|
here keeps a generous config default (32768) safe on small-cap models.
|
||||||
|
Unknown models pass through unchanged.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
info = litellm.get_model_info(model)
|
||||||
|
cap = info.get("max_output_tokens") or info.get("max_tokens")
|
||||||
|
except Exception:
|
||||||
|
return requested
|
||||||
|
if not cap or requested <= int(cap):
|
||||||
|
return requested
|
||||||
|
if model not in _max_tokens_clamp_warned:
|
||||||
|
_max_tokens_clamp_warned.add(model)
|
||||||
|
logger.info(
|
||||||
|
"max_tokens {} exceeds output limit {} of model={}; clamping.",
|
||||||
|
requested,
|
||||||
|
cap,
|
||||||
|
model,
|
||||||
|
)
|
||||||
|
return int(cap)
|
||||||
|
|
||||||
|
|
||||||
_CONTEXT_WINDOW_OVERRIDES: tuple[tuple[str, int], ...] = (
|
_CONTEXT_WINDOW_OVERRIDES: tuple[tuple[str, int], ...] = (
|
||||||
@@ -485,7 +512,7 @@ class LLMProvider:
|
|||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
model = self._select_model(task_type)
|
model = self._select_model(task_type)
|
||||||
temp = temperature if temperature is not None else self.config.temperature
|
temp = temperature if temperature is not None else self.config.temperature
|
||||||
max_tok = max_tokens if max_tokens is not None else self.config.max_tokens
|
max_tok = _clamp_max_tokens(model, max_tokens if max_tokens is not None else self.config.max_tokens)
|
||||||
|
|
||||||
call_kwargs: dict[str, Any] = {
|
call_kwargs: dict[str, Any] = {
|
||||||
"model": model,
|
"model": model,
|
||||||
@@ -630,7 +657,7 @@ class LLMProvider:
|
|||||||
) -> AsyncIterator[RuntimeLLMEvent]:
|
) -> AsyncIterator[RuntimeLLMEvent]:
|
||||||
model = self._select_model(task_type)
|
model = self._select_model(task_type)
|
||||||
temp = temperature if temperature is not None else self.config.temperature
|
temp = temperature if temperature is not None else self.config.temperature
|
||||||
max_tok = max_tokens if max_tokens is not None else self.config.max_tokens
|
max_tok = _clamp_max_tokens(model, max_tokens if max_tokens is not None else self.config.max_tokens)
|
||||||
|
|
||||||
call_kwargs: dict[str, Any] = {
|
call_kwargs: dict[str, Any] = {
|
||||||
"model": model,
|
"model": model,
|
||||||
|
|||||||
Reference in New Issue
Block a user