fix(llm): resolve context window via max_input_tokens with 128k fallback for unmapped models

- get_context_window() now reads litellm.get_model_info().max_input_tokens
  instead of get_max_tokens(), which returns the output cap and severely
  under-reported the window for every mapped model (e.g. deepseek 8k vs 1M)
- models litellm cannot map fall back to 128000 with a single warning per
  model instead of warning on every call and returning None
- raise LLMConfig.max_tokens default 8192 -> 32768 to match the template
- README: configure the API key directly in llm_config.yaml; document
  max_tokens / context_window in the example

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
LZH-YS1998
2026-07-04 17:53:12 +08:00
parent 08e48c2f9c
commit 6c8d3f3dc9
5 changed files with 82 additions and 38 deletions
+26 -4
View File
@@ -82,6 +82,12 @@ def _normalized_model_name(model: str) -> str:
return model.strip().lower()
# Used when neither user config nor litellm can supply a window. Conservative
# enough for modern models so compaction still has a real denominator.
_CONTEXT_WINDOW_FALLBACK = 128_000
_context_window_fallback_warned: set[str] = set()
_CONTEXT_WINDOW_OVERRIDES: tuple[tuple[str, int], ...] = (
("gpt-5.4-pro", 1_050_000),
("gpt-5.4-mini", 400_000),
@@ -269,11 +275,27 @@ class LLMProvider:
if override is not None:
return override
try:
limit = litellm.get_max_tokens(resolved_model)
return int(limit) if limit else None
# max_input_tokens is the context window; litellm.get_max_tokens()
# returns the "max_tokens" map entry, which for many models (e.g.
# deepseek) is the OUTPUT cap and would wildly under-report here.
info = litellm.get_model_info(resolved_model)
limit = info.get("max_input_tokens") or info.get("max_tokens")
if limit:
return int(limit)
reason = "model is not mapped in litellm"
except Exception as e:
logger.warning(f"Unable to resolve context window for model={resolved_model}: {e}")
return None
reason = str(e)
if resolved_model not in _context_window_fallback_warned:
_context_window_fallback_warned.add(resolved_model)
logger.warning(
"Unable to resolve context window for model={} ({}); assuming {} tokens. "
"Set llm.context_window or llm.context_window_overrides in llm_config.yaml "
"to use the model's real window.",
resolved_model,
reason,
_CONTEXT_WINDOW_FALLBACK,
)
return _CONTEXT_WINDOW_FALLBACK
def count_input_tokens(
self,