Gitinbox/OpenOPC
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(llm): resolve context window via max_input_tokens with 128k fallback for unmapped models

Browse Source 
- get_context_window() now reads litellm.get_model_info().max_input_tokens
  instead of get_max_tokens(), which returns the output cap and severely
  under-reported the window for every mapped model (e.g. deepseek 8k vs 1M)
- models litellm cannot map fall back to 128000 with a single warning per
  model instead of warning on every call and returning None
- raise LLMConfig.max_tokens default 8192 -> 32768 to match the template
- README: configure the API key directly in llm_config.yaml; document
  max_tokens / context_window in the example

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
LZH-YS1998
2026-07-04 17:53:12 +08:00
parent 08e48c2f9c
commit 6c8d3f3dc9
5 changed files with 82 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_llm_provider_context_window.py
+36 -16
View File
@@ -35,7 +35,7 @@ class TestLLMProviderContextWindow(unittest.TestCase):
def test_gpt_5_4_override_applies_on_official_openai_base(self) -> None:
provider = LLMProvider(LLMConfig(default_model="openai/gpt-5.4"))
with patch("opc.llm.provider.litellm.get_max_tokens", return_value=128000):
with patch("opc.llm.provider.litellm.get_model_info", return_value={"max_input_tokens": 128000}):
self.assertEqual(provider.get_context_window(), 1_050_000)
def test_gpt_5_4_override_does_not_apply_on_proxy_base(self) -> None:
@@ -44,7 +44,7 @@ class TestLLMProviderContextWindow(unittest.TestCase):
api_base="https://openrouter.ai/api/v1",
))
with patch("opc.llm.provider.litellm.get_max_tokens", return_value=128000):
with patch("opc.llm.provider.litellm.get_model_info", return_value={"max_input_tokens": 128000}):
self.assertEqual(provider.get_context_window(), 128000)
def test_poe_claude_sonnet_4_5_model_uses_local_context_window(self) -> None:
@@ -53,9 +53,9 @@ class TestLLMProviderContextWindow(unittest.TestCase):
api_base="https://api.poe.com/v1",
))
with patch("opc.llm.provider.litellm.get_max_tokens") as get_max_tokens:
with patch("opc.llm.provider.litellm.get_model_info") as get_model_info:
self.assertEqual(provider.get_context_window(), 64_000)
get_max_tokens.assert_not_called()
get_model_info.assert_not_called()
def test_poe_openai_compatible_legacy_prefix_uses_same_context_window(self) -> None:
provider = LLMProvider(LLMConfig(
@@ -63,16 +63,27 @@ class TestLLMProviderContextWindow(unittest.TestCase):
api_base="https://api.poe.com/v1",
))
with patch("opc.llm.provider.litellm.get_max_tokens") as get_max_tokens:
with patch("opc.llm.provider.litellm.get_model_info") as get_model_info:
self.assertEqual(provider.get_context_window(), 64_000)
get_max_tokens.assert_not_called()
get_model_info.assert_not_called()
def test_non_overridden_model_still_uses_litellm(self) -> None:
provider = LLMProvider(LLMConfig(default_model="openai/gpt-4o"))
with patch("opc.llm.provider.litellm.get_max_tokens", return_value=128000):
with patch("opc.llm.provider.litellm.get_model_info", return_value={"max_input_tokens": 128000}):
self.assertEqual(provider.get_context_window(), 128000)
def test_context_window_uses_max_input_tokens_not_output_cap(self) -> None:
"""deepseek-style entries: max_tokens is the OUTPUT cap (8192), the
context window is max_input_tokens (1M). The window must not be 8192."""
provider = LLMProvider(LLMConfig(default_model="deepseek/deepseek-v4-pro"))
with patch(
"opc.llm.provider.litellm.get_model_info",
return_value={"max_input_tokens": 1_000_000, "max_tokens": 8192, "max_output_tokens": 8192},
):
self.assertEqual(provider.get_context_window(), 1_000_000)
def test_config_scalar_override_supplies_window_for_unmapped_model(self) -> None:
"""Unmapped proxy models (doubao/minimax/…) get a real window from config."""
provider = LLMProvider(LLMConfig(
@@ -81,19 +92,28 @@ class TestLLMProviderContextWindow(unittest.TestCase):
context_window=256000,
))
with patch("opc.llm.provider.litellm.get_max_tokens", return_value=None) as get_max_tokens:
with patch("opc.llm.provider.litellm.get_model_info", return_value={}) as get_model_info:
self.assertEqual(provider.get_context_window(), 256000)
get_max_tokens.assert_not_called()
get_model_info.assert_not_called()
def test_unmapped_model_without_override_returns_none(self) -> None:
"""No override + litellm can't map → None (unchanged fallback)."""
def test_unmapped_model_without_override_falls_back_to_default(self) -> None:
"""No override + litellm can't map → 128000 fallback, not None."""
provider = LLMProvider(LLMConfig(
default_model="openai/doubao-seed-2.0-pro",
api_base="https://ark.cn-beijing.volces.com/api/coding/v3",
))
with patch("opc.llm.provider.litellm.get_max_tokens", return_value=None):
self.assertIsNone(provider.get_context_window())
with patch("opc.llm.provider.litellm.get_model_info", return_value={}):
self.assertEqual(provider.get_context_window(), 128000)
def test_unmapped_model_litellm_error_falls_back_to_default(self) -> None:
provider = LLMProvider(LLMConfig(default_model="deepseek/deepseek-v4-pro"))
with patch(
"opc.llm.provider.litellm.get_model_info",
side_effect=Exception("Model deepseek-v4-pro isn't mapped yet."),
):
self.assertEqual(provider.get_context_window(), 128000)
def test_config_per_model_override_takes_precedence(self) -> None:
provider = LLMProvider(LLMConfig(
@@ -102,15 +122,15 @@ class TestLLMProviderContextWindow(unittest.TestCase):
context_window_overrides={"doubao-seed-2.0-pro": 262144},
))
with patch("opc.llm.provider.litellm.get_max_tokens", return_value=None):
with patch("opc.llm.provider.litellm.get_model_info", return_value={}):
self.assertEqual(provider.get_context_window(), 262144)
def test_config_override_wins_over_litellm_for_mapped_model(self) -> None:
provider = LLMProvider(LLMConfig(default_model="openai/gpt-4o", context_window=50000))
with patch("opc.llm.provider.litellm.get_max_tokens", return_value=128000) as get_max_tokens:
with patch("opc.llm.provider.litellm.get_model_info", return_value={"max_input_tokens": 128000}) as get_model_info:
self.assertEqual(provider.get_context_window(), 50000)
get_max_tokens.assert_not_called()
get_model_info.assert_not_called()
if __name__ == "__main__":