From a78a55370e5a3e81538e588fddd258aed12cca3b Mon Sep 17 00:00:00 2001 From: akushonkamen Date: Fri, 9 Oct 2026 00:15:58 +0800 Subject: [PATCH] Reject the invalid 'llm.model' config key with an actionable error dacite silently drops config keys that do not match a dataclass field, and LLMConfig has no 'model' field - models are configured via 'llm.models' or 'llm.primary_model'. A config using 'model:' therefore passed validation with an empty model ensemble, and every LLM call then failed with 'IndexError: list index out of range', retried forever, and flooded the log (issue #427). The reporter hit this by following the README's own provider examples, which used 'model:' in three places. Two-part fix: - Config.from_dict now raises ValueError when 'llm.model' is present, pointing at 'llm.primary_model' / 'llm.models'. The check lives in from_dict (not __post_init__) so internally constructed Config() objects keep their empty-default ensemble, matching the existing explicit-ValueError style of from_dict. - README provider examples (Gemini, Ollama/vLLM, OptiLLM) corrected from 'model:' to 'primary_model:', consistent with examples/*/config.yaml and the rest of the configuration docs. Fixes #427 --- README.md | 6 +-- openevolve/config.py | 10 +++++ tests/test_llm_model_key_rejected.py | 63 ++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 tests/test_llm_model_key_rejected.py diff --git a/README.md b/README.md index cece488ae..7c0d246a5 100644 --- a/README.md +++ b/README.md @@ -356,7 +356,7 @@ export OPENAI_API_KEY="sk-..." # config.yaml llm: api_base: "https://generativelanguage.googleapis.com/v1beta/openai/" - model: "gemini-2.5-pro" + primary_model: "gemini-2.5-pro" ``` ```bash @@ -372,7 +372,7 @@ export OPENAI_API_KEY="your-gemini-api-key" # config.yaml llm: api_base: "http://localhost:11434/v1" # Ollama - model: "codellama:7b" + primary_model: "codellama:7b" ``` @@ -396,7 +396,7 @@ export OPENAI_API_KEY="your-actual-key" ```yaml llm: api_base: "http://localhost:8000/v1" - model: "moa&readurls-o3" # Test-time compute + web access + primary_model: "moa&readurls-o3" # Test-time compute + web access ``` diff --git a/openevolve/config.py b/openevolve/config.py index 3d08ceb14..d68b701f8 100644 --- a/openevolve/config.py +++ b/openevolve/config.py @@ -489,6 +489,16 @@ def from_dict(cls, config_dict: Dict[str, Any]) -> "Config": del config_dict["llm"]["temperature"] if "top_p" in config_dict["llm"] and config_dict["llm"]["top_p"] is None: del config_dict["llm"]["top_p"] + # 'model' is not a config field (models are 'llm.models' or + # 'llm.primary_model'), and dacite silently drops unknown keys. Left + # unchecked, such configs pass validation with an empty model ensemble + # and every LLM call later fails with "IndexError: list index out of + # range" (issue #427). Fail here with the correct spelling instead. + if isinstance(config_dict["llm"], dict) and "model" in config_dict["llm"]: + raise ValueError( + "Invalid config key 'llm.model'. Use 'llm.primary_model' or the " + "'llm.models' array instead (see README configuration section)." + ) config: Config = dacite.from_dict( data_class=cls, diff --git a/tests/test_llm_model_key_rejected.py b/tests/test_llm_model_key_rejected.py new file mode 100644 index 000000000..6439f8d85 --- /dev/null +++ b/tests/test_llm_model_key_rejected.py @@ -0,0 +1,63 @@ +""" +Tests for rejection of the legacy/mistaken 'llm.model' config key (issue #427). + +The LLM config has no 'model' field - models are configured via 'llm.models' +or 'llm.primary_model'. dacite silently drops unknown keys, so a config using +'model:' (as several README provider examples taught) produced an empty model +ensemble that later crashed every LLM call with "IndexError: list index out +of range". Config.from_dict must reject the key with an actionable error. +""" + +import unittest + +from openevolve.config import Config + + +class TestLLMModelKeyRejected(unittest.TestCase): + """The 'llm.model' key must fail loudly instead of building an empty ensemble""" + + def test_model_key_raises_helpful_error(self): + """A bare llm.model key raises ValueError pointing at the right keys""" + with self.assertRaises(ValueError) as ctx: + Config.from_dict({"llm": {"model": "glm-ocr"}}) + self.assertIn("primary_model", str(ctx.exception)) + self.assertIn("llm.models", str(ctx.exception)) + + def test_model_key_with_other_keys_still_raises(self): + """llm.model raises even when mixed into an otherwise valid config""" + config_dict = { + "max_iterations": 100, + "llm": { + "api_base": "http://localhost:3000/api/", + "api_key": "test", + "model": "glm-ocr", + "temperature": 0.7, + }, + } + with self.assertRaises(ValueError) as ctx: + Config.from_dict(config_dict) + self.assertIn("llm.model", str(ctx.exception)) + + def test_primary_model_config_still_valid(self): + """The documented primary_model spelling keeps working""" + config = Config.from_dict( + {"llm": {"primary_model": "glm-ocr", "api_base": "http://localhost:3000/api/"}} + ) + self.assertEqual(len(config.llm.models), 1) + self.assertEqual(config.llm.models[0].name, "glm-ocr") + + def test_models_array_config_still_valid(self): + """The models array spelling keeps working""" + config = Config.from_dict({"llm": {"models": [{"name": "gpt-4o-mini", "weight": 1.0}]}}) + self.assertEqual(len(config.llm.models), 1) + self.assertEqual(config.llm.models[0].name, "gpt-4o-mini") + + def test_code_constructed_config_unaffected(self): + """Internally constructed Config() (which never goes through from_dict) + keeps its empty default ensemble - that path is not a user error""" + config = Config() + self.assertEqual(config.llm.models, []) + + +if __name__ == "__main__": + unittest.main()