diff --git a/README.md b/README.md index cece488ae..7c0d246a5 100644 --- a/README.md +++ b/README.md @@ -356,7 +356,7 @@ export OPENAI_API_KEY="sk-..." # config.yaml llm: api_base: "https://generativelanguage.googleapis.com/v1beta/openai/" - model: "gemini-2.5-pro" + primary_model: "gemini-2.5-pro" ``` ```bash @@ -372,7 +372,7 @@ export OPENAI_API_KEY="your-gemini-api-key" # config.yaml llm: api_base: "http://localhost:11434/v1" # Ollama - model: "codellama:7b" + primary_model: "codellama:7b" ``` @@ -396,7 +396,7 @@ export OPENAI_API_KEY="your-actual-key" ```yaml llm: api_base: "http://localhost:8000/v1" - model: "moa&readurls-o3" # Test-time compute + web access + primary_model: "moa&readurls-o3" # Test-time compute + web access ``` diff --git a/openevolve/config.py b/openevolve/config.py index 3d08ceb14..d68b701f8 100644 --- a/openevolve/config.py +++ b/openevolve/config.py @@ -489,6 +489,16 @@ def from_dict(cls, config_dict: Dict[str, Any]) -> "Config": del config_dict["llm"]["temperature"] if "top_p" in config_dict["llm"] and config_dict["llm"]["top_p"] is None: del config_dict["llm"]["top_p"] + # 'model' is not a config field (models are 'llm.models' or + # 'llm.primary_model'), and dacite silently drops unknown keys. Left + # unchecked, such configs pass validation with an empty model ensemble + # and every LLM call later fails with "IndexError: list index out of + # range" (issue #427). Fail here with the correct spelling instead. + if isinstance(config_dict["llm"], dict) and "model" in config_dict["llm"]: + raise ValueError( + "Invalid config key 'llm.model'. Use 'llm.primary_model' or the " + "'llm.models' array instead (see README configuration section)." + ) config: Config = dacite.from_dict( data_class=cls, diff --git a/tests/test_llm_model_key_rejected.py b/tests/test_llm_model_key_rejected.py new file mode 100644 index 000000000..6439f8d85 --- /dev/null +++ b/tests/test_llm_model_key_rejected.py @@ -0,0 +1,63 @@ +""" +Tests for rejection of the legacy/mistaken 'llm.model' config key (issue #427). + +The LLM config has no 'model' field - models are configured via 'llm.models' +or 'llm.primary_model'. dacite silently drops unknown keys, so a config using +'model:' (as several README provider examples taught) produced an empty model +ensemble that later crashed every LLM call with "IndexError: list index out +of range". Config.from_dict must reject the key with an actionable error. +""" + +import unittest + +from openevolve.config import Config + + +class TestLLMModelKeyRejected(unittest.TestCase): + """The 'llm.model' key must fail loudly instead of building an empty ensemble""" + + def test_model_key_raises_helpful_error(self): + """A bare llm.model key raises ValueError pointing at the right keys""" + with self.assertRaises(ValueError) as ctx: + Config.from_dict({"llm": {"model": "glm-ocr"}}) + self.assertIn("primary_model", str(ctx.exception)) + self.assertIn("llm.models", str(ctx.exception)) + + def test_model_key_with_other_keys_still_raises(self): + """llm.model raises even when mixed into an otherwise valid config""" + config_dict = { + "max_iterations": 100, + "llm": { + "api_base": "http://localhost:3000/api/", + "api_key": "test", + "model": "glm-ocr", + "temperature": 0.7, + }, + } + with self.assertRaises(ValueError) as ctx: + Config.from_dict(config_dict) + self.assertIn("llm.model", str(ctx.exception)) + + def test_primary_model_config_still_valid(self): + """The documented primary_model spelling keeps working""" + config = Config.from_dict( + {"llm": {"primary_model": "glm-ocr", "api_base": "http://localhost:3000/api/"}} + ) + self.assertEqual(len(config.llm.models), 1) + self.assertEqual(config.llm.models[0].name, "glm-ocr") + + def test_models_array_config_still_valid(self): + """The models array spelling keeps working""" + config = Config.from_dict({"llm": {"models": [{"name": "gpt-4o-mini", "weight": 1.0}]}}) + self.assertEqual(len(config.llm.models), 1) + self.assertEqual(config.llm.models[0].name, "gpt-4o-mini") + + def test_code_constructed_config_unaffected(self): + """Internally constructed Config() (which never goes through from_dict) + keeps its empty default ensemble - that path is not a user error""" + config = Config() + self.assertEqual(config.llm.models, []) + + +if __name__ == "__main__": + unittest.main()