Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -356,7 +356,7 @@ export OPENAI_API_KEY="sk-..."
# config.yaml
llm:
api_base: "https://generativelanguage.googleapis.com/v1beta/openai/"
model: "gemini-2.5-pro"
primary_model: "gemini-2.5-pro"
```

```bash
Expand All @@ -372,7 +372,7 @@ export OPENAI_API_KEY="your-gemini-api-key"
# config.yaml
llm:
api_base: "http://localhost:11434/v1" # Ollama
model: "codellama:7b"
primary_model: "codellama:7b"
```

</details>
Expand All @@ -396,7 +396,7 @@ export OPENAI_API_KEY="your-actual-key"
```yaml
llm:
api_base: "http://localhost:8000/v1"
model: "moa&readurls-o3" # Test-time compute + web access
primary_model: "moa&readurls-o3" # Test-time compute + web access
```

</details>
Expand Down
10 changes: 10 additions & 0 deletions openevolve/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -489,6 +489,16 @@ def from_dict(cls, config_dict: Dict[str, Any]) -> "Config":
del config_dict["llm"]["temperature"]
if "top_p" in config_dict["llm"] and config_dict["llm"]["top_p"] is None:
del config_dict["llm"]["top_p"]
# 'model' is not a config field (models are 'llm.models' or
# 'llm.primary_model'), and dacite silently drops unknown keys. Left
# unchecked, such configs pass validation with an empty model ensemble
# and every LLM call later fails with "IndexError: list index out of
# range" (issue #427). Fail here with the correct spelling instead.
if isinstance(config_dict["llm"], dict) and "model" in config_dict["llm"]:
raise ValueError(
"Invalid config key 'llm.model'. Use 'llm.primary_model' or the "
"'llm.models' array instead (see README configuration section)."
)

config: Config = dacite.from_dict(
data_class=cls,
Expand Down
63 changes: 63 additions & 0 deletions tests/test_llm_model_key_rejected.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
"""
Tests for rejection of the legacy/mistaken 'llm.model' config key (issue #427).

The LLM config has no 'model' field - models are configured via 'llm.models'
or 'llm.primary_model'. dacite silently drops unknown keys, so a config using
'model:' (as several README provider examples taught) produced an empty model
ensemble that later crashed every LLM call with "IndexError: list index out
of range". Config.from_dict must reject the key with an actionable error.
"""

import unittest

from openevolve.config import Config


class TestLLMModelKeyRejected(unittest.TestCase):
"""The 'llm.model' key must fail loudly instead of building an empty ensemble"""

def test_model_key_raises_helpful_error(self):
"""A bare llm.model key raises ValueError pointing at the right keys"""
with self.assertRaises(ValueError) as ctx:
Config.from_dict({"llm": {"model": "glm-ocr"}})
self.assertIn("primary_model", str(ctx.exception))
self.assertIn("llm.models", str(ctx.exception))

def test_model_key_with_other_keys_still_raises(self):
"""llm.model raises even when mixed into an otherwise valid config"""
config_dict = {
"max_iterations": 100,
"llm": {
"api_base": "http://localhost:3000/api/",
"api_key": "test",
"model": "glm-ocr",
"temperature": 0.7,
},
}
with self.assertRaises(ValueError) as ctx:
Config.from_dict(config_dict)
self.assertIn("llm.model", str(ctx.exception))

def test_primary_model_config_still_valid(self):
"""The documented primary_model spelling keeps working"""
config = Config.from_dict(
{"llm": {"primary_model": "glm-ocr", "api_base": "http://localhost:3000/api/"}}
)
self.assertEqual(len(config.llm.models), 1)
self.assertEqual(config.llm.models[0].name, "glm-ocr")

def test_models_array_config_still_valid(self):
"""The models array spelling keeps working"""
config = Config.from_dict({"llm": {"models": [{"name": "gpt-4o-mini", "weight": 1.0}]}})
self.assertEqual(len(config.llm.models), 1)
self.assertEqual(config.llm.models[0].name, "gpt-4o-mini")

def test_code_constructed_config_unaffected(self):
"""Internally constructed Config() (which never goes through from_dict)
keeps its empty default ensemble - that path is not a user error"""
config = Config()
self.assertEqual(config.llm.models, [])


if __name__ == "__main__":
unittest.main()
Loading