From 5649a5f2258b471b1128e4216fc15a6e7d1e304c Mon Sep 17 00:00:00 2001 From: Dmytro Nikolaiev Date: Mon, 8 Dec 2025 19:03:40 -0500 Subject: [PATCH 1/6] feat: implementation --- openevolve/config.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/openevolve/config.py b/openevolve/config.py index 762356d1f..b317ddce0 100644 --- a/openevolve/config.py +++ b/openevolve/config.py @@ -19,7 +19,9 @@ class LLMModelConfig: # API configuration api_base: str = None - api_key: Optional[str] = None + api_key: Optional[Union[str, Dict[str, str]]] = ( + None # either api_key or from_env: api_key_var_name + ) name: str = None # Custom LLM client @@ -45,6 +47,20 @@ class LLMModelConfig: # Reasoning parameters reasoning_effort: Optional[str] = None + def __post_init__(self): + """Post-initialization to set up API key""" + if self.api_key is not None: + if isinstance(self.api_key, dict): + env_var_name = self.api_key.get("from_env") + if env_var_name is None: + raise ValueError( + f"api_key is a dict but 'from_env' key is not set: {self.api_key}" + ) + key = os.environ.get(env_var_name) + if key is None: + raise ValueError(f"Environment variable {env_var_name} is not set") + self.api_key = key + @dataclass class LLMConfig(LLMModelConfig): @@ -81,6 +97,8 @@ class LLMConfig(LLMModelConfig): def __post_init__(self): """Post-initialization to set up model configurations""" + super().__post_init__() # resolve from_env for api_key at LLMConfig level + # Handle backward compatibility for primary_model(_weight) and secondary_model(_weight). if self.primary_model: # Create primary model From 70357a4ee6ca30daf3168faadf87164e0c78e09b Mon Sep 17 00:00:00 2001 From: Dmytro Nikolaiev Date: Mon, 8 Dec 2025 19:03:43 -0500 Subject: [PATCH 2/6] test: implementation --- tests/test_api_key_from_env.py | 178 +++++++++++++++++++++++++++++++++ 1 file changed, 178 insertions(+) create mode 100644 tests/test_api_key_from_env.py diff --git a/tests/test_api_key_from_env.py b/tests/test_api_key_from_env.py new file mode 100644 index 000000000..ef7831489 --- /dev/null +++ b/tests/test_api_key_from_env.py @@ -0,0 +1,178 @@ +""" +Tests for api_key from_env configuration feature +""" + +import os +import tempfile +import unittest + +from openevolve.config import Config, LLMModelConfig + + +class TestApiKeyFromEnv(unittest.TestCase): + """Tests for api_key from_env parameter handling in configuration""" + + def setUp(self): + """Set up test environment variables""" + self.test_env_var = "TEST_OPENEVOLVE_API_KEY" + self.test_api_key = "test-secret-key-12345" + os.environ[self.test_env_var] = self.test_api_key + + def tearDown(self): + """Clean up test environment variables""" + if self.test_env_var in os.environ: + del os.environ[self.test_env_var] + + def test_api_key_from_env_in_model_config(self): + """Test that api_key can be loaded from environment variable via from_env""" + model_config = LLMModelConfig(name="test-model", api_key={"from_env": self.test_env_var}) + + self.assertEqual(model_config.api_key, self.test_api_key) + + def test_api_key_direct_value(self): + """Test that direct api_key value still works""" + direct_key = "direct-api-key-value" + model_config = LLMModelConfig(name="test-model", api_key=direct_key) + + self.assertEqual(model_config.api_key, direct_key) + + def test_api_key_none(self): + """Test that api_key can be None""" + model_config = LLMModelConfig(name="test-model", api_key=None) + + self.assertIsNone(model_config.api_key) + + def test_api_key_from_env_missing_env_var(self): + """Test that missing environment variable raises ValueError""" + with self.assertRaises(ValueError) as context: + LLMModelConfig(name="test-model", api_key={"from_env": "NONEXISTENT_ENV_VAR_12345"}) + + self.assertIn("NONEXISTENT_ENV_VAR_12345", str(context.exception)) + self.assertIn("is not set", str(context.exception)) + + def test_api_key_dict_without_from_env_key(self): + """Test that dict without from_env key raises ValueError""" + with self.assertRaises(ValueError) as context: + LLMModelConfig(name="test-model", api_key={"wrong_key": "value"}) + + self.assertIn("from_env", str(context.exception)) + + def test_api_key_from_env_in_llm_config(self): + """Test that api_key from_env works at LLM config level""" + yaml_config = { + "log_level": "INFO", + "llm": { + "api_base": "https://api.openai.com/v1", + "api_key": {"from_env": self.test_env_var}, + "models": [{"name": "test-model", "weight": 1.0}], + }, + } + + config = Config.from_dict(yaml_config) + + self.assertEqual(config.llm.api_key, self.test_api_key) + # Models should inherit the resolved api_key + self.assertEqual(config.llm.models[0].api_key, self.test_api_key) + + def test_api_key_from_env_per_model(self): + """Test that api_key from_env can be specified per model""" + # Set up a second env var for testing + second_env_var = "TEST_OPENEVOLVE_API_KEY_2" + second_api_key = "second-secret-key-67890" + os.environ[second_env_var] = second_api_key + + try: + yaml_config = { + "log_level": "INFO", + "llm": { + "api_base": "https://api.openai.com/v1", + "models": [ + { + "name": "model-1", + "weight": 1.0, + "api_key": {"from_env": self.test_env_var}, + }, + {"name": "model-2", "weight": 0.5, "api_key": {"from_env": second_env_var}}, + ], + }, + } + + config = Config.from_dict(yaml_config) + + self.assertEqual(config.llm.models[0].api_key, self.test_api_key) + self.assertEqual(config.llm.models[1].api_key, second_api_key) + finally: + if second_env_var in os.environ: + del os.environ[second_env_var] + + def test_api_key_from_env_in_evaluator_models(self): + """Test that api_key from_env works in evaluator_models""" + yaml_config = { + "log_level": "INFO", + "llm": { + "api_base": "https://api.openai.com/v1", + "models": [{"name": "evolution-model", "weight": 1.0, "api_key": "direct-key"}], + "evaluator_models": [ + { + "name": "evaluator-model", + "weight": 1.0, + "api_key": {"from_env": self.test_env_var}, + } + ], + }, + } + + config = Config.from_dict(yaml_config) + + self.assertEqual(config.llm.evaluator_models[0].api_key, self.test_api_key) + + def test_yaml_file_loading_with_from_env(self): + """Test loading api_key from_env from actual YAML file""" + yaml_content = f""" +log_level: INFO +llm: + api_base: https://api.openai.com/v1 + api_key: + from_env: {self.test_env_var} + models: + - name: test-model + weight: 1.0 +""" + + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write(yaml_content) + f.flush() + + try: + config = Config.from_yaml(f.name) + self.assertEqual(config.llm.api_key, self.test_api_key) + finally: + os.unlink(f.name) + + def test_mixed_api_key_sources(self): + """Test mixing direct api_key and from_env in same config""" + yaml_config = { + "log_level": "INFO", + "llm": { + "api_base": "https://api.openai.com/v1", + "api_key": "llm-level-direct-key", + "models": [ + { + "name": "model-with-env", + "weight": 1.0, + "api_key": {"from_env": self.test_env_var}, + }, + {"name": "model-with-direct", "weight": 0.5, "api_key": "model-direct-key"}, + ], + }, + } + + config = Config.from_dict(yaml_config) + + self.assertEqual(config.llm.api_key, "llm-level-direct-key") + self.assertEqual(config.llm.models[0].api_key, self.test_api_key) + self.assertEqual(config.llm.models[1].api_key, "model-direct-key") + + +if __name__ == "__main__": + unittest.main() From f257016f676d5dd0001348893687d22476db45f6 Mon Sep 17 00:00:00 2001 From: Dmytro Nikolaiev Date: Mon, 8 Dec 2025 19:03:57 -0500 Subject: [PATCH 3/6] doc: from_env in default_config.yaml --- configs/default_config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configs/default_config.yaml b/configs/default_config.yaml index f46106d1d..87ac61c36 100644 --- a/configs/default_config.yaml +++ b/configs/default_config.yaml @@ -39,6 +39,9 @@ llm: # API configuration api_base: "https://generativelanguage.googleapis.com/v1beta/openai/" # Base URL for API (change for non-OpenAI models) api_key: null # API key (defaults to OPENAI_API_KEY env variable) + # Or use from_env to specify which environment variable to read from: + # api_key: + # from_env: GEMINI_API_KEY # Reads API key from $GEMINI_API_KEY # Generation parameters temperature: 0.7 # Temperature for generation (higher = more creative) From fd52490388ff9fb51b57c7390646c97f961f186c Mon Sep 17 00:00:00 2001 From: Dmytro Nikolaiev Date: Tue, 9 Dec 2025 12:46:38 -0500 Subject: [PATCH 4/6] feat: update to ${VAR} --- openevolve/config.py | 53 +++++++++++++++++++++++++++++++------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/openevolve/config.py b/openevolve/config.py index b317ddce0..543874496 100644 --- a/openevolve/config.py +++ b/openevolve/config.py @@ -3,6 +3,7 @@ """ import os +import re from dataclasses import asdict, dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union @@ -13,15 +14,45 @@ from openevolve.llm.base import LLMInterface +_ENV_VAR_PATTERN = re.compile(r"^\$\{([^}]+)\}$") # ${VAR} + + +def _resolve_env_var(value: Optional[str]) -> Optional[str]: + """ + Resolve ${VAR} environment variable reference in a string value. + In current implementation pattern must match the entire string (e.g., "${OPENAI_API_KEY}"), + not embedded within other text. + + Args: + value: The string value that may contain ${VAR} syntax + + Returns: + The resolved value with environment variable expanded, or original value if no match + + Raises: + ValueError: If the environment variable is referenced but not set + """ + if value is None: + return None + + match = _ENV_VAR_PATTERN.match(value) + if not match: + return value + + var_name = match.group(1) + env_value = os.environ.get(var_name) + if env_value is None: + raise ValueError(f"Environment variable {var_name} is not set") + return env_value + + @dataclass class LLMModelConfig: """Configuration for a single LLM model""" # API configuration api_base: str = None - api_key: Optional[Union[str, Dict[str, str]]] = ( - None # either api_key or from_env: api_key_var_name - ) + api_key: Optional[str] = None name: str = None # Custom LLM client @@ -48,18 +79,8 @@ class LLMModelConfig: reasoning_effort: Optional[str] = None def __post_init__(self): - """Post-initialization to set up API key""" - if self.api_key is not None: - if isinstance(self.api_key, dict): - env_var_name = self.api_key.get("from_env") - if env_var_name is None: - raise ValueError( - f"api_key is a dict but 'from_env' key is not set: {self.api_key}" - ) - key = os.environ.get(env_var_name) - if key is None: - raise ValueError(f"Environment variable {env_var_name} is not set") - self.api_key = key + """Post-initialization to resolve ${VAR} env var references in api_key""" + self.api_key = _resolve_env_var(self.api_key) @dataclass @@ -97,7 +118,7 @@ class LLMConfig(LLMModelConfig): def __post_init__(self): """Post-initialization to set up model configurations""" - super().__post_init__() # resolve from_env for api_key at LLMConfig level + super().__post_init__() # Resolve ${VAR} in api_key at LLMConfig level # Handle backward compatibility for primary_model(_weight) and secondary_model(_weight). if self.primary_model: From 56c80ce62d1ad0b68394c501f9c04e7457544567 Mon Sep 17 00:00:00 2001 From: Dmytro Nikolaiev Date: Tue, 9 Dec 2025 12:46:52 -0500 Subject: [PATCH 5/6] test: update to ${VAR} --- tests/test_api_key_from_env.py | 93 ++++++++++++++++++++-------------- 1 file changed, 54 insertions(+), 39 deletions(-) diff --git a/tests/test_api_key_from_env.py b/tests/test_api_key_from_env.py index ef7831489..d126d13dc 100644 --- a/tests/test_api_key_from_env.py +++ b/tests/test_api_key_from_env.py @@ -1,16 +1,16 @@ """ -Tests for api_key from_env configuration feature +Tests for api_key ${VAR} environment variable substitution in configuration. """ import os import tempfile import unittest -from openevolve.config import Config, LLMModelConfig +from openevolve.config import Config, LLMModelConfig, _resolve_env_var -class TestApiKeyFromEnv(unittest.TestCase): - """Tests for api_key from_env parameter handling in configuration""" +class TestEnvVarSubstitution(unittest.TestCase): + """Tests for ${VAR} environment variable substitution in api_key fields""" def setUp(self): """Set up test environment variables""" @@ -23,9 +23,32 @@ def tearDown(self): if self.test_env_var in os.environ: del os.environ[self.test_env_var] - def test_api_key_from_env_in_model_config(self): - """Test that api_key can be loaded from environment variable via from_env""" - model_config = LLMModelConfig(name="test-model", api_key={"from_env": self.test_env_var}) + def test_resolve_env_var_with_match(self): + """Test that _resolve_env_var resolves ${VAR} syntax""" + result = _resolve_env_var(f"${{{self.test_env_var}}}") + self.assertEqual(result, self.test_api_key) + + def test_resolve_env_var_no_match(self): + """Test that strings without ${VAR} are returned unchanged""" + result = _resolve_env_var("regular-api-key") + self.assertEqual(result, "regular-api-key") + + def test_resolve_env_var_none(self): + """Test that None is returned unchanged""" + result = _resolve_env_var(None) + self.assertIsNone(result) + + def test_resolve_env_var_missing_var(self): + """Test that missing environment variable raises ValueError""" + with self.assertRaises(ValueError) as context: + _resolve_env_var("${NONEXISTENT_ENV_VAR_12345}") + + self.assertIn("NONEXISTENT_ENV_VAR_12345", str(context.exception)) + self.assertIn("is not set", str(context.exception)) + + def test_api_key_env_var_in_model_config(self): + """Test that api_key ${VAR} works in LLMModelConfig""" + model_config = LLMModelConfig(name="test-model", api_key=f"${{{self.test_env_var}}}") self.assertEqual(model_config.api_key, self.test_api_key) @@ -42,28 +65,13 @@ def test_api_key_none(self): self.assertIsNone(model_config.api_key) - def test_api_key_from_env_missing_env_var(self): - """Test that missing environment variable raises ValueError""" - with self.assertRaises(ValueError) as context: - LLMModelConfig(name="test-model", api_key={"from_env": "NONEXISTENT_ENV_VAR_12345"}) - - self.assertIn("NONEXISTENT_ENV_VAR_12345", str(context.exception)) - self.assertIn("is not set", str(context.exception)) - - def test_api_key_dict_without_from_env_key(self): - """Test that dict without from_env key raises ValueError""" - with self.assertRaises(ValueError) as context: - LLMModelConfig(name="test-model", api_key={"wrong_key": "value"}) - - self.assertIn("from_env", str(context.exception)) - - def test_api_key_from_env_in_llm_config(self): - """Test that api_key from_env works at LLM config level""" + def test_api_key_env_var_in_llm_config(self): + """Test that api_key ${VAR} works at LLM config level""" yaml_config = { "log_level": "INFO", "llm": { "api_base": "https://api.openai.com/v1", - "api_key": {"from_env": self.test_env_var}, + "api_key": f"${{{self.test_env_var}}}", "models": [{"name": "test-model", "weight": 1.0}], }, } @@ -74,8 +82,8 @@ def test_api_key_from_env_in_llm_config(self): # Models should inherit the resolved api_key self.assertEqual(config.llm.models[0].api_key, self.test_api_key) - def test_api_key_from_env_per_model(self): - """Test that api_key from_env can be specified per model""" + def test_api_key_env_var_per_model(self): + """Test that api_key ${VAR} can be specified per model""" # Set up a second env var for testing second_env_var = "TEST_OPENEVOLVE_API_KEY_2" second_api_key = "second-secret-key-67890" @@ -90,9 +98,13 @@ def test_api_key_from_env_per_model(self): { "name": "model-1", "weight": 1.0, - "api_key": {"from_env": self.test_env_var}, + "api_key": f"${{{self.test_env_var}}}", + }, + { + "name": "model-2", + "weight": 0.5, + "api_key": f"${{{second_env_var}}}", }, - {"name": "model-2", "weight": 0.5, "api_key": {"from_env": second_env_var}}, ], }, } @@ -105,8 +117,8 @@ def test_api_key_from_env_per_model(self): if second_env_var in os.environ: del os.environ[second_env_var] - def test_api_key_from_env_in_evaluator_models(self): - """Test that api_key from_env works in evaluator_models""" + def test_api_key_env_var_in_evaluator_models(self): + """Test that api_key ${VAR} works in evaluator_models""" yaml_config = { "log_level": "INFO", "llm": { @@ -116,7 +128,7 @@ def test_api_key_from_env_in_evaluator_models(self): { "name": "evaluator-model", "weight": 1.0, - "api_key": {"from_env": self.test_env_var}, + "api_key": f"${{{self.test_env_var}}}", } ], }, @@ -126,14 +138,13 @@ def test_api_key_from_env_in_evaluator_models(self): self.assertEqual(config.llm.evaluator_models[0].api_key, self.test_api_key) - def test_yaml_file_loading_with_from_env(self): - """Test loading api_key from_env from actual YAML file""" + def test_yaml_file_loading_with_env_var(self): + """Test loading api_key ${VAR} from actual YAML file""" yaml_content = f""" log_level: INFO llm: api_base: https://api.openai.com/v1 - api_key: - from_env: {self.test_env_var} + api_key: ${{{self.test_env_var}}} models: - name: test-model weight: 1.0 @@ -150,7 +161,7 @@ def test_yaml_file_loading_with_from_env(self): os.unlink(f.name) def test_mixed_api_key_sources(self): - """Test mixing direct api_key and from_env in same config""" + """Test mixing direct api_key and ${VAR} in same config""" yaml_config = { "log_level": "INFO", "llm": { @@ -160,9 +171,13 @@ def test_mixed_api_key_sources(self): { "name": "model-with-env", "weight": 1.0, - "api_key": {"from_env": self.test_env_var}, + "api_key": f"${{{self.test_env_var}}}", + }, + { + "name": "model-with-direct", + "weight": 0.5, + "api_key": "model-direct-key", }, - {"name": "model-with-direct", "weight": 0.5, "api_key": "model-direct-key"}, ], }, } From ff162f47d11b093c7ddf140c5e196a74082ffd56 Mon Sep 17 00:00:00 2001 From: Dmytro Nikolaiev Date: Tue, 9 Dec 2025 12:47:03 -0500 Subject: [PATCH 6/6] doc: update default_config.yaml --- configs/default_config.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/configs/default_config.yaml b/configs/default_config.yaml index 87ac61c36..928465bf5 100644 --- a/configs/default_config.yaml +++ b/configs/default_config.yaml @@ -39,9 +39,8 @@ llm: # API configuration api_base: "https://generativelanguage.googleapis.com/v1beta/openai/" # Base URL for API (change for non-OpenAI models) api_key: null # API key (defaults to OPENAI_API_KEY env variable) - # Or use from_env to specify which environment variable to read from: - # api_key: - # from_env: GEMINI_API_KEY # Reads API key from $GEMINI_API_KEY + # or use ${VAR} syntax to specify which environment variable to read from: + # api_key: ${GEMINI_API_KEY} # Reads API key from $GEMINI_API_KEY # Generation parameters temperature: 0.7 # Temperature for generation (higher = more creative)