diff --git a/README.md b/README.md index 740909f5e..82546c86c 100644 --- a/README.md +++ b/README.md @@ -119,14 +119,17 @@ def bubble_sort(arr): arr[j], arr[j+1] = arr[j+1], arr[j] return arr -result = evolve_function( - bubble_sort, - test_cases=[([3,1,2], [1,2,3]), ([5,2,8], [2,5,8])], - iterations=50 -) -print(f"Evolved sorting algorithm: {result.best_code}") +if __name__ == '__main__': + result = evolve_function( + bubble_sort, + test_cases=[([3,1,2], [1,2,3]), ([5,2,8], [2,5,8])], + iterations=50 + ) + print(f"Evolved sorting algorithm: {result.best_code}") ``` +> **Note:** On macOS and Windows, Python uses `spawn` for multiprocessing. You must wrap evolution calls in `if __name__ == '__main__':` to avoid subprocess bootstrap errors. + **Prefer Docker?** See the [Installation & Setup](#installation--setup) section for Docker options. ## See It In Action diff --git a/openevolve/api.py b/openevolve/api.py index 65c370206..b00acd83a 100644 --- a/openevolve/api.py +++ b/openevolve/api.py @@ -3,6 +3,7 @@ """ import asyncio +import pickle import tempfile import os import uuid @@ -246,47 +247,33 @@ def _prepare_evaluator( # If it's a callable, create a wrapper module if callable(evaluator): - # Try to get the source code of the callable so it can be serialized - # into a standalone file that works in subprocesses try: - func_source = inspect.getsource(evaluator) - # Dedent in case the function was defined inside another scope - import textwrap + import cloudpickle as _pickle_mod + except ImportError: + _pickle_mod = pickle - func_source = textwrap.dedent(func_source) - func_name = evaluator.__name__ + # Serialize the callable to a file so subprocess workers can load it + if temp_dir is None: + temp_dir = tempfile.gettempdir() - # Build a self-contained evaluator module with the function source - # and an evaluate() entry point that calls it - evaluator_code = f""" -# Auto-generated evaluator from user-provided callable -import importlib.util -import sys -import os -import copy -import json -import time + pickle_path = os.path.join(temp_dir, f"evaluator_{uuid.uuid4().hex[:8]}.pkl") + with open(pickle_path, "wb") as pf: + _pickle_mod.dump(evaluator, pf) + temp_files.append(pickle_path) -{func_source} - -def evaluate(program_path): - '''Wrapper that calls the user-provided evaluator function''' - return {func_name}(program_path) -""" - except (OSError, TypeError): - # If we can't get source (e.g. built-in, lambda, or closure), - # fall back to the globals-based approach - evaluator_id = f"_openevolve_evaluator_{uuid.uuid4().hex[:8]}" - globals()[evaluator_id] = evaluator + evaluator_code = f""" +# Wrapper for user-provided evaluator function (serialized to disk for cross-process access) +import pickle - evaluator_code = f""" -# Wrapper for user-provided evaluator function -import {__name__} as api_module +_cached_evaluator = None def evaluate(program_path): - '''Wrapper for user-provided evaluator function''' - user_evaluator = getattr(api_module, '{evaluator_id}') - return user_evaluator(program_path) + '''Wrapper that loads the evaluator from a pickle file''' + global _cached_evaluator + if _cached_evaluator is None: + with open({pickle_path!r}, 'rb') as f: + _cached_evaluator = pickle.load(f) + return _cached_evaluator(program_path) """ else: # Treat as code string diff --git a/pyproject.toml b/pyproject.toml index f95cd439c..412ecc6a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "tqdm>=4.64.0", "flask", "dacite>=1.9.2", + "cloudpickle>=2.0.0", ] [project.optional-dependencies] diff --git a/tests/test_api.py b/tests/test_api.py index 9316efc1d..cb001a81b 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -121,50 +121,12 @@ def my_evaluator(program_path): result = _prepare_evaluator(my_evaluator, self.temp_dir, temp_files) self.assertTrue(os.path.exists(result)) - self.assertEqual(len(temp_files), 1) + # 2 temp files: the pickle file + the wrapper .py + self.assertEqual(len(temp_files), 2) with open(result, 'r') as f: content = f.read() self.assertIn("def evaluate(program_path)", content) - self.assertIn("my_evaluator", content) - - def test_prepare_evaluator_callable_works_in_subprocess(self): - """Test that callable evaluator can be executed in a subprocess""" - import subprocess - import sys - - def my_evaluator(program_path): - return {"score": 0.8, "combined_score": 0.8} - - temp_files = [] - eval_file = _prepare_evaluator(my_evaluator, self.temp_dir, temp_files) - - # Write a dummy program file for the evaluator to receive - program_file = os.path.join(self.temp_dir, "dummy_program.py") - with open(program_file, 'w') as f: - f.write("x = 1\n") - - # Run the evaluator in a subprocess (simulating process-based parallelism) - test_script = os.path.join(self.temp_dir, "run_eval.py") - with open(test_script, 'w') as f: - f.write(f""" -import sys -import importlib.util -spec = importlib.util.spec_from_file_location("evaluator", {eval_file!r}) -mod = importlib.util.module_from_spec(spec) -spec.loader.exec_module(mod) -result = mod.evaluate({program_file!r}) -assert isinstance(result, dict), f"Expected dict, got {{type(result)}}" -assert result["score"] == 0.8, f"Expected score 0.8, got {{result['score']}}" -print("OK") -""") - - proc = subprocess.run( - [sys.executable, test_script], - capture_output=True, text=True, timeout=10 - ) - self.assertEqual(proc.returncode, 0, f"Subprocess failed: {proc.stderr}") - self.assertIn("OK", proc.stdout) def test_prepare_evaluator_from_string(self): """Test _prepare_evaluator with code string""" @@ -382,5 +344,77 @@ def test_run_evolution_cleanup_false(self): mock_async.assert_called_once() +class TestEvaluatorCrossProcess(unittest.TestCase): + """Test that callable evaluators work across process boundaries""" + + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + + def tearDown(self): + import shutil + shutil.rmtree(self.temp_dir, ignore_errors=True) + + def test_callable_evaluator_works_in_subprocess(self): + """Test that a callable evaluator serialized by _prepare_evaluator + can be loaded and executed in a separate process (simulating + ProcessPoolExecutor workers).""" + def my_evaluator(program_path): + return {"score": 0.42, "passed": True} + + temp_files = [] + eval_file = _prepare_evaluator(my_evaluator, self.temp_dir, temp_files) + + # Load and run the evaluator in a subprocess — this is what + # process_parallel.py workers do. + import subprocess, sys, json + result = subprocess.run( + [ + sys.executable, "-c", + f""" +import importlib.util, json, sys +spec = importlib.util.spec_from_file_location("eval_mod", {eval_file!r}) +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) +print(json.dumps(mod.evaluate("dummy_path.py"))) +""" + ], + capture_output=True, text=True, timeout=10 + ) + self.assertEqual(result.returncode, 0, f"Subprocess failed: {result.stderr}") + metrics = json.loads(result.stdout.strip()) + self.assertAlmostEqual(metrics["score"], 0.42) + self.assertTrue(metrics["passed"]) + + def test_callable_evaluator_with_closure(self): + """Test that a closure (capturing local variables) works across processes.""" + threshold = 0.5 + func_name = "my_func" + + def closure_evaluator(program_path): + return {"score": threshold, "func": func_name} + + temp_files = [] + eval_file = _prepare_evaluator(closure_evaluator, self.temp_dir, temp_files) + + import subprocess, sys, json + result = subprocess.run( + [ + sys.executable, "-c", + f""" +import importlib.util, json +spec = importlib.util.spec_from_file_location("eval_mod", {eval_file!r}) +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) +print(json.dumps(mod.evaluate("dummy.py"))) +""" + ], + capture_output=True, text=True, timeout=10 + ) + self.assertEqual(result.returncode, 0, f"Subprocess failed: {result.stderr}") + metrics = json.loads(result.stdout.strip()) + self.assertAlmostEqual(metrics["score"], 0.5) + self.assertEqual(metrics["func"], "my_func") + + if __name__ == '__main__': unittest.main() \ No newline at end of file