Merge pull request #1279 from automl/1264-bug-reset-after-initialization

1264 bug reset after initialization

Author: LukasFehring

docs/advanced_usage/11_reproducibility.md

@@ -1,3 +1,6 @@
 # Reproducibility
 
-Reproducibility can only be ensured if one worker is used and no time (wallclock or CPU time) is involved.
+Reproducibility can only be ensured if one worker is used and no time (wallclock or CPU time) is involved.
+
+!!! warning
+    SMBO.reset() will not seed smac with the original seed. If you want to have a full reset, please set the seed again after calling reset.

examples/1_basics/3_ask_and_tell.py

@@ -62,7 +62,7 @@ def train(self, config: Configuration, seed: int = 0) -> float:
     )
 
     # We can ask SMAC which trials should be evaluated next
-    for _ in range(10):
+    for _ in range(30):
         info = smac.ask()
         assert info.seed is not None
 

smac/main/smbo.py

@@ -325,8 +325,8 @@ def optimize(self, *, data_to_scatter: dict[str, Any] | None = None) -> Configur
 
             # Some statistics
             logger.debug(
-                f"Remaining wallclock time: {self.remaining_walltime}; "
-                f"Remaining cpu time: {self.remaining_cputime}; "
+                f"Remaining wallclock time: {self.remaining_walltime}, "
+                f"Remaining cpu time: {self.remaining_cputime}, "
                 f"Remaining trials: {self.remaining_trials}"
             )
 
@@ -375,6 +375,7 @@ def reset(self) -> None:
         # We also reset runhistory and intensifier here
         self._runhistory.reset()
         self._intensifier.reset()
+        self._trial_generator = iter(self._intensifier)
 
     def exists(self, filename: str | Path) -> bool:
         """Checks if the files associated with the run already exist.
@@ -538,7 +539,7 @@ def _initialize_state(self) -> None:
                     )
                     logger.info(
                         f"Found old run in `{self._scenario.output_directory}` but it is not the same as the current "
-                        f"one:\n{diff}"
+                        f"one: \n{diff}"
                     )
 
                     feedback = input(

tests/test_main/test_smbo.py

@@ -5,11 +5,17 @@
 
 def test_termination_cost_threshold(rosenbrock):
     termination_cost_threshold = 100
-    scenario = Scenario(rosenbrock.configspace, n_trials=200, termination_cost_threshold=termination_cost_threshold)
+    scenario = Scenario(
+        rosenbrock.configspace,
+        n_trials=200,
+        termination_cost_threshold=termination_cost_threshold,
+    )
     smac = HyperparameterOptimizationFacade(
         scenario,
         rosenbrock.train,
-        intensifier=HyperparameterOptimizationFacade.get_intensifier(scenario, max_config_calls=1),
+        intensifier=HyperparameterOptimizationFacade.get_intensifier(
+            scenario, max_config_calls=1
+        ),
         overwrite=True,
     )
     i = smac.optimize()
@@ -55,3 +61,29 @@ def test_termination_cost_threshold_with_fidelities(rosenbrock):
     assert config == i
     assert counter == 1
     assert smac.validate(i) < termination_cost_threshold
+
+
+def test_smbo_reset(rosenbrock):
+    scenario = Scenario(rosenbrock.configspace, n_trials=3)
+    smac = HyperparameterOptimizationFacade(
+        scenario,
+        rosenbrock.train,
+        intensifier=HyperparameterOptimizationFacade.get_intensifier(
+            scenario, max_config_calls=1
+        ),
+        overwrite=True,
+    )
+
+    smac.optimize()
+    runhistory_len_before_reset = len(smac.runhistory)
+    assert runhistory_len_before_reset == 3
+
+    smac.optimizer.reset()
+
+    assert smac.optimizer._used_target_function_walltime == 0.0
+    assert smac.optimizer._used_target_function_cputime == 0.0
+    assert len(smac.runhistory) == 0
+
+    smac.optimize()
+    runhistory_len_after_reset = len(smac.runhistory)
+    assert runhistory_len_after_reset == 3