feat: import pre-evaluated trials

.gitignore

@@ -52,3 +52,5 @@ path
 .data
 
 .trace.lock
+
+copilot/

neps/__init__.py

@@ -1,11 +1,12 @@
-from neps.api import run, save_pipeline_results
+from neps.api import import_trials, run, save_pipeline_results
 from neps.optimizers import algorithms
 from neps.optimizers.ask_and_tell import AskAndTell
 from neps.optimizers.optimizer import SampledConfig
 from neps.plot.plot import plot
 from neps.plot.tensorboard_eval import tblogger
 from neps.space import Categorical, Constant, Float, Integer, SearchSpace
 from neps.state import BudgetInfo, Trial
+from neps.state.pipeline_eval import UserResultDict
 from neps.status.status import status
 from neps.utils.files import load_and_merge_yamls as load_yamls
 
@@ -19,7 +20,9 @@
     "SampledConfig",
     "SearchSpace",
     "Trial",
+    "UserResultDict",
     "algorithms",
+    "import_trials",
     "load_yamls",
     "plot",
     "run",

neps/api.py

@@ -4,22 +4,26 @@
 
 import logging
 import warnings
-from collections.abc import Callable, Mapping
+from collections.abc import Callable, Mapping, Sequence
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Concatenate, Literal
 
+from neps.normalization import _normalize_imported_config
 from neps.optimizers import AskFunction, OptimizerChoice, load_optimizer
 from neps.runtime import _launch_runtime, _save_results
 from neps.space.parsing import convert_to_space
+from neps.state import NePSState, OptimizationState, SeedSnapshot
 from neps.status.status import post_run_csv, trajectory_of_improvements
 from neps.utils.common import dynamic_load_object
+from neps.validation import _validate_imported_config, _validate_imported_result
 
 if TYPE_CHECKING:
     from ConfigSpace import ConfigurationSpace
 
     from neps.optimizers.algorithms import CustomOptimizer
     from neps.space import Parameter, SearchSpace
     from neps.state import EvaluatePipelineReturn
+    from neps.state.pipeline_eval import UserResultDict
 
 logger = logging.getLogger(__name__)
 
@@ -41,7 +45,7 @@ def run(  # noqa: C901, D417, PLR0913
     continue_until_max_evaluation_completed: bool = False,
     cost_to_spend: int | float | None = None,
     max_cost_total: int | float | None = None,
-    fidelities_to_spend: int | None = None,
+    fidelities_to_spend: int | float | None = None,
     ignore_errors: bool = False,
     objective_value_on_error: float | None = None,
     cost_value_on_error: float | None = None,
@@ -89,7 +93,7 @@ def evaluate_pipeline(some_parameter: float) -> float:
             ),
             "learning_rate": neps.Float(    # log spaced float
                 lower=1e-5,
-                uperr=1,
+                upper=1,
                 log=True
             ),
             "alpha": neps.Float(            # float with a prior
@@ -217,7 +221,7 @@ def evaluate_pipeline(some_parameter: float) -> float:
             returning a cost in the evaluate_pipeline function, e.g.,
             `return dict(loss=loss, cost=cost)`.
 
-        fidelities_to_spend: Number of evaluations in case of multi-fidelity after which to terminate.
+        fidelities_to_spend: accumulated fidelity spent in case of multi-fidelity after which to terminate.
 
         ignore_errors: Ignore hyperparameter settings that threw an error and do not raise
             an error. Error configs still count towards evaluations_to_spend.
@@ -591,4 +595,92 @@ def save_pipeline_results(
         )
 
 
-__all__ = ["run", "save_pipeline_results"]
+def import_trials(
+    pipeline_space: SearchSpace,
+    evaluated_trials: Sequence[tuple[Mapping[str, Any], UserResultDict],],
+    root_directory: Path | str,
+    optimizer: (
+        OptimizerChoice
+        | Mapping[str, Any]
+        | tuple[OptimizerChoice, Mapping[str, Any]]
+        | Callable[Concatenate[SearchSpace, ...], AskFunction]
+        | CustomOptimizer
+        | Literal["auto"]
+    ) = "auto",
+) -> None:
+    """Import externally evaluated trials into the optimization state.
+
+    This function allows you to add trials that have already been
+    evaluated outside of NePS into the current optimization run.
+    It validates and normalizes the provided configurations,
+    removes duplicates, and updates the optimization state accordingly.
+
+    Args:
+        pipeline_space (SearchSpace): The search space used for the optimization.
+        evaluated_trials (Sequence[tuple[Mapping[str, Any], UserResultDict]]):
+            A sequence of tuples, each containing a configuration dictionary
+            and its corresponding result.
+        root_directory (Path or str): The root directory of the NePS run.
+        optimizer: The optimizer to use for importing trials.
+            Can be a string, mapping, tuple, callable, or CustomOptimizer.
+            Defaults to "auto".
+
+    Returns:
+        None
+
+    Raises:
+        ValueError: If any configuration or result is invalid.
+        FileNotFoundError: If the root directory does not exist.
+        Exception: For unexpected errors during trial import.
+
+    Example:
+        >>> import neps
+        >>> from neps.state.pipeline_eval import UserResultDict
+        >>> pipeline_space = neps.SearchSpace({...})
+        >>> evaluated_trials = [
+        ...     ({"param1": 0.5, "param2": 10},
+        ...     UserResultDict(objective_to_minimize=-5.0)),
+        ... ]
+        >>> neps.import_trials(pipeline_space, evaluated_trials, "my_results")
+    """
+    if isinstance(root_directory, str):
+        root_directory = Path(root_directory)
+
+    optimizer_ask, optimizer_info = load_optimizer(optimizer, pipeline_space)
+
+    state = NePSState.create_or_load(
+        root_directory,
+        optimizer_info=optimizer_info,
+        optimizer_state=OptimizationState(
+            budget=None, seed_snapshot=SeedSnapshot.new_capture(), shared_state={}
+        ),
+    )
+
+    normalized_trials = []
+    for config, result in evaluated_trials:
+        _validate_imported_config(pipeline_space, config)
+        _validate_imported_result(result)
+        normalized_config = _normalize_imported_config(pipeline_space, config)
+        normalized_trials.append((normalized_config, result))
+
+    with state._trial_lock.lock():
+        state_trials = state._trial_repo.latest()
+        # remove duplicates
+        existing_configs = [
+            tuple(sorted(t.config.items())) for t in state_trials.values()
+        ]
+        normalized_trials = [
+            t
+            for t in normalized_trials
+            if tuple(sorted(t[0].items())) not in existing_configs
+        ]
+
+        imported_trials = optimizer_ask.import_trials(
+            external_evaluations=normalized_trials,
+            trials=state_trials,
+        )
+    # create Trial objects and add to state
+    state.lock_and_import_trials(imported_trials, worker_id="external")
+
+
+__all__ = ["import_trials", "run", "save_pipeline_results"]

neps/exceptions.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+from collections.abc import Mapping
 from typing import Any
 
 
@@ -42,3 +43,30 @@ class WorkerRaiseError(NePSError):
 
     Includes additional information on how to recover
     """
+
+
+class TrialValidationError(ValueError):
+    """Exception raised when a trial configuration fails validation.
+
+    Attributes:
+        config: The configuration dictionary that failed validation.
+        message: A detailed error message describing the validation failure.
+    """
+
+    def __init__(self, config: Mapping[str, Any], message: str, *args: Any) -> None:
+        """Initialize TrialValidationError.
+
+        Args:
+            config: The trial configuration that failed validation.
+            message: Description of the validation error.
+            *args: Additional arguments for the base Exception.
+        """
+        super().__init__(message, *args)
+        self.config = config
+        self.message = message
+
+    def __str__(self) -> str:
+        return (
+            f"Trial validation failed for configuration {self.config}. "
+            f"Reason: {self.message}"
+        )

neps/normalization.py

@@ -0,0 +1,39 @@
+"""Validation of the user inputs for NEPS APIs."""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Mapping
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from neps.space import SearchSpace
+
+logger = logging.getLogger(__name__)
+
+
+def _normalize_imported_config(space: SearchSpace, config: Mapping[str, float]) -> dict:
+    """Completes a configuration by adding default values for missing fidelities.
+
+    Args:
+        space: The search space defining the defaults.
+        config: The (potentially incomplete) configuration.
+
+    Returns:
+        A new, completed configuration dictionary.
+    """
+    all_param_keys = set(space.searchables.keys()) | set(space.fidelities.keys())
+
+    # copy to avoid modifying the original config
+    normalized_conf = dict(config)
+
+    for key, param in space.fidelities.items():
+        if key not in normalized_conf:
+            normalized_conf[key] = param.upper
+
+    extra_keys = set(normalized_conf.keys()) - all_param_keys
+    if extra_keys:
+        logger.warning(f"Unknown parameters in config: {extra_keys}, discarding them")
+        for k in extra_keys:
+            normalized_conf.pop(k)
+    return normalized_conf

neps/optimizers/algorithms.py

@@ -26,7 +26,6 @@
 
 import torch
 
-from neps.optimizers.ask_and_tell import AskAndTell  # noqa: F401
 from neps.optimizers.bayesian_optimization import BayesianOptimization
 from neps.optimizers.bracket_optimizer import BracketOptimizer, GPSampler
 from neps.optimizers.grid_search import GridSearch

neps/optimizers/bayesian_optimization.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
 
+import copy
 import itertools
 import math
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Literal
+from typing import TYPE_CHECKING, Any, Literal
 
 import numpy as np
 import torch
@@ -20,13 +21,14 @@
     fit_and_acquire_from_gp,
     make_default_single_obj_gp,
 )
-from neps.optimizers.optimizer import SampledConfig
+from neps.optimizers.optimizer import ImportedConfig, SampledConfig
 from neps.optimizers.utils.initial_design import make_initial_design
 
 if TYPE_CHECKING:
     from neps.sampling import Prior
     from neps.space import ConfigEncoder, SearchSpace
     from neps.state import BudgetInfo, Trial
+    from neps.state.pipeline_eval import UserResultDict
 
 
 def _pibo_exp_term(
@@ -96,7 +98,6 @@ def __call__(  # noqa: C901, PLR0912, PLR0915  # noqa: C901, PLR0912
         # If fidelities exist, sample from them as normal
         # This is a bit of a hack, as we set them to max fidelity
         # afterwards, but we need the complete space to sample
-
         if self.space.fidelity is not None:
             parameters = {**self.space.searchables, **self.space.fidelities}
         else:
@@ -293,6 +294,23 @@ def __call__(  # noqa: C901, PLR0912, PLR0915  # noqa: C901, PLR0912
         )
         return sampled_configs[0] if n is None else sampled_configs
 
+    def import_trials(
+        self,
+        external_evaluations: Sequence[tuple[Mapping[str, Any], UserResultDict]],
+        trials: Mapping[str, Trial],
+    ) -> list[ImportedConfig]:
+        trials_len = len(trials)
+        return [
+            ImportedConfig(
+                id=str(i),
+                config=copy.deepcopy(config),
+                result=copy.deepcopy(result),
+            )
+            for i, (config, result) in enumerate(
+                external_evaluations, start=trials_len + 1
+            )
+        ]
+
 
 def _get_reference_point(loss_vals: np.ndarray) -> np.ndarray:
     """Get the reference point from the completed Trials."""