diff --git a/pyomo/contrib/solver/common/base.py b/pyomo/contrib/solver/common/base.py index 63c1e97ffd6..76921588ef0 100644 --- a/pyomo/contrib/solver/common/base.py +++ b/pyomo/contrib/solver/common/base.py @@ -9,8 +9,9 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -from typing import Sequence, Dict, Optional, Mapping, List, Tuple import os +from contextlib import contextmanager +from typing import Sequence, Dict, Optional, Mapping, List, Tuple from pyomo.core.base.constraint import ConstraintData from pyomo.core.base.var import VarData @@ -45,12 +46,27 @@ class Availability(IntEnum): order to record its availability for use. """ + NoLicenseRequired = 3 + """The solver was found and no license is required to run.""" + FullLicense = 2 + """The solver was found and a full license is accessible to use.""" + LimitedLicense = 1 + """The solver was found and a limited license (e.g., demo license) is + accessible to use.""" + NotFound = 0 - BadVersion = -1 - BadLicense = -2 - NeedsCompiledExtension = -3 + """The solver was not found, either because the executable was not + on the path or the solver package is not importable.""" + + UnsupportedVersion = -1 + """The solver was found but is an unsupported version in Pyomo.""" + + LicenseError = -2 + """The solver was found but no usable license is available. This could + indicate either that no license was found, an expired or malformed + license was found, or the license is incorrect for the problem type.""" def __bool__(self): return self._value_ > 0 @@ -62,6 +78,40 @@ def __str__(self): return self.name +class _LicenseManager: + def acquire(self, timeout: Optional[float] = None) -> None: + """Acquire and lock a license. Default behavior is to simply return + because we assume, unless otherwise noted, that a solver does NOT + require a license.""" + return + + def release(self) -> None: + """Release the lock on a license.""" + return + + def __enter__(self): + self.acquire() + return self + + def __exit__(self, exc_type, exc, tb): + self.release() + return False + + def __call__(self, timeout=None): + """This logic is necessary in order to support this type of + context manager: ``with solver.license(timeout=5):``""" + + @contextmanager + def _cm(): + self.acquire(timeout) + try: + yield self + finally: + self.release() + + return _cm() + + class SolverBase: """The base class for "new-style" Pyomo solver interfaces. @@ -98,6 +148,7 @@ def __init__(self, **kwds) -> None: #: Instance configuration; see CONFIG documentation on derived class self.config = self.CONFIG(value=kwds) + self.license = _LicenseManager() def __enter__(self): return self @@ -138,7 +189,7 @@ def solve(self, model: BlockData, **kwargs) -> Results: f"Derived class {self.__class__.__name__} failed to implement required method 'solve'." ) - def available(self) -> Availability: + def available(self, recheck: bool = False) -> Availability: """Test if the solver is available on this system. Nominally, this will return `True` if the solver interface is @@ -165,7 +216,7 @@ def available(self) -> Availability: f"Derived class {self.__class__.__name__} failed to implement required method 'available'." ) - def version(self) -> Tuple: + def version(self, recheck: bool = False) -> Tuple: """Return the solver version found on the system. Returns diff --git a/pyomo/contrib/solver/solvers/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi_direct.py index 45ea9dcc873..910dcc4f909 100644 --- a/pyomo/contrib/solver/solvers/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi_direct.py @@ -14,12 +14,16 @@ import math import operator import os +import logging +import time +from typing import Optional, Tuple +from contextlib import contextmanager from pyomo.common.collections import ComponentMap, ComponentSet from pyomo.common.config import ConfigValue from pyomo.common.dependencies import attempt_import from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import MouseTrap, ApplicationError +from pyomo.common.errors import MouseTrap from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer @@ -43,9 +47,20 @@ ) from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +logger = logging.getLogger(__name__) gurobipy, gurobipy_available = attempt_import('gurobipy') +# Historically, we stored a live gurobipy.Env object on the class as a shared +# singleton (per-process) for all solver instances. That made dill/pickle of +# the class fail, because gurobipy.Env is a Cython type with a non-trivial +# __cinit__ and no pickling support. +# +# To keep classes picklable (e.g., for multiprocessing), +# we store only a small, picklable key on the class, and put the +# real Env object in this module-global registry. +_GUROBI_ENV_REGISTRY = {} + class GurobiConfigMixin: """ @@ -167,6 +182,79 @@ def get_reduced_costs(self, vars_to_load=None): return ComponentMap(iterator) +class _GurobiLicenseManager: + """ + License handler for Gurobi instances. Handles checkout, locking, + and release. + + The manager: + - creates the Env on first use (or reuses a live one) + - increments a per-class client count + - and decrements it on exit, closing the Env when the last client leaves + + The actual Env object lives in _GUROBI_ENV_REGISTRY, keyed by class + """ + + def __init__(self, owner_cls): + self._cls = owner_cls + + def acquire(self, timeout: Optional[float] = None) -> None: + """Acquire (or reuse) a shared gurobipy.Env.""" + cls = self._cls + # Try to reuse an existing Env; recreate if it was closed + env = cls._ensure_env() + if env is not None: + cls._register_env_client() + return + + if not timeout: + cls._set_env(gurobipy.Env()) + cls._register_env_client() + return + + # timeout implementation + start = time.time() + sleep_for = 0.1 + while time.time() - start < timeout: + try: + cls._set_env(gurobipy.Env()) + cls._register_env_client() + return + except Exception as e: + logger.info( + "Gurobi license not acquired yet; retrying: %s", e, exc_info=True + ) + time.sleep(min(sleep_for, timeout - (time.time() - start))) + sleep_for = min(sleep_for * 2, 2.0) + + logger.warning( + "Timed out after %.2f seconds trying to acquire a Gurobi license.", timeout + ) + + def release(self) -> None: + """Release one client; closes Env when last client releases.""" + self._cls._release_env_client() + + def __enter__(self) -> "_GurobiLicenseManager": + self.acquire() + return self + + def __exit__(self, exc_type, exc, tb) -> bool: + self.release() + return False + + def __call__(self, timeout: Optional[float] = None): + @contextmanager + def _cm(): + self.acquire(timeout) + try: + yield self + finally: + self.release() + + return _cm() + + class GurobiSolverMixin: """ gurobi_direct and gurobi_persistent check availability and set versions @@ -175,82 +263,168 @@ class GurobiSolverMixin: """ _num_gurobipy_env_clients = 0 - _gurobipy_env = None - _available = None - _gurobipy_available = gurobipy_available - - def available(self): - if self._available is None: - # this triggers the deferred import, and for the persistent - # interface, may update the _available flag - # - # Note that we set the _available flag on the *most derived - # class* and not on the instance, or on the base class. That - # allows different derived interfaces to have different - # availability (e.g., persistent has a minimum version - # requirement that the direct interface doesn't) - if not self._gurobipy_available: - if self._available is None: - self.__class__._available = Availability.NotFound - else: - self.__class__._available = self._check_license() - return self._available + # Instead of storing the Env directly on the class (which breaks pickling), + # we store a small key into the process-local registry above + _gurobipy_env_key = None + _available_cache = None + _version_cache = None - @staticmethod - def release_license(): - if GurobiSolverMixin._gurobipy_env is None: - return - if GurobiSolverMixin._num_gurobipy_env_clients: - logger.warning( - "Call to GurobiSolverMixin.release_license() with %s remaining " - "environment clients." % (GurobiSolverMixin._num_gurobipy_env_clients,) - ) - GurobiSolverMixin._gurobipy_env.close() - GurobiSolverMixin._gurobipy_env = None - - @staticmethod - def env(): - if GurobiSolverMixin._gurobipy_env is None: - with capture_output(capture_fd=True): - GurobiSolverMixin._gurobipy_env = gurobipy.Env() - return GurobiSolverMixin._gurobipy_env - - @staticmethod - def _register_env_client(): - GurobiSolverMixin._num_gurobipy_env_clients += 1 - - @staticmethod - def _release_env_client(): - GurobiSolverMixin._num_gurobipy_env_clients -= 1 - if GurobiSolverMixin._num_gurobipy_env_clients <= 0: - # Note that _num_gurobipy_env_clients should never be <0, - # but if it is, release_license will issue a warning (that - # we want to know about) - GurobiSolverMixin.release_license() - - def _check_license(self): - try: - model = gurobipy.Model(env=self.env()) - except gurobipy.GurobiError: - return Availability.BadLicense - - model.setParam('OutputFlag', 0) + def _is_gp_available(self) -> bool: try: - model.addVars(range(2001)) - model.optimize() - return Availability.FullLicense - except gurobipy.GurobiError: - return Availability.LimitedLicense - finally: - model.dispose() - - def version(self): - version = ( + # this triggers the deferred import + return bool(gurobipy_available) + except Exception: + return False + + def available(self, recheck: bool = False) -> Availability: + """ + Best-effort classification: + - NotFound : gurobipy not importable + - FullLicense : check succeeds on a full-size model (>2000 vars) + - LimitedLicense : check triggers limit (e.g., demo/community) + - LicenseError : denial/timeout/bad/unknown licensing states + """ + if not recheck and self._available_cache is not None: + return self._available_cache + # Note that we set the _available_cache on the *most derived + # class* and not on the instance, or on the base class. That + # allows different derived interfaces to have different + # availability (e.g., persistent has a minimum version + # requirement that the direct interface doesn't) + if not self._is_gp_available(): + self.__class__._available_cache = Availability.NotFound + else: + with capture_output(capture_fd=True): + try: + with self.license(): + status = self._check_license_status() + except Exception as e: + logger.debug( + "License check failed in available(): %s", e, exc_info=True + ) + status = Availability.LicenseError + + self.__class__._available_cache = status + return self._available_cache + + def version(self, recheck: bool = False) -> Optional[Tuple[int, int, int]]: + if not recheck and self._version_cache is not None: + return self._version_cache + + if not self._is_gp_available(): + return None + + self.__class__._version_cache = ( gurobipy.GRB.VERSION_MAJOR, gurobipy.GRB.VERSION_MINOR, gurobipy.GRB.VERSION_TECHNICAL, ) - return version + return self._version_cache + + def _check_license_status(self) -> Availability: + """ + Build a tiny model (>2000 vars) to test demo/community limits and + classify license level. + """ + env = None + m = None + try: + env = gurobipy.Env() + env.setParam("OutputFlag", 0) + m = gurobipy.Model(env=env) + m.Params.OutputFlag = 0 + m.addVars(range(2001)) + m.optimize() + return Availability.FullLicense + except gurobipy.GurobiError as e: + msg = (str(e) or "").lower() + errno = getattr(e, "errno", None) + if errno in (10010,) or "too large" in msg: + return Availability.LimitedLicense + if ( + "no gurobi license" in msg + or "not licensed" in msg + or "license not found" in msg + or "expired" in msg + or "queue" in msg + or "timeout" in msg + or errno in (10009,) + ): + return Availability.LicenseError + # Treat any other unexpected status as an error + return Availability.LicenseError + finally: + try: + if m is not None: + m.dispose() + except Exception: + pass + try: + if env is not None: + env.close() + except Exception: + pass + + @classmethod + def _register_env_client(cls): + cls._num_gurobipy_env_clients += 1 + + @classmethod + def _release_env_client(cls): + if cls._num_gurobipy_env_clients > 0: + cls._num_gurobipy_env_clients -= 1 + env = cls._get_env() + if cls._num_gurobipy_env_clients <= 0 and env is not None: + if cls._num_gurobipy_env_clients < 0: + logger.warning( + "Gurobi env client refcount went negative " + f"({cls._num_gurobipy_env_clients}). " + "This should not have happened and should be reported to " + "Pyomo development team." + ) + try: + env.close() + except Exception as err: + logger.warning(f"Exception while closing Gurobi environment: {err!r}") + finally: + cls._set_env(None) + + @classmethod + def _get_env(cls): + """Return the current shared Env (or None)""" + if cls._gurobipy_env_key is None: + return None + return _GUROBI_ENV_REGISTRY.get(cls._gurobipy_env_key) + + @classmethod + def _set_env(cls, env): + """Install or clear the current process-local Env""" + if env is None: + if cls._gurobipy_env_key is not None: + _GUROBI_ENV_REGISTRY.pop(cls._gurobipy_env_key, None) + cls._gurobipy_env_key = None + else: + if cls._gurobipy_env_key is None: + cls._gurobipy_env_key = id(cls) + _GUROBI_ENV_REGISTRY[cls._gurobipy_env_key] = env + + @classmethod + def _ensure_env(cls): + """ + Return a live gurobipy.Env. If current env exists but is closed, + recreate it. + """ + env = cls._get_env() + if env is None: + return None + if not hasattr(env, "_cenv"): + try: + env.close() + except Exception: + pass + env = gurobipy.Env() + cls._set_env(env) + return env class GurobiDirect(GurobiSolverMixin, SolverBase): @@ -264,21 +438,11 @@ class GurobiDirect(GurobiSolverMixin, SolverBase): def __init__(self, **kwds): super().__init__(**kwds) - self._register_env_client() - - def __del__(self): - if not python_is_shutting_down(): - self._release_env_client() + self.license = _GurobiLicenseManager(type(self)) def solve(self, model, **kwds) -> Results: start_timestamp = datetime.datetime.now(datetime.timezone.utc) config = self.config(value=kwds, preserve_implicit=True) - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) if config.timer is None: config.timer = HierarchicalTimer() timer = config.timer @@ -314,7 +478,7 @@ def solve(self, model, **kwds) -> Results: ) for v in repn.columns ] - sense_type = list('=<>') # Note: ordering matches 0, 1, -1 + sense_type = list('=<>') sense = [sense_type[r[1]] for r in repn.rows] timer.stop('prepare_matrices') @@ -325,59 +489,64 @@ def solve(self, model, **kwds) -> Results: try: if config.working_dir: os.chdir(config.working_dir) - with capture_output(TeeStream(*ostreams), capture_fd=False): - gurobi_model = gurobipy.Model(env=self.env()) - - timer.start('transfer_model') - x = gurobi_model.addMVar( - len(repn.columns), - lb=lb, - ub=ub, - obj=repn.c.todense()[0] if repn.c.shape[0] else 0, - vtype=vtype, + + # Acquire a Gurobi env for the duration of solve (opt + postsolve): + with self.license(): + env = type(self)._ensure_env() + + with capture_output(TeeStream(*ostreams), capture_fd=False): + gurobi_model = gurobipy.Model(env=env) + + timer.start('transfer_model') + x = gurobi_model.addMVar( + len(repn.columns), + lb=lb, + ub=ub, + obj=repn.c.todense()[0] if repn.c.shape[0] else 0, + vtype=vtype, + ) + A = gurobi_model.addMConstr(repn.A, x, sense, repn.rhs) + if repn.c.shape[0]: + gurobi_model.setAttr('ObjCon', repn.c_offset[0]) + gurobi_model.setAttr( + 'ModelSense', int(repn.objectives[0].sense) + ) + timer.stop('transfer_model') + + options = config.solver_options + gurobi_model.setParam('LogToConsole', 1) + + if config.threads is not None: + gurobi_model.setParam('Threads', config.threads) + if config.time_limit is not None: + gurobi_model.setParam('TimeLimit', config.time_limit) + if config.rel_gap is not None: + gurobi_model.setParam('MIPGap', config.rel_gap) + if config.abs_gap is not None: + gurobi_model.setParam('MIPGapAbs', config.abs_gap) + + if config.use_mipstart: + raise MouseTrap("MIPSTART not yet supported") + + for key, option in options.items(): + gurobi_model.setParam(key, option) + + timer.start('optimize') + gurobi_model.optimize() + timer.stop('optimize') + + # Build Results while the env is still alive + res = self._postsolve( + timer, + config, + GurobiDirectSolutionLoader( + gurobi_model, A, x, repn.rows, repn.columns, repn.objectives + ), ) - A = gurobi_model.addMConstr(repn.A, x, sense, repn.rhs) - if repn.c.shape[0]: - gurobi_model.setAttr('ObjCon', repn.c_offset[0]) - gurobi_model.setAttr('ModelSense', int(repn.objectives[0].sense)) - # Note: calling gurobi_model.update() here is not - # necessary (it will happen as part of optimize()): - # gurobi_model.update() - timer.stop('transfer_model') - - options = config.solver_options - - gurobi_model.setParam('LogToConsole', 1) - - if config.threads is not None: - gurobi_model.setParam('Threads', config.threads) - if config.time_limit is not None: - gurobi_model.setParam('TimeLimit', config.time_limit) - if config.rel_gap is not None: - gurobi_model.setParam('MIPGap', config.rel_gap) - if config.abs_gap is not None: - gurobi_model.setParam('MIPGapAbs', config.abs_gap) - - if config.use_mipstart: - raise MouseTrap("MIPSTART not yet supported") - - for key, option in options.items(): - gurobi_model.setParam(key, option) - - timer.start('optimize') - gurobi_model.optimize() - timer.stop('optimize') + finally: os.chdir(orig_cwd) - res = self._postsolve( - timer, - config, - GurobiDirectSolutionLoader( - gurobi_model, A, x, repn.rows, repn.columns, repn.objectives - ), - ) - res.solver_config = config res.solver_name = 'Gurobi' res.solver_version = self.version() diff --git a/pyomo/contrib/solver/solvers/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi_persistent.py index ea3693c1c70..8ad3ecde21c 100644 --- a/pyomo/contrib/solver/solvers/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi_persistent.py @@ -40,6 +40,7 @@ from pyomo.contrib.solver.solvers.gurobi_direct import ( GurobiConfigMixin, GurobiSolverMixin, + _GurobiLicenseManager, ) from pyomo.contrib.solver.common.util import ( NoFeasibleSolutionError, @@ -252,7 +253,7 @@ def __init__(self, **kwds): PersistentSolverUtils.__init__( self, treat_fixed_vars_as_params=treat_fixed_vars_as_params ) - self._register_env_client() + self.license = _GurobiLicenseManager(type(self)) self._solver_model = None self._symbol_map = SymbolMap() self._labeler = None @@ -273,12 +274,10 @@ def __init__(self, **kwds): self._last_results_object: Optional[Results] = None def release_license(self): + # Reinitialize our persistent solver state (but do not re-acquire a new env) self._reinit() - self.__class__.release_license() - - def __del__(self): - if not python_is_shutting_down(): - self._release_env_client() + # Release the shared Env that we explicitly acquired in set_instance + self.license.release() @property def symbol_map(self): @@ -410,21 +409,13 @@ def _reinit(self): saved_config = self.config saved_tmp_config = self._active_config self.__init__(treat_fixed_vars_as_params=self._treat_fixed_vars_as_params) - # Note that __init__ registers a new env client, so we need to - # release it here: - self._release_env_client() self.config = saved_config self._active_config = saved_tmp_config def set_instance(self, model): if self._last_results_object is not None: self._last_results_object.solution_loader.invalidate() - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) + self.license.acquire() self._reinit() self._model = model @@ -433,7 +424,8 @@ def set_instance(self, model): else: self._labeler = NumericLabeler('x') - self._solver_model = gurobipy.Model(name=model.name or '', env=self.env()) + env = type(self)._ensure_env() + self._solver_model = gurobipy.Model(name=model.name or '', env=env) self.add_block(model) if self._objective is None: diff --git a/pyomo/contrib/solver/solvers/highs.py b/pyomo/contrib/solver/solvers/highs.py index fb83117bb81..3d9d28e52d8 100644 --- a/pyomo/contrib/solver/solvers/highs.py +++ b/pyomo/contrib/solver/solvers/highs.py @@ -11,7 +11,7 @@ import logging import io -from typing import List, Optional +from typing import List, Optional, Tuple from pyomo.common.collections import ComponentMap from pyomo.common.dependencies import attempt_import @@ -242,8 +242,6 @@ class Highs(PersistentSolverMixin, PersistentSolverUtils, PersistentSolverBase): CONFIG = PersistentBranchAndBoundConfig() - _available = None - def __init__(self, **kwds): treat_fixed_vars_as_params = kwds.pop('treat_fixed_vars_as_params', True) PersistentSolverBase.__init__(self, **kwds) @@ -258,27 +256,36 @@ def __init__(self, **kwds): self._mutable_bounds = {} self._last_results_object: Optional[Results] = None self._sol = None + self._available_cache = None + self._version_cache = None - def available(self): - if highspy_available: - return Availability.FullLicense - return Availability.NotFound - - def version(self): - try: - version = ( - highspy.HIGHS_VERSION_MAJOR, - highspy.HIGHS_VERSION_MINOR, - highspy.HIGHS_VERSION_PATCH, - ) - except AttributeError: - # Older versions of Highs do not have the above attributes - # and the solver version can only be obtained by making - # an instance of the solver class. - tmp = highspy.Highs() - version = (tmp.versionMajor(), tmp.versionMinor(), tmp.versionPatch()) - - return version + def available(self, recheck: bool = False) -> Availability: + if recheck or self._available_cache is None: + if not highspy_available: + self._available_cache = Availability.NotFound + else: + self._available_cache = Availability.NoLicenseRequired + return self._available_cache + + def version(self, recheck: bool = False) -> Optional[Tuple[int, int, int]]: + if recheck or self._version_cache is None: + try: + self._version_cache = ( + highspy.HIGHS_VERSION_MAJOR, + highspy.HIGHS_VERSION_MINOR, + highspy.HIGHS_VERSION_PATCH, + ) + except AttributeError: + # Older versions of Highs do not have the above attributes + # and the solver version can only be obtained by making + # an instance of the solver class. + tmp = highspy.Highs() + self._version_cache = ( + tmp.versionMajor(), + tmp.versionMinor(), + tmp.versionPatch(), + ) + return self._version_cache def _solve(self): config = self._active_config diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 7c73e07af38..ee5b3b649f7 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -244,24 +244,26 @@ def __init__(self, **kwds: Any) -> None: #: see :ref:`pyomo.contrib.solver.solvers.ipopt.Ipopt::CONFIG`. self.config = self.config - def available(self, config: Optional[IpoptConfig] = None) -> Availability: - if config is None: - config = self.config - pth = config.executable.path() - if self._available_cache is None or self._available_cache[0] != pth: + def available(self, recheck: bool = False) -> Availability: + pth = self.config.executable.path() + check_availability = ( + recheck or self._available_cache is None or self._available_cache[0] != pth + ) + + if check_availability: if pth is None: self._available_cache = (None, Availability.NotFound) else: - self._available_cache = (pth, Availability.FullLicense) + self._available_cache = (pth, Availability.NoLicenseRequired) + return self._available_cache[1] - def version( - self, config: Optional[IpoptConfig] = None - ) -> Optional[Tuple[int, int, int]]: - if config is None: - config = self.config - pth = config.executable.path() - if self._version_cache is None or self._version_cache[0] != pth: + def version(self, recheck: bool = False) -> Optional[Tuple[int, int, int]]: + pth = self.config.executable.path() + check_version = ( + recheck or self._version_cache is None or self._version_cache[0] != pth + ) + if check_version: if pth is None: self._version_cache = (None, None) else: @@ -346,12 +348,6 @@ def solve(self, model, **kwds) -> Results: start_timestamp = datetime.datetime.now(datetime.timezone.utc) # Update configuration options, based on keywords passed to solve config: IpoptConfig = self.config(value=kwds, preserve_implicit=True) - # Check if solver is available - avail = self.available(config) - if not avail: - raise ApplicationError( - f'Solver {self.__class__} is not available ({avail}).' - ) if config.threads: logger.log( logging.WARNING, @@ -525,7 +521,7 @@ def solve(self, model, **kwds) -> Results: raise NoOptimalSolutionError() results.solver_name = self.name - results.solver_version = self.version(config) + results.solver_version = self.version() if config.load_solutions: if results.solution_status == SolutionStatus.noSolution: diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_direct.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_direct.py new file mode 100644 index 00000000000..e050f0ff195 --- /dev/null +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_direct.py @@ -0,0 +1,214 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +from pyomo.common.tee import capture_output +from pyomo.common import unittest + +from pyomo.contrib.solver.solvers.gurobi_direct import GurobiSolverMixin, GurobiDirect + +from pyomo.contrib.solver.common.base import Availability + +opt = GurobiDirect() +if not opt.available(): + raise unittest.SkipTest("Gurobi is not available") + + +class TestGurobiMixin(unittest.TestCase): + + MODULE_PATH = "pyomo.contrib.solver.solvers.gurobi_direct" + + def setUp(self): + # Reset shared state before each test + GurobiSolverMixin._gurobipy_env = None + GurobiSolverMixin._available_cache = None + GurobiSolverMixin._version_cache = None + GurobiSolverMixin._num_gurobipy_env_clients = 0 + + class GurobiError(Exception): + def __init__(self, msg="", errno=None): + super().__init__(msg) + self.errno = errno + + class Env: + def __init__(self, *args, **kwargs): + self.closed = False + + def setParam(self, *args, **kwargs): + pass + + def close(self): + self.closed = True + + class Model: + def __init__(self, env=None, license_status="ok"): + self.license_status = license_status + self.Params = type("P", (), {})() + self.Params.OutputFlag = 0 + self._disposed = False + + def addVars(self, rng): + return None + + def optimize(self): + if self.license_status == "ok": + return + if self.license_status == "too_large": + raise TestGurobiMixin.GurobiError("Model too large", errno=10010) + if self.license_status == "timeout": + raise TestGurobiMixin.GurobiError("timeout waiting for license") + if self.license_status == "no_license": + raise TestGurobiMixin.GurobiError("no gurobi license", errno=10009) + if self.license_status == "bad": + raise TestGurobiMixin.GurobiError("other licensing problem") + + def dispose(self): + self._disposed = True + + @staticmethod + def mocked_gurobipy(license_status="ok", env_side_effect=None): + """ + Build a fake gurobipy module. + - license_status controls Model.optimize() behavior + - env_side_effect (callable or Exception) controls Env() behavior + e.g. env_side_effect=TestGurobiMixin.GurobiError("no gurobi license", errno=10009) + """ + + # Arbitrarily picking a version + class GRB: + VERSION_MAJOR = 12 + VERSION_MINOR = 0 + VERSION_TECHNICAL = 1 + + mocker = unittest.mock.MagicMock() + if env_side_effect is None: + mocker.Env = unittest.mock.MagicMock(return_value=TestGurobiMixin.Env()) + else: + if isinstance(env_side_effect, Exception): + mocker.Env = unittest.mock.MagicMock(side_effect=env_side_effect) + else: + mocker.Env = unittest.mock.MagicMock(side_effect=env_side_effect) + mocker.Model = unittest.mock.MagicMock( + side_effect=lambda *a, **kw: TestGurobiMixin.Model( + license_status=license_status + ) + ) + mocker.GRB = GRB + mocker.GurobiError = TestGurobiMixin.GurobiError + return mocker + + def test_available_notfound(self): + mixin = GurobiSolverMixin() + with unittest.mock.patch.object( + GurobiSolverMixin, "_is_gp_available", return_value=False + ): + self.assertEqual(mixin.available(), Availability.NotFound) + + def test_available_full_license(self): + opt = GurobiDirect() + mock_gp = self.mocked_gurobipy("ok") + with ( + unittest.mock.patch.object( + type(opt), "_is_gp_available", return_value=True + ), + unittest.mock.patch(f"{self.MODULE_PATH}.gurobipy", mock_gp), + ): + with capture_output(capture_fd=True): + self.assertEqual(opt.available(recheck=True), Availability.FullLicense) + + def test_available_limited_license(self): + opt = GurobiDirect() + mock_gp = self.mocked_gurobipy("too_large") + with ( + unittest.mock.patch.object( + GurobiSolverMixin, "_is_gp_available", return_value=True + ), + unittest.mock.patch(f"{self.MODULE_PATH}.gurobipy", mock_gp), + ): + with capture_output(capture_fd=True): + self.assertEqual( + opt.available(recheck=True), Availability.LimitedLicense + ) + + def test_available_license_error_no_license(self): + mixin = GurobiSolverMixin() + env_error = self.GurobiError("no gurobi license", errno=10009) + mock_gp = self.mocked_gurobipy(license_status="ok", env_side_effect=env_error) + with ( + unittest.mock.patch.object( + GurobiSolverMixin, "_is_gp_available", return_value=True + ), + unittest.mock.patch(f"{self.MODULE_PATH}.gurobipy", mock_gp), + ): + with capture_output(capture_fd=True): + self.assertEqual( + mixin.available(recheck=True), Availability.LicenseError + ) + + def test_available_cache_and_recheck(self): + opt = GurobiDirect() + # FullLicense + mock_full = self.mocked_gurobipy("ok") + with ( + unittest.mock.patch.object( + GurobiSolverMixin, "_is_gp_available", return_value=True + ), + unittest.mock.patch(f"{self.MODULE_PATH}.gurobipy", mock_full), + ): + self.assertEqual(opt.available(recheck=True), Availability.FullLicense) + # Change behavior to license error; without recheck should use cache + env_error = self.GurobiError("no gurobi license", errno=10009) + mock_err = self.mocked_gurobipy("ok", env_side_effect=env_error) + with unittest.mock.patch(f"{self.MODULE_PATH}.gurobipy", mock_err): + self.assertEqual(opt.available(), Availability.FullLicense) + # Now recheck + self.assertEqual(opt.available(recheck=True), Availability.LicenseError) + + def test_version_cache(self): + mixin = GurobiSolverMixin() + mock_gp = self.mocked_gurobipy() + with unittest.mock.patch(f"{self.MODULE_PATH}.gurobipy", mock_gp): + self.assertEqual(mixin.version(), (12, 0, 1)) + # Change the version, but we didn't ask for a recheck, so + # the cached version should stay the same + mock_gp.GRB.VERSION_MINOR = 99 + self.assertEqual(mixin.version(), (12, 0, 1)) + + def test_license_acquire_release(self): + opt = GurobiDirect() + mock_gp = self.mocked_gurobipy() + with unittest.mock.patch(f"{self.MODULE_PATH}.gurobipy", mock_gp): + cls = GurobiDirect + # Before acquire - nothing locked + self.assertIsNone(cls._get_env()) + self.assertIsNone(cls._gurobipy_env_key) + # Acquire - creates and registers env + opt.license.acquire() + try: + env_inst = mock_gp.Env.return_value + self.assertIsNotNone(cls._gurobipy_env_key) + self.assertIs(cls._get_env(), env_inst) + finally: + # Client count hits zero -> env closed and cleared + opt.license.release() + self.assertIsNone(cls._get_env()) + self.assertIsNone(cls._gurobipy_env_key) + + +class TestGurobiDirectInterface(unittest.TestCase): + def test_available_cache(self): + opt = GurobiDirect() + opt.available() + self.assertIsNotNone(opt._available_cache) + + def test_version_cache(self): + opt = GurobiDirect() + opt.version() + self.assertIsNotNone(opt._version_cache) diff --git a/pyomo/contrib/solver/tests/solvers/test_highs.py b/pyomo/contrib/solver/tests/solvers/test_highs.py index f59a0bfa42d..2ae4ab47cea 100644 --- a/pyomo/contrib/solver/tests/solvers/test_highs.py +++ b/pyomo/contrib/solver/tests/solvers/test_highs.py @@ -12,13 +12,97 @@ import pyomo.common.unittest as unittest import pyomo.environ as pyo -from pyomo.contrib.solver.solvers.highs import Highs - -opt = Highs() -if not opt.available(): - raise unittest.SkipTest - +import pyomo.contrib.solver.solvers.highs as highs +from pyomo.contrib.solver.common.base import Availability +from pyomo.contrib.solver.common.results import SolutionStatus + +highs_available = highs.Highs().available() + + +@unittest.skipIf(not highs_available, "highspy is not available") +class TestHighsInterface(unittest.TestCase): + def test_default_instantiation(self): + opt = highs.Highs() + self.assertTrue(opt.is_persistent()) + self.assertEqual(opt.name, 'highs') + self.assertEqual(opt.CONFIG, opt.config) + self.assertTrue(opt.available()) + self.assertIsNotNone(opt.version()) + + def test_available_cache_and_recheck(self): + opt = highs.Highs() + + first = opt.available() + self.assertTrue(first) + self.assertIsNotNone(opt._available_cache) + + # Make sure that recheck works by faking highspy_available + with unittest.mock.patch( + 'pyomo.contrib.solver.solvers.highs.highspy_available', False + ): + self.assertEqual(opt.available(), first) + self.assertEqual(opt.available(recheck=True), Availability.NotFound) + + def test_version_cache_and_recheck_with_attrs(self): + opt = highs.Highs() + version = opt.version() + self.assertIsNotNone(version) + self.assertIsNotNone(opt._version_cache) + + +@unittest.skipIf(not highs_available, "highspy is not available") +class TestHighs(unittest.TestCase): + def create_lp_model(self): + m = pyo.ConcreteModel() + m.x = pyo.Var(domain=pyo.NonNegativeReals) + m.y = pyo.Var(domain=pyo.NonNegativeReals) + m.con = pyo.Constraint(expr=m.x + m.y >= 1) + m.obj = pyo.Objective(expr=m.x + m.y, sense=pyo.minimize) + return m + def create_mip_model(self): + m = pyo.ConcreteModel() + m.x = pyo.Var(domain=pyo.NonNegativeIntegers) + m.y = pyo.Var(domain=pyo.NonNegativeReals) + m.con1 = pyo.Constraint(expr=m.x + m.y >= 3) + m.con2 = pyo.Constraint(expr=m.y <= 2) + m.obj = pyo.Objective(expr=3 * m.x + m.y, sense=pyo.minimize) + return m + + def test_lp_solve(self): + m = self.create_lp_model() + res = highs.Highs().solve(m) + self.assertEqual(res.solver_name, 'highs') + self.assertEqual(res.solution_status, SolutionStatus.optimal) + self.assertAlmostEqual(pyo.value(m.obj), 1.0, places=7) + self.assertGreaterEqual(m.x.value, 0.0) + self.assertGreaterEqual(m.y.value, 0.0) + self.assertGreaterEqual(m.x.value + m.y.value, 1.0 - 1e-7) + + def test_mip_solve(self): + m = self.create_mip_model() + res = highs.Highs().solve(m) + self.assertEqual(res.solver_name, 'highs') + self.assertEqual(res.solution_status, SolutionStatus.optimal) + self.assertAlmostEqual(pyo.value(m.obj), 5.0, places=7) + self.assertAlmostEqual(m.x.value, 1.0, places=7) + self.assertAlmostEqual(m.y.value, 2.0, places=7) + + def test_persistent_update_path(self): + m = self.create_lp_model() + opt = highs.Highs() + res1 = opt.solve(m) + self.assertEqual(res1.solution_status, SolutionStatus.optimal) + self.assertAlmostEqual(pyo.value(m.obj), 1.0, places=7) + + # Tighten the constraint + m.con.set_value(m.x + m.y >= 1.5) + res2 = opt.solve(m) + self.assertEqual(res2.solution_status, SolutionStatus.optimal) + self.assertAlmostEqual(pyo.value(m.obj), 1.5, places=7) + + +@unittest.skipIf(not highs_available, "highspy is not available") class TestBugs(unittest.TestCase): def test_mutable_params_with_remove_cons(self): m = pyo.ConcreteModel() @@ -35,7 +119,7 @@ def test_mutable_params_with_remove_cons(self): m.p1.value = 1 m.p2.value = 1 - opt = Highs() + opt = highs.Highs() res = opt.solve(m) self.assertAlmostEqual(res.objective_bound, 1) @@ -62,7 +146,7 @@ def test_mutable_params_with_remove_vars(self): m.p1.value = -10 m.p2.value = 10 - opt = Highs() + opt = highs.Highs() res = opt.solve(m) self.assertAlmostEqual(res.objective_bound, 1) @@ -87,7 +171,7 @@ def test_fix_and_unfix(self): m.obj = pyo.Objective(expr=m.fx * 0.5 + m.fy * 0.4, sense=pyo.maximize) - opt = Highs() + opt = highs.Highs() # solution 1 has m.x == 1 and m.y == 0 r = opt.solve(m) diff --git a/pyomo/contrib/solver/tests/solvers/test_ipopt.py b/pyomo/contrib/solver/tests/solvers/test_ipopt.py index f88b18300f4..dd0bbab052e 100644 --- a/pyomo/contrib/solver/tests/solvers/test_ipopt.py +++ b/pyomo/contrib/solver/tests/solvers/test_ipopt.py @@ -17,7 +17,7 @@ from pyomo.common.envvar import is_windows from pyomo.common.fileutils import ExecutableData from pyomo.common.config import ConfigDict, ADVANCED_OPTION -from pyomo.common.errors import DeveloperError +from pyomo.common.errors import DeveloperError, ApplicationError from pyomo.common.tee import capture_output import pyomo.contrib.solver.solvers.ipopt as ipopt from pyomo.contrib.solver.common.util import NoSolutionError @@ -126,6 +126,7 @@ def test_class_member_list(self): 'available', 'has_linear_solver', 'is_persistent', + 'license', 'solve', 'version', 'name', @@ -154,10 +155,9 @@ def test_available_cache(self): opt.available() self.assertTrue(opt._available_cache[1]) self.assertIsNotNone(opt._available_cache[0]) - # Now we will try with a custom config that has a fake path - config = ipopt.IpoptConfig() - config.executable = Executable('/a/bogus/path') - opt.available(config=config) + # Now we will change the executable to a fake path + opt.config.executable = Executable('/a/bogus/path') + opt.available(recheck=True) self.assertFalse(opt._available_cache[1]) self.assertIsNone(opt._available_cache[0]) @@ -166,10 +166,9 @@ def test_version_cache(self): opt.version() self.assertIsNotNone(opt._version_cache[0]) self.assertIsNotNone(opt._version_cache[1]) - # Now we will try with a custom config that has a fake path - config = ipopt.IpoptConfig() - config.executable = Executable('/a/bogus/path') - opt.version(config=config) + # Now we will change the executable to a fake path + opt.config.executable = Executable('/a/bogus/path') + opt.version(recheck=True) self.assertIsNone(opt._version_cache[0]) self.assertIsNone(opt._version_cache[1]) @@ -589,6 +588,13 @@ def test_ipopt_solve(self): self.assertAlmostEqual(model.x.value, 1) self.assertAlmostEqual(model.y.value, 1) + def test_ipopt_solve_not_available(self): + model = self.create_model() + opt = ipopt.Ipopt() + opt.config.executable = Executable('/a/bogus/path') + with self.assertRaises(ApplicationError): + opt.solve(model) + def test_ipopt_quiet_print_level(self): model = self.create_model() result = ipopt.Ipopt().solve(model, solver_options={'print_level': 0}) diff --git a/pyomo/contrib/solver/tests/unit/test_base.py b/pyomo/contrib/solver/tests/unit/test_base.py index 217b02b9999..268fefa9091 100644 --- a/pyomo/contrib/solver/tests/unit/test_base.py +++ b/pyomo/contrib/solver/tests/unit/test_base.py @@ -42,6 +42,8 @@ def test_init(self): self.assertEqual(instance.name, 'solverbase') self.assertEqual(instance.api_version().name, 'V2') self.assertEqual(instance.CONFIG, instance.config) + self.assertTrue(hasattr(instance, 'license')) + self.assertIsInstance(instance.license, base._LicenseManager) with self.assertRaises(NotImplementedError): self.assertEqual(instance.version(), None) with self.assertRaises(NotImplementedError): @@ -63,6 +65,18 @@ def test_custom_solver_name(self): instance = base.SolverBase(name='my_unique_name') self.assertEqual(instance.name, 'my_unique_name') + def test_default_license_behavior(self): + instance = base.SolverBase() + # plain context manager + with instance.license: + pass + # context manager with timeout + with instance.license(timeout=0.1): + pass + # explicit calls also work + instance.license.acquire(timeout=0.1) + instance.license.release() + class TestPersistentSolverBase(unittest.TestCase): def test_class_method_list(self):