From e5524392a9c99208448fe863952586e545776927 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 18 Mar 2024 15:14:12 +0100 Subject: [PATCH 001/104] Add first draft of SCIP persistent solving --- pyomo/solvers/plugins/solvers/__init__.py | 2 + pyomo/solvers/plugins/solvers/scip_direct.py | 838 ++++++++++++++++++ .../plugins/solvers/scip_persistent.py | 185 ++++ pyomo/solvers/tests/checks/test_SCIPDirect.py | 335 +++++++ .../tests/checks/test_SCIPPersistent.py | 318 +++++++ pyomo/solvers/tests/solvers.py | 21 + 6 files changed, 1699 insertions(+) create mode 100644 pyomo/solvers/plugins/solvers/scip_direct.py create mode 100644 pyomo/solvers/plugins/solvers/scip_persistent.py create mode 100644 pyomo/solvers/tests/checks/test_SCIPDirect.py create mode 100644 pyomo/solvers/tests/checks/test_SCIPPersistent.py diff --git a/pyomo/solvers/plugins/solvers/__init__.py b/pyomo/solvers/plugins/solvers/__init__.py index 9b2507d876c..e8f4e00e31a 100644 --- a/pyomo/solvers/plugins/solvers/__init__.py +++ b/pyomo/solvers/plugins/solvers/__init__.py @@ -30,3 +30,5 @@ import pyomo.solvers.plugins.solvers.mosek_persistent import pyomo.solvers.plugins.solvers.xpress_direct import pyomo.solvers.plugins.solvers.xpress_persistent +import pyomo.solvers.plugins.solvers.scip_direct +import pyomo.solvers.plugins.solvers.scip_persistent diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py new file mode 100644 index 00000000000..0aafb596007 --- /dev/null +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -0,0 +1,838 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import logging +import re +import sys + +from pyomo.common.collections import ComponentSet, ComponentMap, Bunch +from pyomo.common.tempfiles import TempfileManager +from pyomo.core import Var +from pyomo.core.expr.numeric_expr import ( + SumExpression, + ProductExpression, + UnaryFunctionExpression, + PowExpression, + DivisionExpression, +) +from pyomo.core.expr.numvalue import is_fixed +from pyomo.core.expr.numvalue import value +from pyomo.core.staleflag import StaleFlagManager +from pyomo.repn import generate_standard_repn +from pyomo.solvers.plugins.solvers.direct_solver import DirectSolver +from pyomo.solvers.plugins.solvers.direct_or_persistent_solver import ( + DirectOrPersistentSolver, +) +from pyomo.core.kernel.objective import minimize, maximize +from pyomo.opt.results.results_ import SolverResults +from pyomo.opt.results.solution import Solution, SolutionStatus +from pyomo.opt.results.solver import TerminationCondition, SolverStatus +from pyomo.opt.base import SolverFactory +from pyomo.core.base.suffix import Suffix + + +logger = logging.getLogger("pyomo.solvers") + + +class DegreeError(ValueError): + pass + + +def _is_numeric(x): + try: + float(x) + except ValueError: + return False + return True + + +@SolverFactory.register("scip_direct", doc="Direct python interface to SCIP") +class SCIPDirect(DirectSolver): + + def __init__(self, **kwds): + kwds["type"] = "scipdirect" + DirectSolver.__init__(self, **kwds) + self._init() + self._solver_model = None + + def _init(self): + try: + import pyscipopt + + self._scip = pyscipopt + self._python_api_exists = True + self._version = str(self._scip.Model().version()) + self._version_major = self._version.split(".")[0] + except ImportError: + self._python_api_exists = False + except Exception as e: + print("Import of pyscipopt failed - SCIP message=" + str(e) + "\n") + self._python_api_exists = False + + # Note: Undefined capabilities default to None + self._max_constraint_degree = None + self._max_obj_degree = 1 + self._capabilities.linear = True + self._capabilities.quadratic_objective = False + self._capabilities.quadratic_constraint = True + self._capabilities.integer = True + self._capabilities.sos1 = True + self._capabilities.sos2 = True + + # Dictionary used exclusively for SCIP, as we want the constraint expressions + self._pyomo_var_to_solver_var_expr_map = ComponentMap() + self._pyomo_con_to_solver_con_expr_map = dict() + + def _apply_solver(self): + StaleFlagManager.mark_all_as_stale() + + # Supress solver output if requested + if self._tee: + self._solver_model.hideOutput(quiet=False) + else: + self._solver_model.hideOutput(quiet=True) + + # Redirect solver output to a logfile if requested + if self._keepfiles: + # Only save log file when the user wants to keep it. + self._solver_model.setLogfile(self._log_file) + print("Solver log file: " + self._log_file) + + # Set user specified parameters + for key, option in self.options.items(): + try: + key_type = type(self._solver_model.getParam(key)) + except KeyError: + raise ValueError(f"Key {key} is an invalid parameter for SCIP") + + if key_type == str: + self._solver_model.setParam(key, option) + else: + if not _is_numeric(option): + raise ValueError( + f"Value {option} for parameter {key} is not a string and can't be converted to float" + ) + self._solver_model.setParam(key, float(option)) + + self._solver_model.optimize() + + # TODO: Check if this is even needed, or if it is sufficient to close the open file + # if self._keepfiles: + # self._solver_model.setLogfile(None) + + # FIXME: can we get a return code indicating if SCIP had a significant failure? + return Bunch(rc=None, log=None) + + def _get_expr_from_pyomo_repn(self, repn, max_degree=None): + referenced_vars = ComponentSet() + + new_expr = repn.constant + + if len(repn.linear_vars) > 0: + referenced_vars.update(repn.linear_vars) + new_expr += sum( + repn.linear_coefs[i] * self._pyomo_var_to_solver_var_expr_map[var] + for i, var in enumerate(repn.linear_vars) + ) + + for i, v in enumerate(repn.quadratic_vars): + x, y = v + new_expr += ( + repn.quadratic_coefs[i] + * self._pyomo_var_to_solver_var_expr_map[x] + * self._pyomo_var_to_solver_var_expr_map[y] + ) + referenced_vars.add(x) + referenced_vars.add(y) + + # TODO: Introduce handling on non-linear expressions + if repn.nonlinear_expr is not None: + + def get_nl_expr_recursively(pyomo_expr): + if not hasattr(pyomo_expr, "args"): + if not isinstance(pyomo_expr, Var): + return float(pyomo_expr) + else: + referenced_vars.add(pyomo_expr) + return self._pyomo_var_to_solver_var_expr_map[pyomo_expr] + scip_expr_list = [0 for i in range(pyomo_expr.nargs())] + for i in range(pyomo_expr.nargs()): + scip_expr_list[i] = get_nl_expr_recursively(pyomo_expr.args[i]) + if isinstance(pyomo_expr, PowExpression): + if len(scip_expr_list) != 2: + raise ValueError( + f"PowExpression has {len(scip_expr_list)} many terms instead of two!" + ) + return scip_expr_list[0] ** (scip_expr_list[1]) + elif isinstance(pyomo_expr, ProductExpression): + return self._scip.quickprod(scip_expr_list) + elif isinstance(pyomo_expr, SumExpression): + return self._scip.quicksum(scip_expr_list) + elif isinstance(pyomo_expr, DivisionExpression): + if len(scip_expr_list) != 2: + raise ValueError( + f"DivisonExpression has {len(scip_expr_list)} many terms instead of two!" + ) + return scip_expr_list[0] / scip_expr_list[1] + elif isinstance(pyomo_expr, UnaryFunctionExpression): + if len(scip_expr_list) != 1: + raise ValueError( + f"UnaryExpression has {len(scip_expr_list)} many terms instead of one!" + ) + if pyomo_expr.name == "sin": + return self._scip.sin(scip_expr_list[0]) + elif pyomo_expr.name == "cos": + return self._scip.cos(scip_expr_list[0]) + elif pyomo_expr.name == "exp": + return self._scip.exp(scip_expr_list[0]) + elif pyomo_expr.name == "log": + return self._scip.log(scip_expr_list[0]) + else: + raise NotImplementedError( + f"PySCIPOpt through Pyomo does not support the unary function {pyomo_expr.name}" + ) + else: + raise NotImplementedError( + f"PySCIPOpt through Pyomo does not yet support expression type {type(pyomo_expr)}" + ) + + new_expr += get_nl_expr_recursively(repn.nonlinear_expr) + + return new_expr, referenced_vars + + def _get_expr_from_pyomo_expr(self, expr, max_degree=None): + if max_degree is None or max_degree >= 2: + repn = generate_standard_repn(expr, quadratic=True) + else: + repn = generate_standard_repn(expr, quadratic=False) + + scip_expr, referenced_vars = self._get_expr_from_pyomo_repn(repn, max_degree) + + return scip_expr, referenced_vars + + def _scip_lb_ub_from_var(self, var): + if var.is_fixed(): + val = var.value + return val, val + if var.has_lb(): + lb = value(var.lb) + else: + lb = -self._solver_model.infinity() + if var.has_ub(): + ub = value(var.ub) + else: + ub = self._solver_model.infinity() + return lb, ub + + def _add_var(self, var): + varname = self._symbol_map.getSymbol(var, self._labeler) + vtype = self._scip_vtype_from_var(var) + lb, ub = self._scip_lb_ub_from_var(var) + + scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype, name=varname) + + self._pyomo_var_to_solver_var_expr_map[var] = scip_var + self._pyomo_var_to_solver_var_map[var] = scip_var.name + self._solver_var_to_pyomo_var_map[varname] = var + self._referenced_variables[var] = 0 + + def close(self): + """Frees SCIP resources used by this solver instance.""" + + if self._solver_model is not None: + self._solver_model.freeProb() + self._solver_model = None + + def __exit__(self, t, v, traceback): + super().__exit__(t, v, traceback) + self.close() + + def _set_instance(self, model, kwds={}): + DirectOrPersistentSolver._set_instance(self, model, kwds) + try: + self._solver_model = self._scip.Model() + except Exception: + e = sys.exc_info()[1] + msg = ( + "Unable to create SCIP model. " + "Have you installed PySCIPOpt correctly?\n\n\t" + + "Error message: {0}".format(e) + ) + raise Exception(msg) + + self._add_block(model) + + for var, n_ref in self._referenced_variables.items(): + if n_ref != 0: + if var.fixed: + if not self._output_fixed_variable_bounds: + raise ValueError( + "Encountered a fixed variable (%s) inside " + "an active objective or constraint " + "expression on model %s, which is usually " + "indicative of a preprocessing error. Use " + "the IO-option 'output_fixed_variable_bounds=True' " + "to suppress this error and fix the variable " + "by overwriting its bounds in the SCIP instance." + % (var.name, self._pyomo_model.name) + ) + + def _add_block(self, block): + DirectOrPersistentSolver._add_block(self, block) + + def _add_constraint(self, con): + if not con.active: + return None + + if is_fixed(con.body) and self._skip_trivial_constraints: + return None + + conname = self._symbol_map.getSymbol(con, self._labeler) + + if con._linear_canonical_form: + scip_expr, referenced_vars = self._get_expr_from_pyomo_repn( + con.canonical_form(), self._max_constraint_degree + ) + else: + scip_expr, referenced_vars = self._get_expr_from_pyomo_expr( + con.body, self._max_constraint_degree + ) + + if con.has_lb(): + if not is_fixed(con.lower): + raise ValueError( + "Lower bound of constraint {0} is not constant.".format(con) + ) + if con.has_ub(): + if not is_fixed(con.upper): + raise ValueError( + "Upper bound of constraint {0} is not constant.".format(con) + ) + + if con.equality: + scip_cons = self._solver_model.addCons( + scip_expr == value(con.lower), name=conname + ) + elif con.has_lb() and con.has_ub(): + scip_cons = self._solver_model.addCons( + value(con.lower) <= (scip_expr <= value(con.upper)), name=conname + ) + elif con.has_lb(): + scip_cons = self._solver_model.addCons( + value(con.lower) <= scip_expr, name=conname + ) + elif con.has_ub(): + scip_cons = self._solver_model.addCons( + scip_expr <= value(con.upper), name=conname + ) + else: + raise ValueError( + "Constraint does not have a lower " + "or an upper bound: {0} \n".format(con) + ) + + for var in referenced_vars: + self._referenced_variables[var] += 1 + self._vars_referenced_by_con[con] = referenced_vars + self._pyomo_con_to_solver_con_expr_map[con] = scip_cons + self._pyomo_con_to_solver_con_map[con] = scip_cons.name + self._solver_con_to_pyomo_con_map[conname] = con + + def _add_sos_constraint(self, con): + if not con.active: + return None + + conname = self._symbol_map.getSymbol(con, self._labeler) + level = con.level + if level not in [1, 2]: + raise ValueError(f"Solver does not support SOS level {level} constraints") + + scip_vars = [] + weights = [] + + self._vars_referenced_by_con[con] = ComponentSet() + + if hasattr(con, "get_items"): + # aml sos constraint + sos_items = list(con.get_items()) + else: + # kernel sos constraint + sos_items = list(con.items()) + + for v, w in sos_items: + self._vars_referenced_by_con[con].add(v) + scip_vars.append(self._pyomo_var_to_solver_var_expr_map[v]) + self._referenced_variables[v] += 1 + weights.append(w) + + if level == 1: + scip_cons = self._solver_model.addConsSOS1( + scip_vars, weights=weights, name=conname + ) + else: + scip_cons = self._solver_model.addConsSOS2( + scip_vars, weights=weights, name=conname + ) + self._pyomo_con_to_solver_con_expr_map[con] = scip_cons + self._pyomo_con_to_solver_con_map[con] = scip_cons.name + self._solver_con_to_pyomo_con_map[conname] = con + + def _scip_vtype_from_var(self, var): + """ + This function takes a pyomo variable and returns the appropriate SCIP variable type + :param var: pyomo.core.base.var.Var + :return: B, I, or C + """ + if var.is_binary(): + vtype = "B" + elif var.is_integer(): + vtype = "I" + elif var.is_continuous(): + vtype = "C" + else: + raise ValueError( + "Variable domain type is not recognized for {0}".format(var.domain) + ) + return vtype + + def _set_objective(self, obj): + if self._objective is not None: + for var in self._vars_referenced_by_obj: + self._referenced_variables[var] -= 1 + self._vars_referenced_by_obj = ComponentSet() + self._objective = None + + if obj.active is False: + raise ValueError("Cannot add inactive objective to solver.") + + if obj.sense == minimize: + sense = "minimize" + elif obj.sense == maximize: + sense = "maximize" + else: + raise ValueError("Objective sense is not recognized: {0}".format(obj.sense)) + + scip_expr, referenced_vars = self._get_expr_from_pyomo_expr( + obj.expr, self._max_obj_degree + ) + + for var in referenced_vars: + self._referenced_variables[var] += 1 + + self._solver_model.setObjective(scip_expr, sense=sense) + self._objective = obj + self._vars_referenced_by_obj = referenced_vars + + self._needs_updated = True + + def _postsolve(self): + # the only suffixes that we extract from SCIP are + # constraint duals, constraint slacks, and variable + # reduced-costs. scan through the solver suffix list + # and throw an exception if the user has specified + # any others. + extract_duals = False + extract_slacks = False + extract_reduced_costs = False + for suffix in self._suffixes: + flag = False + if re.match(suffix, "dual"): + extract_duals = True + flag = True + if re.match(suffix, "slack"): + extract_slacks = True + flag = True + if re.match(suffix, "rc"): + extract_reduced_costs = True + flag = True + if not flag: + raise RuntimeError( + "***The scip_direct solver plugin cannot extract solution suffix=" + + suffix + ) + + scip = self._solver_model + status = scip.getStatus() + scip_vars = scip.getVars() + n_bin_vars = sum([scip_var.vtype() == "BINARY" for scip_var in scip_vars]) + n_int_vars = sum([scip_var.vtype() == "INTEGER" for scip_var in scip_vars]) + n_con_vars = sum([scip_var.vtype() == "CONTINUOUS" for scip_var in scip_vars]) + + if n_bin_vars + n_int_vars > 0: + if extract_reduced_costs: + logger.warning("Cannot get reduced costs for MIP.") + if extract_duals: + logger.warning("Cannot get duals for MIP.") + extract_reduced_costs = False + extract_duals = False + + self.results = SolverResults() + soln = Solution() + + self.results.solver.name = f"SCIP{self._version}" + self.results.solver.wallclock_time = scip.getSolvingTime() + + if scip.getStage() == 1: # SCIP Model is created but not yet optimized + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Model is loaded, but no solution information is available." + ) + self.results.solver.termination_condition = TerminationCondition.error + soln.status = SolutionStatus.unknown + elif status == "optimal": # optimal + self.results.solver.status = SolverStatus.ok + self.results.solver.termination_message = ( + "Model was solved to optimality (subject to tolerances), " + "and an optimal solution is available." + ) + self.results.solver.termination_condition = TerminationCondition.optimal + soln.status = SolutionStatus.optimal + elif status == "infeasible": + self.results.solver.status = SolverStatus.warning + self.results.solver.termination_message = ( + "Model was proven to be infeasible" + ) + self.results.solver.termination_condition = TerminationCondition.infeasible + soln.status = SolutionStatus.infeasible + elif status == "inforunbd": + self.results.solver.status = SolverStatus.warning + self.results.solver.termination_message = ( + "Problem proven to be infeasible or unbounded." + ) + self.results.solver.termination_condition = ( + TerminationCondition.infeasibleOrUnbounded + ) + soln.status = SolutionStatus.unsure + elif status == "unbounded": + self.results.solver.status = SolverStatus.warning + self.results.solver.termination_message = ( + "Model was proven to be unbounded." + ) + self.results.solver.termination_condition = TerminationCondition.unbounded + soln.status = SolutionStatus.unbounded + elif status == "gaplimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the gap dropped below " + "the value specified in the " + "limits/gap parameter." + ) + self.results.solver.termination_condition = TerminationCondition.unknown + soln.status = SolutionStatus.stoppedByLimit + elif status == "stallnodelimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the stalling node limit " + "exceeded the value specified in the " + "limits/stallnodes parameter." + ) + self.results.solver.termination_condition = TerminationCondition.unknown + soln.status = SolutionStatus.stoppedByLimit + elif status == "restartlimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the total number of restarts " + "exceeded the value specified in the " + "limits/restarts parameter." + ) + self.results.solver.termination_condition = TerminationCondition.unknown + soln.status = SolutionStatus.stoppedByLimit + elif status == "nodelimit" or status == "totalnodelimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the number of " + "branch-and-cut nodes explored exceeded the limits specified " + "in the limits/nodes or limits/totalnodes parameter" + ) + self.results.solver.termination_condition = ( + TerminationCondition.maxEvaluations + ) + soln.status = SolutionStatus.stoppedByLimit + elif status == "timelimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the time expended exceeded " + "the value specified in the limits/time parameter." + ) + self.results.solver.termination_condition = ( + TerminationCondition.maxTimeLimit + ) + soln.status = SolutionStatus.stoppedByLimit + elif status == "sollimit" or status == "bestsollimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the number of solutions found " + "reached the value specified in the limits/solutions or" + "limits/bestsol parameter." + ) + self.results.solver.termination_condition = TerminationCondition.unknown + soln.status = SolutionStatus.stoppedByLimit + elif status == "memlimit": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization terminated because the memory used exceeded " + "the value specified in the limits/memory parameter." + ) + self.results.solver.termination_condition = TerminationCondition.unknown + soln.status = SolutionStatus.stoppedByLimit + elif status == "userinterrupt": + self.results.solver.status = SolverStatus.aborted + self.results.solver.termination_message = ( + "Optimization was terminated by the user." + ) + self.results.solver.termination_condition = TerminationCondition.error + soln.status = SolutionStatus.error + else: + self.results.solver.status = SolverStatus.error + self.results.solver.termination_message = ( + "Unhandled SCIP status (" + str(status) + ")" + ) + self.results.solver.termination_condition = TerminationCondition.error + soln.status = SolutionStatus.error + + self.results.problem.name = scip.getProbName() + + if scip.getObjectiveSense() == "minimize": + self.results.problem.sense = minimize + elif scip.getObjectiveSense() == "maximize": + self.results.problem.sense = maximize + else: + raise RuntimeError( + f"Unrecognized SCIP objective sense: {scip.getObjectiveSense()}" + ) + + self.results.problem.upper_bound = None + self.results.problem.lower_bound = None + if scip.getNSols() > 0: + scip_has_sol = True + else: + scip_has_sol = False + if not scip_has_sol and (status == "inforunbd" or status == "infeasible"): + pass + else: + if n_bin_vars + n_int_vars == 0: + self.results.problem.upper_bound = scip.getObjVal() + self.results.problem.lower_bound = scip.getObjVal() + elif scip.getObjectiveSense() == "minimize": # minimizing + if scip_has_sol: + self.results.problem.upper_bound = scip.getObjVal() + else: + self.results.problem.upper_bound = scip.infinity() + self.results.problem.lower_bound = scip.getDualbound() + else: # maximizing + self.results.problem.upper_bound = scip.getDualbound() + if scip_has_sol: + self.results.problem.lower_bound = scip.getObjVal() + else: + self.results.problem.lower_bound = -scip.infinity() + + try: + soln.gap = ( + self.results.problem.upper_bound - self.results.problem.lower_bound + ) + except TypeError: + soln.gap = None + + # TODO: Should these values be of the transformed or the original problem? + self.results.problem.number_of_constraints = scip.getNConss() + # self.results.problem.number_of_nonzeros = None + self.results.problem.number_of_variables = scip.getNVars() + self.results.problem.number_of_binary_variables = n_bin_vars + self.results.problem.number_of_integer_variables = n_int_vars + self.results.problem.number_of_continuous_variables = n_con_vars + self.results.problem.number_of_objectives = 1 + self.results.problem.number_of_solutions = scip.getNSols() + + # if a solve was stopped by a limit, we still need to check to + # see if there is a solution available - this may not always + # be the case, both in LP and MIP contexts. + if self._save_results: + """ + This code in this if statement is only needed for backwards compatibility. It is more efficient to set + _save_results to False and use load_vars, load_duals, etc. + """ + if scip.getNSols() > 0: + soln_variables = soln.variable + soln_constraints = soln.constraint + scip_sol = scip.getBestSol() + + scip_vars = scip.getVars() + scip_var_names = [scip_var.name for scip_var in scip_vars] + var_names = set(self._solver_var_to_pyomo_var_map.keys()) + assert set(scip_var_names) == var_names + var_vals = [scip.getVal(scip_var) for scip_var in scip_vars] + + for scip_var, val, name in zip(scip_vars, var_vals, scip_var_names): + pyomo_var = self._solver_var_to_pyomo_var_map[name] + if self._referenced_variables[pyomo_var] > 0: + soln_variables[name] = {"Value": val} + + if extract_reduced_costs: + vals = [scip.getVarRedcost(scip_var) for scip_var in scip_vars] + for scip_var, val, name in zip(scip_vars, vals, scip_var_names): + pyomo_var = self._solver_var_to_pyomo_var_map[name] + if self._referenced_variables[pyomo_var] > 0: + soln_variables[name]["Rc"] = val + + if extract_duals or extract_slacks: + scip_cons = scip.getConss() + con_names = [cons.name for cons in scip_cons] + assert set(self._solver_con_to_pyomo_con_map.keys()) == set( + con_names + ) + for name in con_names: + soln_constraints[name] = {} + + if extract_duals: + vals = [scip.getDualSolVal(con) for con in scip_cons] + for val, name in zip(vals, con_names): + soln_constraints[name]["Dual"] = val + + if extract_slacks: + vals = [scip.getSlack(con, scip_sol) for con in scip_cons] + for val, name in zip(vals, con_names): + soln_constraints[name]["Slack"] = val + + elif self._load_solutions: + if scip.getNSols() > 0: + self.load_vars() + + if extract_reduced_costs: + self._load_rc() + + if extract_duals: + self._load_duals() + + if extract_slacks: + self._load_slacks() + + self.results.solution.insert(soln) + + # finally, clean any temporary files registered with the temp file + # manager, created populated *directly* by this plugin. + TempfileManager.pop(remove=not self._keepfiles) + + return DirectOrPersistentSolver._postsolve(self) + + def warm_start_capable(self): + return True + + def _warm_start(self): + scip_sol = self._solver_model.createSol() + for pyomo_var, scip_var in self._pyomo_var_to_solver_var_expr_map.items(): + if pyomo_var.value is not None: + scip_sol[scip_var] = value(pyomo_var) + self._solver_model.trySol(scip_sol, free=True) + + def _load_vars(self, vars_to_load=None): + var_map = self._pyomo_var_to_solver_var_expr_map + ref_vars = self._referenced_variables + if vars_to_load is None: + vars_to_load = var_map.keys() + + scip_vars_to_load = [var_map[pyomo_var] for pyomo_var in vars_to_load] + vals = [self._solver_model.getVal(scip_var) for scip_var in scip_vars_to_load] + + for var, val in zip(vars_to_load, vals): + if ref_vars[var] > 0: + var.set_value(val, skip_validation=True) + + def _load_rc(self, vars_to_load=None): + if not hasattr(self._pyomo_model, "rc"): + self._pyomo_model.rc = Suffix(direction=Suffix.IMPORT) + var_map = self._pyomo_var_to_solver_var_expr_map + ref_vars = self._referenced_variables + rc = self._pyomo_model.rc + if vars_to_load is None: + vars_to_load = var_map.keys() + + scip_vars_to_load = [var_map[pyomo_var] for pyomo_var in vars_to_load] + vals = [ + self._solver_model.getVarRedcost(scip_var) for scip_var in scip_vars_to_load + ] + + for var, val in zip(vars_to_load, vals): + if ref_vars[var] > 0: + rc[var] = val + + def _load_duals(self, cons_to_load=None): + if not hasattr(self._pyomo_model, "dual"): + self._pyomo_model.dual = Suffix(direction=Suffix.IMPORT) + con_map = self._pyomo_con_to_solver_con_map + reverse_con_map = self._solver_con_to_pyomo_con_map + dual = self._pyomo_model.dual + scip_cons = self._solver_model.getConss() + + if cons_to_load is None: + con_names = [con.name for con in scip_cons] + vals = [self._solver_model.getDualSolVal(con) for con in scip_cons] + else: + con_names = set([con_map[pyomo_con] for pyomo_con in cons_to_load]) + scip_cons_to_load = [con for con in scip_cons if con.name in con_names] + vals = [self._solver_model.getDualSolVal(con) for con in scip_cons_to_load] + + for i, con_name in enumerate(con_names): + pyomo_con = reverse_con_map[con_name] + dual[pyomo_con] = vals[i] + + def _load_slacks(self, cons_to_load=None): + if not hasattr(self._pyomo_model, "slack"): + self._pyomo_model.slack = Suffix(direction=Suffix.IMPORT) + con_map = self._pyomo_con_to_solver_con_map + reverse_con_map = self._solver_con_to_pyomo_con_map + slack = self._pyomo_model.slack + scip_cons = self._solver_model.getConss() + scip_sol = self._solver_model.getBestSol() + + if cons_to_load is None: + con_names = [con.name for con in scip_cons] + vals = [self._solver_model.getSlack(con, scip_sol) for con in scip_cons] + else: + con_names = set([con_map[pyomo_con] for pyomo_con in cons_to_load]) + scip_cons_to_load = [con for con in scip_cons if con.name in con_names] + vals = [ + self._solver_model.getSlack(con, scip_sol) for con in scip_cons_to_load + ] + + for i, con_name in enumerate(con_names): + pyomo_con = reverse_con_map[con_name] + slack[pyomo_con] = vals[i] + + def load_duals(self, cons_to_load=None): + """ + Load the duals into the 'dual' suffix. The 'dual' suffix must live on the parent model. + + Parameters + ---------- + cons_to_load: list of Constraint + """ + self._load_duals(cons_to_load) + + def load_rc(self, vars_to_load): + """ + Load the reduced costs into the 'rc' suffix. The 'rc' suffix must live on the parent model. + + Parameters + ---------- + vars_to_load: list of Var + """ + self._load_rc(vars_to_load) + + def load_slacks(self, cons_to_load=None): + """ + Load the values of the slack variables into the 'slack' suffix. The 'slack' suffix must live on the parent + model. + + Parameters + ---------- + cons_to_load: list of Constraint + """ + self._load_slacks(cons_to_load) diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py new file mode 100644 index 00000000000..408aa84633f --- /dev/null +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -0,0 +1,185 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +from pyomo.solvers.plugins.solvers.scip_direct import SCIPDirect +from pyomo.solvers.plugins.solvers.persistent_solver import PersistentSolver +from pyomo.opt.base import SolverFactory + + +@SolverFactory.register("scip_persistent", doc="Persistent python interface to SCIP") +class SCIPPersistent(PersistentSolver, SCIPDirect): + """ + A class that provides a persistent interface to SCIP. Direct solver interfaces do not use any file io. + Rather, they interface directly with the python bindings for the specific solver. Persistent solver interfaces + are similar except that they "remember" their model. Thus, persistent solver interfaces allow incremental changes + to the solver model (e.g., the gurobi python model or the cplex python model). Note that users are responsible + for notifying the persistent solver interfaces when changes are made to the corresponding pyomo model. + + Keyword Arguments + ----------------- + model: ConcreteModel + Passing a model to the constructor is equivalent to calling the set_instance method. + type: str + String indicating the class type of the solver instance. + name: str + String representing either the class type of the solver instance or an assigned name. + doc: str + Documentation for the solver + options: dict + Dictionary of solver options + """ + + def __init__(self, **kwds): + kwds["type"] = "scip_persistent" + PersistentSolver.__init__(self, **kwds) + SCIPDirect._init(self) + + self._pyomo_model = kwds.pop("model", None) + if self._pyomo_model is not None: + self.set_instance(self._pyomo_model, **kwds) + + def _remove_constraint(self, solver_conname): + con = self._solver_con_to_pyomo_con_map[solver_conname] + scip_con = self._pyomo_con_to_solver_con_expr_map[con] + self._solver_model.delCons(scip_con) + + def _remove_sos_constraint(self, solver_sos_conname): + con = self._solver_con_to_pyomo_con_map[solver_sos_conname] + scip_con = self._pyomo_con_to_solver_con_expr_map[con] + self._solver_model.delCons(scip_con) + + def _remove_var(self, solver_varname): + var = self._solver_var_to_pyomo_var_map[solver_varname] + scip_var = self._pyomo_var_to_solver_var_expr_map[var] + self._solver_model.delVar(scip_var) + + def _warm_start(self): + SCIPDirect._warm_start(self) + + def update_var(self, var): + """Update a single variable in the solver's model. + + This will update bounds, fix/unfix the variable as needed, and + update the variable type. + + Parameters + ---------- + var: Var (scalar Var or single _VarData) + + """ + # see PR #366 for discussion about handling indexed + # objects and keeping compatibility with the + # pyomo.kernel objects + # if var.is_indexed(): + # for child_var in var.values(): + # self.compile_var(child_var) + # return + if var not in self._pyomo_var_to_solver_var_map: + raise ValueError( + "The Var provided to compile_var needs to be added first: {0}".format( + var + ) + ) + scip_var = self._pyomo_var_to_solver_var_map[var] + vtype = self._scip_vtype_from_var(var) + lb, ub = self._scip_lb_ub_from_var(var) + + self._solver_model.chgVarLb(scip_var, lb) + self._solver_model.chgVarUb(scip_var, ub) + self._solver_model.chgVarType(scip_var, vtype) + + def write(self, filename, filetype=""): + """ + Write the model to a file (e.g., and lp file). + + Parameters + ---------- + filename: str + Name of the file to which the model should be written. + filetype: str + The file type (e.g., lp). + """ + self._solver_model.writeProblem(filename + filetype) + + def set_scip_param(self, param, val): + """ + Set a SCIP parameter. + + Parameters + ---------- + param: str + The SCIP parameter to set. Options include any SCIP parameter. + Please see the SCIP documentation for options. + val: any + The value to set the parameter to. See SCIP documentation for possible values. + """ + self._solver_model.setParam(param, val) + + def get_scip_param(self, param): + """ + Get the value of the SCIP parameter. + + Parameters + ---------- + param: str or int or float + The SCIP parameter to get the value of. See SCIP documentation for possible options. + """ + return self._solver_model.getParam(param) + + def _add_column(self, var, obj_coef, constraints, coefficients): + """Add a column to the solver's model + + This will add the Pyomo variable var to the solver's + model, and put the coefficients on the associated + constraints in the solver model. If the obj_coef is + not zero, it will add obj_coef*var to the objective + of the solver's model. + + Parameters + ---------- + var: Var (scalar Var or single _VarData) + obj_coef: float + constraints: list of solver constraints + coefficients: list of coefficients to put on var in the associated constraint + """ + + # Set-up add var + varname = self._symbol_map.getSymbol(var, self._labeler) + vtype = self._scip_vtype_from_var(var) + lb, ub = self._scip_lb_ub_from_var(var) + + # Add the variable to the model and then to all the constraints + scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype, name=varname) + self._pyomo_var_to_solver_var_expr_map[var] = scip_var + self._solver_var_to_pyomo_var_map[varname] = var + self._referenced_variables[var] = len(coefficients) + + # Get the SCIP cons by passing through two dictionaries + pyomo_cons = [self._solver_con_to_pyomo_con_map[con] for con in constraints] + scip_cons = [ + self._pyomo_con_to_solver_con_expr_map[pyomo_con] + for pyomo_con in pyomo_cons + ] + + for i, scip_con in enumerate(scip_cons): + if not scip_con.isLinear(): + raise ValueError( + "_add_column functionality not supported for non-linear constraints" + ) + self._solver_model.addConsCoeff(scip_con, scip_var, coefficients[i]) + con = self._solver_con_to_pyomo_con_map[scip_con.name] + self._vars_referenced_by_con[con].add(var) + + sense = self._solver_model.getObjectiveSense() + self._solver_model.setObjective(obj_coef * scip_var, sense=sense, clear=False) + + def reset(self): + self._solver_model.freeTransform() diff --git a/pyomo/solvers/tests/checks/test_SCIPDirect.py b/pyomo/solvers/tests/checks/test_SCIPDirect.py new file mode 100644 index 00000000000..ee37f5ddcc8 --- /dev/null +++ b/pyomo/solvers/tests/checks/test_SCIPDirect.py @@ -0,0 +1,335 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import sys + +import pyomo.common.unittest as unittest + +from pyomo.environ import ( + ConcreteModel, + AbstractModel, + Var, + Objective, + Block, + Constraint, + Suffix, + NonNegativeIntegers, + NonNegativeReals, + Integers, + Binary, + value, +) +from pyomo.opt import SolverFactory, TerminationCondition, SolutionStatus + +try: + import pyscipopt + + scip_available = True +except ImportError: + scip_available = False + + +class SCIPDirectTests(unittest.TestCase): + def setUp(self): + self.stderr = sys.stderr + sys.stderr = None + + def tearDown(self): + sys.stderr = self.stderr + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_infeasible_lp(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var(within=NonNegativeReals) + model.C1 = Constraint(expr=model.X == 1) + model.C2 = Constraint(expr=model.X == 2) + model.O = Objective(expr=model.X) + + results = opt.solve(model) + + self.assertEqual( + results.solver.termination_condition, TerminationCondition.infeasible + ) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_unbounded_lp(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var() + model.O = Objective(expr=model.X) + + results = opt.solve(model) + + self.assertIn( + results.solver.termination_condition, + ( + TerminationCondition.unbounded, + TerminationCondition.infeasibleOrUnbounded, + ), + ) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_optimal_lp(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var(within=NonNegativeReals) + model.O = Objective(expr=model.X) + + results = opt.solve(model, load_solutions=False) + + self.assertEqual(results.solution.status, SolutionStatus.optimal) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_get_duals_lp(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var(within=NonNegativeReals) + model.Y = Var(within=NonNegativeReals) + + model.C1 = Constraint(expr=2 * model.X + model.Y >= 8) + model.C2 = Constraint(expr=model.X + 3 * model.Y >= 6) + + model.O = Objective(expr=model.X + model.Y) + + results = opt.solve(model, suffixes=["dual"], load_solutions=False) + + model.dual = Suffix(direction=Suffix.IMPORT) + model.solutions.load_from(results) + + self.assertAlmostEqual(model.dual[model.C1], 0.4) + self.assertAlmostEqual(model.dual[model.C2], 0.2) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_infeasible_mip(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var(within=NonNegativeIntegers) + model.C1 = Constraint(expr=model.X == 1) + model.C2 = Constraint(expr=model.X == 2) + model.O = Objective(expr=model.X) + + results = opt.solve(model) + + self.assertEqual( + results.solver.termination_condition, TerminationCondition.infeasible + ) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_unbounded_mip(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = AbstractModel() + model.X = Var(within=Integers) + model.O = Objective(expr=model.X) + + instance = model.create_instance() + results = opt.solve(instance) + + self.assertIn( + results.solver.termination_condition, + ( + TerminationCondition.unbounded, + TerminationCondition.infeasibleOrUnbounded, + ), + ) + + @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") + def test_optimal_mip(self): + with SolverFactory("scip_direct", solver_io="python") as opt: + model = ConcreteModel() + model.X = Var(within=NonNegativeIntegers) + model.O = Objective(expr=model.X) + + results = opt.solve(model, load_solutions=False) + + self.assertEqual(results.solution.status, SolutionStatus.optimal) + + +@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") +class TestAddVar(unittest.TestCase): + def test_add_single_variable(self): + """Test that the variable is added correctly to `solver_model`.""" + model = ConcreteModel() + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNVars(), 0) + + model.X = Var(within=Binary) + + opt._add_var(model.X) + + self.assertEqual(opt._solver_model.getNVars(), 1) + self.assertEqual(opt._solver_model.getVars()[0].vtype(), "BINARY") + + def test_add_block_containing_single_variable(self): + """Test that the variable is added correctly to `solver_model`.""" + model = ConcreteModel() + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNVars(), 0) + + model.X = Var(within=Binary) + + opt._add_block(model) + + self.assertEqual(opt._solver_model.getNVars(), 1) + self.assertEqual(opt._solver_model.getVars()[0].vtype(), "BINARY") + + def test_add_block_containing_multiple_variables(self): + """Test that: + - The variable is added correctly to `solver_model` + - Fixed variable bounds are set correctly + """ + model = ConcreteModel() + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNVars(), 0) + + model.X1 = Var(within=Binary) + model.X2 = Var(within=NonNegativeReals) + model.X3 = Var(within=NonNegativeIntegers) + + model.X3.fix(5) + + opt._add_block(model) + + self.assertEqual(opt._solver_model.getNVars(), 3) + scip_vars = opt._solver_model.getVars() + vtypes = [scip_var.vtype() for scip_var in scip_vars] + assert "BINARY" in vtypes and "CONTINUOUS" in vtypes and "INTEGER" in vtypes + lbs = [scip_var.getLbGlobal() for scip_var in scip_vars] + ubs = [scip_var.getUbGlobal() for scip_var in scip_vars] + assert 0 in lbs and 5 in lbs + assert ( + 1 in ubs + and 5 in ubs + and any([opt._solver_model.isInfinity(ub) for ub in ubs]) + ) + + +@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") +class TestAddCon(unittest.TestCase): + def test_add_single_constraint(self): + model = ConcreteModel() + model.X = Var(within=Binary) + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNConss(), 0) + + model.C = Constraint(expr=model.X == 1) + + opt._add_constraint(model.C) + + self.assertEqual(opt._solver_model.getNConss(), 1) + con = opt._solver_model.getConss()[0] + self.assertEqual(con.isLinear(), 1) + self.assertEqual(opt._solver_model.getRhs(con), 1) + + def test_add_block_containing_single_constraint(self): + model = ConcreteModel() + model.X = Var(within=Binary) + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNConss(), 0) + + model.B = Block() + model.B.C = Constraint(expr=model.X == 1) + + opt._add_block(model.B) + + self.assertEqual(opt._solver_model.getNConss(), 1) + con = opt._solver_model.getConss()[0] + self.assertEqual(con.isLinear(), 1) + self.assertEqual(opt._solver_model.getRhs(con), 1) + + def test_add_block_containing_multiple_constraints(self): + model = ConcreteModel() + model.X = Var(within=Binary) + + opt = SolverFactory("scip_direct", solver_io="python") + opt._set_instance(model) + + self.assertEqual(opt._solver_model.getNConss(), 0) + + model.B = Block() + model.B.C1 = Constraint(expr=model.X == 1) + model.B.C2 = Constraint(expr=model.X <= 1) + model.B.C3 = Constraint(expr=model.X >= 1) + + opt._add_block(model.B) + + self.assertEqual(opt._solver_model.getNConss(), 3) + + +@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") +class TestLoadVars(unittest.TestCase): + def setUp(self): + opt = SolverFactory("scip_direct", solver_io="python") + model = ConcreteModel() + model.X = Var(within=NonNegativeReals, initialize=0) + model.Y = Var(within=NonNegativeReals, initialize=0) + + model.C1 = Constraint(expr=2 * model.X + model.Y >= 8) + model.C2 = Constraint(expr=model.X + 3 * model.Y >= 6) + + model.O = Objective(expr=model.X + model.Y) + + opt.solve(model, load_solutions=False, save_results=False) + + self._model = model + self._opt = opt + + def test_all_vars_are_loaded(self): + self.assertTrue(self._model.X.stale) + self.assertTrue(self._model.Y.stale) + self.assertEqual(value(self._model.X), 0) + self.assertEqual(value(self._model.Y), 0) + + self._opt.load_vars() + + self.assertFalse(self._model.X.stale) + self.assertFalse(self._model.Y.stale) + self.assertAlmostEqual(value(self._model.X), 3.6) + self.assertAlmostEqual(value(self._model.Y), 0.8) + + def test_only_specified_vars_are_loaded(self): + self.assertTrue(self._model.X.stale) + self.assertTrue(self._model.Y.stale) + self.assertEqual(value(self._model.X), 0) + self.assertEqual(value(self._model.Y), 0) + + self._opt.load_vars([self._model.X]) + + self.assertFalse(self._model.X.stale) + self.assertTrue(self._model.Y.stale) + self.assertAlmostEqual(value(self._model.X), 3.6) + self.assertEqual(value(self._model.Y), 0) + + self._opt.load_vars([self._model.Y]) + + self.assertFalse(self._model.X.stale) + self.assertFalse(self._model.Y.stale) + self.assertAlmostEqual(value(self._model.X), 3.6) + self.assertAlmostEqual(value(self._model.Y), 0.8) + + +if __name__ == "__main__": + unittest.main() diff --git a/pyomo/solvers/tests/checks/test_SCIPPersistent.py b/pyomo/solvers/tests/checks/test_SCIPPersistent.py new file mode 100644 index 00000000000..0cf1aab65f6 --- /dev/null +++ b/pyomo/solvers/tests/checks/test_SCIPPersistent.py @@ -0,0 +1,318 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2024 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import pyomo.environ +import pyomo.common.unittest as unittest + +from pyomo.core import ( + ConcreteModel, + Var, + Objective, + Constraint, + NonNegativeReals, + NonNegativeIntegers, + Reals, + Binary, + SOSConstraint, + Set, + sin, + cos, + exp, + log, +) +from pyomo.opt import SolverFactory + +try: + import pyscipopt + + scip_available = True +except ImportError: + scip_available = False + + +@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") +class TestQuadraticObjective(unittest.TestCase): + def test_quadratic_objective_linear_surrogate_is_set(self): + m = ConcreteModel() + m.X = Var(bounds=(-2, 2)) + m.Y = Var(bounds=(-2, 2)) + m.Z = Var(within=Reals) + m.O = Objective(expr=m.Z) + m.C1 = Constraint(expr=m.Y >= 2 * m.X - 1) + m.C2 = Constraint(expr=m.Y >= -m.X + 2) + m.C3 = Constraint(expr=m.Z >= m.X**2 + m.Y**2) + opt = SolverFactory("scip_persistent") + opt.set_instance(m) + opt.solve() + + self.assertAlmostEqual(m.X.value, 1, places=3) + self.assertAlmostEqual(m.Y.value, 1, places=3) + + opt.reset() + + opt.remove_constraint(m.C3) + del m.C3 + m.C3 = Constraint(expr=m.Z >= m.X**2) + opt.add_constraint(m.C3) + opt.solve() + self.assertAlmostEqual(m.X.value, 0, places=3) + self.assertAlmostEqual(m.Y.value, 2, places=3) + + def test_add_and_remove_sos(self): + m = ConcreteModel() + m.I = Set(initialize=[1, 2, 3]) + m.X = Var(m.I, bounds=(-2, 2)) + + m.C = SOSConstraint(var=m.X, sos=1) + + m.O = Objective(expr=m.X[1] + m.X[2]) + + opt = SolverFactory("scip_persistent") + + opt.set_instance(m) + opt.solve() + + zero_val_var = 0 + for i in range(1, 4): + if -0.001 < m.X[i].value < 0.001: + zero_val_var += 1 + assert zero_val_var == 2 + + opt.reset() + + opt.remove_sos_constraint(m.C) + del m.C + + m.C = SOSConstraint(var=m.X, sos=2) + opt.add_sos_constraint(m.C) + + opt.solve() + + zero_val_var = 0 + for i in range(1, 4): + if -0.001 < m.X[i].value < 0.001: + zero_val_var += 1 + assert zero_val_var == 1 + + def test_get_and_set_param(self): + m = ConcreteModel() + m.X = Var(bounds=(-2, 2)) + m.O = Objective(expr=m.X) + m.C3 = Constraint(expr=m.X <= 2) + opt = SolverFactory("scip_persistent") + opt.set_instance(m) + + opt.set_scip_param("limits/time", 60) + + assert opt.get_scip_param("limits/time") == 60 + + def test_non_linear(self): + + PI = 3.141592653589793238462643 + NWIRES = 11 + DIAMETERS = [ + 0.207, + 0.225, + 0.244, + 0.263, + 0.283, + 0.307, + 0.331, + 0.362, + 0.394, + 0.4375, + 0.500, + ] + PRELOAD = 300.0 + MAXWORKLOAD = 1000.0 + MAXDEFLECT = 6.0 + DEFLECTPRELOAD = 1.25 + MAXFREELEN = 14.0 + MAXCOILDIAM = 3.0 + MAXSHEARSTRESS = 189000.0 + SHEARMOD = 11500000.0 + + m = ConcreteModel() + m.coil = Var(within=NonNegativeReals) + m.wire = Var(within=NonNegativeReals) + m.defl = Var( + bounds=(DEFLECTPRELOAD / (MAXWORKLOAD - PRELOAD), MAXDEFLECT / PRELOAD) + ) + m.ncoils = Var(within=NonNegativeIntegers) + m.const1 = Var(within=NonNegativeReals) + m.const2 = Var(within=NonNegativeReals) + m.volume = Var(within=NonNegativeReals) + m.I = Set(initialize=[i for i in range(NWIRES)]) + m.y = Var(m.I, within=Binary) + + m.O = Objective(expr=m.volume) + + m.c1 = Constraint( + expr=PI / 2 * (m.ncoils + 2) * m.coil * m.wire**2 - m.volume == 0 + ) + + m.c2 = Constraint(expr=m.coil / m.wire - m.const1 == 0) + + m.c3 = Constraint( + expr=(4 * m.const1 - 1) / (4 * m.const1 - 4) + 0.615 / m.const1 - m.const2 + == 0 + ) + + m.c4 = Constraint( + expr=8.0 * MAXWORKLOAD / PI * m.const1 * m.const2 + - MAXSHEARSTRESS * m.wire**2 + <= 0 + ) + + m.c5 = Constraint( + expr=8 / SHEARMOD * m.ncoils * m.const1**3 / m.wire - m.defl == 0 + ) + + m.c6 = Constraint( + expr=MAXWORKLOAD * m.defl + 1.05 * m.ncoils * m.wire + 2.1 * m.wire + <= MAXFREELEN + ) + + m.c7 = Constraint(expr=m.coil + m.wire <= MAXCOILDIAM) + + m.c8 = Constraint( + expr=sum(m.y[i] * DIAMETERS[i] for i in range(NWIRES)) - m.wire == 0 + ) + + m.c9 = Constraint(expr=sum(m.y[i] for i in range(NWIRES)) == 1) + + opt = SolverFactory("scip_persistent") + opt.set_instance(m) + + opt.solve() + + self.assertAlmostEqual(m.volume.value, 1.6924910128, places=2) + + def test_non_linear_unary_expressions(self): + + m = ConcreteModel() + m.X = Var(bounds=(1, 2)) + m.Y = Var(within=Reals) + + m.O = Objective(expr=m.Y) + + m.C = Constraint(expr=exp(m.X) == m.Y) + + opt = SolverFactory("scip_persistent") + opt.set_instance(m) + + opt.solve() + self.assertAlmostEqual(m.X.value, 1, places=3) + self.assertAlmostEqual(m.Y.value, exp(1), places=3) + + opt.reset() + opt.remove_constraint(m.C) + del m.C + + m.C = Constraint(expr=log(m.X) == m.Y) + opt.add_constraint(m.C) + opt.solve() + self.assertAlmostEqual(m.X.value, 1, places=3) + self.assertAlmostEqual(m.Y.value, 0, places=3) + + opt.reset() + opt.remove_constraint(m.C) + del m.C + + m.C = Constraint(expr=sin(m.X) == m.Y) + opt.add_constraint(m.C) + opt.solve() + self.assertAlmostEqual(m.X.value, 1, places=3) + self.assertAlmostEqual(m.Y.value, sin(1), places=3) + + opt.reset() + opt.remove_constraint(m.C) + del m.C + + m.C = Constraint(expr=cos(m.X) == m.Y) + opt.add_constraint(m.C) + opt.solve() + self.assertAlmostEqual(m.X.value, 2, places=3) + self.assertAlmostEqual(m.Y.value, cos(2), places=3) + + def test_add_column(self): + m = ConcreteModel() + m.x = Var(within=NonNegativeReals) + m.c = Constraint(expr=(0, m.x, 1)) + m.obj = Objective(expr=-m.x) + + opt = SolverFactory("scip_persistent") + opt.set_instance(m) + opt.solve() + self.assertAlmostEqual(m.x.value, 1) + + m.y = Var(within=NonNegativeReals) + + opt.reset() + + opt.add_column(m, m.y, -3, [m.c], [2]) + opt.solve() + + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 0.5) + + def test_add_column_exceptions(self): + m = ConcreteModel() + m.x = Var() + m.c = Constraint(expr=(0, m.x, 1)) + m.ci = Constraint([1, 2], rule=lambda m, i: (0, m.x, i + 1)) + m.cd = Constraint(expr=(0, -m.x, 1)) + m.cd.deactivate() + m.obj = Objective(expr=-m.x) + + opt = SolverFactory("scip_persistent") + + # set_instance not called + self.assertRaises(RuntimeError, opt.add_column, m, m.x, 0, [m.c], [1]) + + opt.set_instance(m) + + m2 = ConcreteModel() + m2.y = Var() + m2.c = Constraint(expr=(0, m.x, 1)) + + # different model than attached to opt + self.assertRaises(RuntimeError, opt.add_column, m2, m2.y, 0, [], []) + # pyomo var attached to different model + self.assertRaises(RuntimeError, opt.add_column, m, m2.y, 0, [], []) + + z = Var() + # pyomo var floating + self.assertRaises(RuntimeError, opt.add_column, m, z, -2, [m.c, z], [1]) + + m.y = Var() + # len(coefficients) == len(constraints) + self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c], [1, 2]) + self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c, z], [1]) + + # add indexed constraint + self.assertRaises(AttributeError, opt.add_column, m, m.y, -2, [m.ci], [1]) + # add something not a _ConstraintData + self.assertRaises(AttributeError, opt.add_column, m, m.y, -2, [m.x], [1]) + + # constraint not on solver model + self.assertRaises(KeyError, opt.add_column, m, m.y, -2, [m2.c], [1]) + + # inactive constraint + self.assertRaises(KeyError, opt.add_column, m, m.y, -2, [m.cd], [1]) + + opt.add_var(m.y) + # var already in solver model + self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c], [1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/pyomo/solvers/tests/solvers.py b/pyomo/solvers/tests/solvers.py index 918a801ae37..3ad944de8d1 100644 --- a/pyomo/solvers/tests/solvers.py +++ b/pyomo/solvers/tests/solvers.py @@ -376,6 +376,27 @@ def test_solver_cases(*args): name='scip', io='nl', capabilities=_scip_capabilities, import_suffixes=[] ) + # + # SCIP PERSISTENT + # + + _scip_persistent_capabilities = set( + [ + "linear", + "integer", + "quadratic_constraint", + "sos1", + "sos2", + ] + ) + + _test_solver_cases["scip_persistent", "python"] = initialize( + name="scip_persistent", + io="python", + capabilities=_scip_persistent_capabilities, + import_suffixes=["slack", "dual", "rc"], + ) + # # CONOPT # From 6a14f108636dc9afb4e854b2fb27512aeb719ad0 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 18 Mar 2024 17:20:01 +0100 Subject: [PATCH 002/104] Add SCIPPersistent to docs --- doc/OnlineDocs/library_reference/solvers/index.rst | 1 + .../library_reference/solvers/scip_persistent.rst | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 doc/OnlineDocs/library_reference/solvers/scip_persistent.rst diff --git a/doc/OnlineDocs/library_reference/solvers/index.rst b/doc/OnlineDocs/library_reference/solvers/index.rst index 400032df076..628f9cfdab0 100644 --- a/doc/OnlineDocs/library_reference/solvers/index.rst +++ b/doc/OnlineDocs/library_reference/solvers/index.rst @@ -9,3 +9,4 @@ Solver Interfaces gurobi_direct.rst gurobi_persistent.rst xpress_persistent.rst + scip_persistent.rst diff --git a/doc/OnlineDocs/library_reference/solvers/scip_persistent.rst b/doc/OnlineDocs/library_reference/solvers/scip_persistent.rst new file mode 100644 index 00000000000..63ed55b74e3 --- /dev/null +++ b/doc/OnlineDocs/library_reference/solvers/scip_persistent.rst @@ -0,0 +1,7 @@ +SCIPPersistent +================ + +.. autoclass:: pyomo.solvers.plugins.solvers.scip_persistent.SCIPPersistent + :members: + :inherited-members: + :show-inheritance: \ No newline at end of file From c1079090567bbe95b290402a3918c936a0ded576 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 11:48:19 +0100 Subject: [PATCH 003/104] Add SCIp to Github action scripts --- .github/workflows/test_branches.yml | 6 ++++++ .github/workflows/test_pr_and_main.yml | 6 ++++++ pyomo/solvers/plugins/solvers/scip_persistent.py | 5 +++++ 3 files changed, 17 insertions(+) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 55f903a37f9..89e789db5ba 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -268,6 +268,12 @@ jobs: || echo "WARNING: Gurobi is not available" python -m pip install --cache-dir cache/pip xpress \ || echo "WARNING: Xpress Community Edition is not available" + if [[ ${{matrix.python}} == pypy* ]]; then + echo "skipping SCIP for pypy" + else + python -m pip install --cache-dir cache/pip pyscipopt==5.0.0 \ + || echo "WARNING: SCIP is not available" + fi if [[ ${{matrix.python}} == pypy* ]]; then echo "skipping wntr for pypy" else diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 76ec6de951a..a6cf6ef7eec 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -298,6 +298,12 @@ jobs: || echo "WARNING: Gurobi is not available" python -m pip install --cache-dir cache/pip xpress \ || echo "WARNING: Xpress Community Edition is not available" + if [[ ${{matrix.python}} == pypy* ]]; then + echo "skipping SCIP for pypy" + else + python -m pip install --cache-dir cache/pip pyscipopt==5.0.0 \ + || echo "WARNING: SCIP is not available" + fi if [[ ${{matrix.python}} == pypy* ]]; then echo "skipping wntr for pypy" else diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index 408aa84633f..e28c91073ab 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -182,4 +182,9 @@ def _add_column(self, var, obj_coef, constraints, coefficients): self._solver_model.setObjective(obj_coef * scip_var, sense=sense, clear=False) def reset(self): + """ This function is necessary to call before making any changes to the + SCIP model after optimizing. It frees solution run specific information + that is not automatically done when changes to an already solved model + are made. Making changes to an already optimized model, e.g. adding additional + constraints will raise an error unless this function is called. """ self._solver_model.freeTransform() From e00ded8e33e823b8a9146facff85166291e17d71 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 14:55:52 +0100 Subject: [PATCH 004/104] Remove 5.0.0 specific version. Add conda to workflow --- .github/workflows/test_branches.yml | 4 ++-- .github/workflows/test_pr_and_main.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 89e789db5ba..1d61aaf2d77 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -271,7 +271,7 @@ jobs: if [[ ${{matrix.python}} == pypy* ]]; then echo "skipping SCIP for pypy" else - python -m pip install --cache-dir cache/pip pyscipopt==5.0.0 \ + python -m pip install --cache-dir cache/pip pyscipopt \ || echo "WARNING: SCIP is not available" fi if [[ ${{matrix.python}} == pypy* ]]; then @@ -347,7 +347,7 @@ jobs: if test -z "${{matrix.slim}}"; then PYVER=$(echo "py${{matrix.python}}" | sed 's/\.//g') echo "Installing for $PYVER" - for PKG in 'cplex>=12.10' docplex 'gurobi=10.0.3' xpress cyipopt pymumps scip; do + for PKG in 'cplex>=12.10' docplex 'gurobi=10.0.3' xpress cyipopt pymumps scip pyscipopt; do echo "" echo "*** Install $PKG ***" # conda can literally take an hour to determine that a diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index a6cf6ef7eec..89fd90c41d0 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -301,7 +301,7 @@ jobs: if [[ ${{matrix.python}} == pypy* ]]; then echo "skipping SCIP for pypy" else - python -m pip install --cache-dir cache/pip pyscipopt==5.0.0 \ + python -m pip install --cache-dir cache/pip pyscipopt \ || echo "WARNING: SCIP is not available" fi if [[ ${{matrix.python}} == pypy* ]]; then @@ -376,7 +376,7 @@ jobs: if test -z "${{matrix.slim}}"; then PYVER=$(echo "py${{matrix.python}}" | sed 's/\.//g') echo "Installing for $PYVER" - for PKG in 'cplex>=12.10' docplex 'gurobi=10.0.3' xpress cyipopt pymumps scip; do + for PKG in 'cplex>=12.10' docplex 'gurobi=10.0.3' xpress cyipopt pymumps scip pyscipopt; do echo "" echo "*** Install $PKG ***" # conda can literally take an hour to determine that a From a0b625060217e04aa126dc9f2cb9a410e0968078 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:14:54 +0100 Subject: [PATCH 005/104] Standardise string formatting to fstring --- pyomo/solvers/plugins/solvers/scip_direct.py | 35 +++++++------------ .../plugins/solvers/scip_persistent.py | 8 ++--- 2 files changed, 16 insertions(+), 27 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 0aafb596007..e93e5579f26 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -74,7 +74,7 @@ def _init(self): except ImportError: self._python_api_exists = False except Exception as e: - print("Import of pyscipopt failed - SCIP message=" + str(e) + "\n") + print(f"Import of pyscipopt failed - SCIP message={str(e)}\n") self._python_api_exists = False # Note: Undefined capabilities default to None @@ -104,7 +104,7 @@ def _apply_solver(self): if self._keepfiles: # Only save log file when the user wants to keep it. self._solver_model.setLogfile(self._log_file) - print("Solver log file: " + self._log_file) + print(f"Solver log file: {self._log_file}") # Set user specified parameters for key, option in self.options.items(): @@ -257,14 +257,14 @@ def __exit__(self, t, v, traceback): def _set_instance(self, model, kwds={}): DirectOrPersistentSolver._set_instance(self, model, kwds) + self.available() try: self._solver_model = self._scip.Model() except Exception: e = sys.exc_info()[1] msg = ( "Unable to create SCIP model. " - "Have you installed PySCIPOpt correctly?\n\n\t" - + "Error message: {0}".format(e) + f"Have you installed PySCIPOpt correctly?\n\n\t Error message: {e}" ) raise Exception(msg) @@ -275,14 +275,13 @@ def _set_instance(self, model, kwds={}): if var.fixed: if not self._output_fixed_variable_bounds: raise ValueError( - "Encountered a fixed variable (%s) inside " + f"Encountered a fixed variable {var.name} inside " "an active objective or constraint " - "expression on model %s, which is usually " + f"expression on model {self._pyomo_model.name}, which is usually " "indicative of a preprocessing error. Use " "the IO-option 'output_fixed_variable_bounds=True' " "to suppress this error and fix the variable " "by overwriting its bounds in the SCIP instance." - % (var.name, self._pyomo_model.name) ) def _add_block(self, block): @@ -308,14 +307,10 @@ def _add_constraint(self, con): if con.has_lb(): if not is_fixed(con.lower): - raise ValueError( - "Lower bound of constraint {0} is not constant.".format(con) - ) + raise ValueError(f"Lower bound of constraint {con} is not constant.") if con.has_ub(): if not is_fixed(con.upper): - raise ValueError( - "Upper bound of constraint {0} is not constant.".format(con) - ) + raise ValueError(f"Upper bound of constraint {con} is not constant.") if con.equality: scip_cons = self._solver_model.addCons( @@ -335,8 +330,7 @@ def _add_constraint(self, con): ) else: raise ValueError( - "Constraint does not have a lower " - "or an upper bound: {0} \n".format(con) + f"Constraint does not have a lower or an upper bound: {con} \n" ) for var in referenced_vars: @@ -398,9 +392,7 @@ def _scip_vtype_from_var(self, var): elif var.is_continuous(): vtype = "C" else: - raise ValueError( - "Variable domain type is not recognized for {0}".format(var.domain) - ) + raise ValueError(f"Variable domain type is not recognized for {var.domain}") return vtype def _set_objective(self, obj): @@ -418,7 +410,7 @@ def _set_objective(self, obj): elif obj.sense == maximize: sense = "maximize" else: - raise ValueError("Objective sense is not recognized: {0}".format(obj.sense)) + raise ValueError(f"Objective sense is not recognized: {obj.sense}") scip_expr, referenced_vars = self._get_expr_from_pyomo_expr( obj.expr, self._max_obj_degree @@ -455,8 +447,7 @@ def _postsolve(self): flag = True if not flag: raise RuntimeError( - "***The scip_direct solver plugin cannot extract solution suffix=" - + suffix + f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" ) scip = self._solver_model @@ -593,7 +584,7 @@ def _postsolve(self): else: self.results.solver.status = SolverStatus.error self.results.solver.termination_message = ( - "Unhandled SCIP status (" + str(status) + ")" + f"Unhandled SCIP status ({str(status)})" ) self.results.solver.termination_condition = TerminationCondition.error soln.status = SolutionStatus.error diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index e28c91073ab..abb85b8dbca 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -84,9 +84,7 @@ def update_var(self, var): # return if var not in self._pyomo_var_to_solver_var_map: raise ValueError( - "The Var provided to compile_var needs to be added first: {0}".format( - var - ) + f"The Var provided to compile_var needs to be added first: {var}" ) scip_var = self._pyomo_var_to_solver_var_map[var] vtype = self._scip_vtype_from_var(var) @@ -182,9 +180,9 @@ def _add_column(self, var, obj_coef, constraints, coefficients): self._solver_model.setObjective(obj_coef * scip_var, sense=sense, clear=False) def reset(self): - """ This function is necessary to call before making any changes to the + """This function is necessary to call before making any changes to the SCIP model after optimizing. It frees solution run specific information that is not automatically done when changes to an already solved model are made. Making changes to an already optimized model, e.g. adding additional - constraints will raise an error unless this function is called. """ + constraints will raise an error unless this function is called.""" self._solver_model.freeTransform() From d0816eb008bae43eff5bff54390873c98e5b7a1b Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:17:29 +0100 Subject: [PATCH 006/104] Add parameter link to docstring --- pyomo/solvers/plugins/solvers/scip_persistent.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index abb85b8dbca..49fe224e72a 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -116,6 +116,7 @@ def set_scip_param(self, param, val): param: str The SCIP parameter to set. Options include any SCIP parameter. Please see the SCIP documentation for options. + Link at: https://www.scipopt.org/doc/html/PARAMETERS.php val: any The value to set the parameter to. See SCIP documentation for possible values. """ @@ -129,6 +130,7 @@ def get_scip_param(self, param): ---------- param: str or int or float The SCIP parameter to get the value of. See SCIP documentation for possible options. + Link at: https://www.scipopt.org/doc/html/PARAMETERS.php """ return self._solver_model.getParam(param) From 0e11f112161b6947ddf30d6530d6167ef174cf44 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:20:07 +0100 Subject: [PATCH 007/104] Remove redundant second objective sense check --- pyomo/solvers/plugins/solvers/scip_direct.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index e93e5579f26..c6285ff53cb 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -591,15 +591,6 @@ def _postsolve(self): self.results.problem.name = scip.getProbName() - if scip.getObjectiveSense() == "minimize": - self.results.problem.sense = minimize - elif scip.getObjectiveSense() == "maximize": - self.results.problem.sense = maximize - else: - raise RuntimeError( - f"Unrecognized SCIP objective sense: {scip.getObjectiveSense()}" - ) - self.results.problem.upper_bound = None self.results.problem.lower_bound = None if scip.getNSols() > 0: From 068ec99277321743b611c5323be3858a5b505ce8 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:35:54 +0100 Subject: [PATCH 008/104] Clean up _post_solve with a helper function for status handling --- pyomo/solvers/plugins/solvers/scip_direct.py | 109 +++++++++++-------- 1 file changed, 63 insertions(+), 46 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index c6285ff53cb..9074d40870f 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -382,8 +382,16 @@ def _add_sos_constraint(self, con): def _scip_vtype_from_var(self, var): """ This function takes a pyomo variable and returns the appropriate SCIP variable type - :param var: pyomo.core.base.var.Var - :return: B, I, or C + + Parameters + ---------- + var: pyomo.core.base.var.Var + The pyomo variable that we want to retrieve the SCIP vtype of + + Returns + ------- + vtype: str + B for Binary, I for Integer, or C for Continuous """ if var.is_binary(): vtype = "B" @@ -425,52 +433,12 @@ def _set_objective(self, obj): self._needs_updated = True - def _postsolve(self): - # the only suffixes that we extract from SCIP are - # constraint duals, constraint slacks, and variable - # reduced-costs. scan through the solver suffix list - # and throw an exception if the user has specified - # any others. - extract_duals = False - extract_slacks = False - extract_reduced_costs = False - for suffix in self._suffixes: - flag = False - if re.match(suffix, "dual"): - extract_duals = True - flag = True - if re.match(suffix, "slack"): - extract_slacks = True - flag = True - if re.match(suffix, "rc"): - extract_reduced_costs = True - flag = True - if not flag: - raise RuntimeError( - f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" - ) - - scip = self._solver_model + def _get_solver_solution_status(self, scip, soln): + """ """ + # Get the status of the SCIP Model currently status = scip.getStatus() - scip_vars = scip.getVars() - n_bin_vars = sum([scip_var.vtype() == "BINARY" for scip_var in scip_vars]) - n_int_vars = sum([scip_var.vtype() == "INTEGER" for scip_var in scip_vars]) - n_con_vars = sum([scip_var.vtype() == "CONTINUOUS" for scip_var in scip_vars]) - - if n_bin_vars + n_int_vars > 0: - if extract_reduced_costs: - logger.warning("Cannot get reduced costs for MIP.") - if extract_duals: - logger.warning("Cannot get duals for MIP.") - extract_reduced_costs = False - extract_duals = False - - self.results = SolverResults() - soln = Solution() - - self.results.solver.name = f"SCIP{self._version}" - self.results.solver.wallclock_time = scip.getSolvingTime() + # Go through each potential case and update appropriately if scip.getStage() == 1: # SCIP Model is created but not yet optimized self.results.solver.status = SolverStatus.aborted self.results.solver.termination_message = ( @@ -588,6 +556,55 @@ def _postsolve(self): ) self.results.solver.termination_condition = TerminationCondition.error soln.status = SolutionStatus.error + return soln + + def _postsolve(self): + # the only suffixes that we extract from SCIP are + # constraint duals, constraint slacks, and variable + # reduced-costs. scan through the solver suffix list + # and throw an exception if the user has specified + # any others. + extract_duals = False + extract_slacks = False + extract_reduced_costs = False + for suffix in self._suffixes: + flag = False + if re.match(suffix, "dual"): + extract_duals = True + flag = True + if re.match(suffix, "slack"): + extract_slacks = True + flag = True + if re.match(suffix, "rc"): + extract_reduced_costs = True + flag = True + if not flag: + raise RuntimeError( + f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" + ) + + scip = self._solver_model + status = scip.getStatus() + scip_vars = scip.getVars() + n_bin_vars = sum([scip_var.vtype() == "BINARY" for scip_var in scip_vars]) + n_int_vars = sum([scip_var.vtype() == "INTEGER" for scip_var in scip_vars]) + n_con_vars = sum([scip_var.vtype() == "CONTINUOUS" for scip_var in scip_vars]) + + if n_bin_vars + n_int_vars > 0: + if extract_reduced_costs: + logger.warning("Cannot get reduced costs for MIP.") + if extract_duals: + logger.warning("Cannot get duals for MIP.") + extract_reduced_costs = False + extract_duals = False + + self.results = SolverResults() + soln = Solution() + + self.results.solver.name = f"SCIP{self._version}" + self.results.solver.wallclock_time = scip.getSolvingTime() + + soln = self._get_solver_solution_status(scip, soln) self.results.problem.name = scip.getProbName() From 63af6d8ce13a28b50a59c3d9a027d57db64d8ca6 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:38:45 +0100 Subject: [PATCH 009/104] Remove individual skip_test option --- pyomo/solvers/tests/checks/test_SCIPDirect.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pyomo/solvers/tests/checks/test_SCIPDirect.py b/pyomo/solvers/tests/checks/test_SCIPDirect.py index ee37f5ddcc8..cc9e114fed1 100644 --- a/pyomo/solvers/tests/checks/test_SCIPDirect.py +++ b/pyomo/solvers/tests/checks/test_SCIPDirect.py @@ -37,6 +37,7 @@ scip_available = False +@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") class SCIPDirectTests(unittest.TestCase): def setUp(self): self.stderr = sys.stderr @@ -45,7 +46,6 @@ def setUp(self): def tearDown(self): sys.stderr = self.stderr - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_infeasible_lp(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() @@ -60,7 +60,6 @@ def test_infeasible_lp(self): results.solver.termination_condition, TerminationCondition.infeasible ) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_unbounded_lp(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() @@ -77,7 +76,6 @@ def test_unbounded_lp(self): ), ) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_optimal_lp(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() @@ -88,7 +86,6 @@ def test_optimal_lp(self): self.assertEqual(results.solution.status, SolutionStatus.optimal) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_get_duals_lp(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() @@ -108,7 +105,6 @@ def test_get_duals_lp(self): self.assertAlmostEqual(model.dual[model.C1], 0.4) self.assertAlmostEqual(model.dual[model.C2], 0.2) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_infeasible_mip(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() @@ -123,7 +119,6 @@ def test_infeasible_mip(self): results.solver.termination_condition, TerminationCondition.infeasible ) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_unbounded_mip(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = AbstractModel() @@ -141,7 +136,6 @@ def test_unbounded_mip(self): ), ) - @unittest.skipIf(not scip_available, "The SCIP python bindings are not available") def test_optimal_mip(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() From 4075d9ad8ed1200c3d06e74ca5d3a678dd4e7239 Mon Sep 17 00:00:00 2001 From: Mark Turner <64978342+Opt-Mucca@users.noreply.github.com> Date: Tue, 19 Mar 2024 15:48:02 +0100 Subject: [PATCH 010/104] Update pyomo/solvers/plugins/solvers/scip_persistent.py Co-authored-by: Miranda Mundt <55767766+mrmundt@users.noreply.github.com> --- pyomo/solvers/plugins/solvers/scip_persistent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index 49fe224e72a..572a1b638e0 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -96,7 +96,7 @@ def update_var(self, var): def write(self, filename, filetype=""): """ - Write the model to a file (e.g., and lp file). + Write the model to a file (e.g., an lp file). Parameters ---------- From f55fcc5aaa329049a6ac521738537fb8498f1a36 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 15:52:10 +0100 Subject: [PATCH 011/104] Update from the black command --- pyomo/solvers/tests/solvers.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/pyomo/solvers/tests/solvers.py b/pyomo/solvers/tests/solvers.py index 3ad944de8d1..1a5c1671f19 100644 --- a/pyomo/solvers/tests/solvers.py +++ b/pyomo/solvers/tests/solvers.py @@ -381,13 +381,7 @@ def test_solver_cases(*args): # _scip_persistent_capabilities = set( - [ - "linear", - "integer", - "quadratic_constraint", - "sos1", - "sos2", - ] + ["linear", "integer", "quadratic_constraint", "sos1", "sos2"] ) _test_solver_cases["scip_persistent", "python"] = initialize( From 91eae7b573624fe35e5fb579debde19efbed740a Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 16:37:13 +0100 Subject: [PATCH 012/104] Fix typos --- pyomo/solvers/plugins/solvers/scip_direct.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 9074d40870f..5ba3395d1d2 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -94,7 +94,7 @@ def _init(self): def _apply_solver(self): StaleFlagManager.mark_all_as_stale() - # Supress solver output if requested + # Suppress solver output if requested if self._tee: self._solver_model.hideOutput(quiet=False) else: @@ -179,7 +179,7 @@ def get_nl_expr_recursively(pyomo_expr): elif isinstance(pyomo_expr, DivisionExpression): if len(scip_expr_list) != 2: raise ValueError( - f"DivisonExpression has {len(scip_expr_list)} many terms instead of two!" + f"DivisionExpression has {len(scip_expr_list)} many terms instead of two!" ) return scip_expr_list[0] / scip_expr_list[1] elif isinstance(pyomo_expr, UnaryFunctionExpression): From 5ee2007a05921ef90a5ef9b31e76a877c9899007 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Tue, 19 Mar 2024 17:28:07 +0100 Subject: [PATCH 013/104] Replace trySol via more safe checkSol --- pyomo/solvers/plugins/solvers/scip_direct.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 5ba3395d1d2..25c668a0a06 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -729,7 +729,14 @@ def _warm_start(self): for pyomo_var, scip_var in self._pyomo_var_to_solver_var_expr_map.items(): if pyomo_var.value is not None: scip_sol[scip_var] = value(pyomo_var) - self._solver_model.trySol(scip_sol, free=True) + feasible = self._solver_model.checkSol(scip_sol) + if feasible: + self._solver_model.addSol(scip_sol) + del scip_sol + else: + logger.warning("Warm start solution was not accepted by SCIP") + self._solver_model.freeSol(scip_sol) + del scip_sol def _load_vars(self, vars_to_load=None): var_map = self._pyomo_var_to_solver_var_expr_map From f6ff0923ba9dbbc2bfd04990846c1446ca9e9ed8 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Wed, 20 Mar 2024 10:01:01 +0100 Subject: [PATCH 014/104] Adds support for partial solution loading --- pyomo/solvers/plugins/solvers/scip_direct.py | 24 +++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 25c668a0a06..0d4ad722459 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -725,18 +725,30 @@ def warm_start_capable(self): return True def _warm_start(self): - scip_sol = self._solver_model.createSol() + partial_sol = False + for pyomo_var in self._pyomo_var_to_solver_var_expr_map: + if pyomo_var.value is None: + partial_sol = True + break + if partial_sol: + scip_sol = self._solver_model.createPartialSol() + else: + scip_sol = self._solver_model.createSol() for pyomo_var, scip_var in self._pyomo_var_to_solver_var_expr_map.items(): if pyomo_var.value is not None: scip_sol[scip_var] = value(pyomo_var) - feasible = self._solver_model.checkSol(scip_sol) - if feasible: + if partial_sol: self._solver_model.addSol(scip_sol) del scip_sol else: - logger.warning("Warm start solution was not accepted by SCIP") - self._solver_model.freeSol(scip_sol) - del scip_sol + feasible = self._solver_model.checkSol(scip_sol) + if feasible: + self._solver_model.addSol(scip_sol) + del scip_sol + else: + logger.warning("Warm start solution was not accepted by SCIP") + self._solver_model.freeSol(scip_sol) + del scip_sol def _load_vars(self, vars_to_load=None): var_map = self._pyomo_var_to_solver_var_expr_map From e7ac980a6e58a9b1035bafc38cf3ee55420900b8 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Thu, 21 Mar 2024 09:58:24 +0100 Subject: [PATCH 015/104] Add error handling for setting non-linear objective --- pyomo/solvers/plugins/solvers/scip_direct.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 0d4ad722459..456b370eff1 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -134,6 +134,15 @@ def _apply_solver(self): def _get_expr_from_pyomo_repn(self, repn, max_degree=None): referenced_vars = ComponentSet() + degree = repn.polynomial_degree() + if (max_degree is not None) and (degree > max_degree): + raise DegreeError( + "While SCIP supports general non-linear constraints, the objective must be linear. " + "Please reformulate the objective by introducing a new variable. " + "For min problems: min z s.t z >= f(x). For max problems: max z s.t z <= f(x). " + "f(x) is the original non-linear objective." + ) + new_expr = repn.constant if len(repn.linear_vars) > 0: From 2540f650df319bcca59cf26d0bc524a7fab7de8c Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Fri, 19 Apr 2024 18:44:29 +0200 Subject: [PATCH 016/104] Remove dual and rc loading for SCIP. Fix bug of ranged rows --- pyomo/solvers/plugins/solvers/scip_direct.py | 151 ++++++------------ .../plugins/solvers/scip_persistent.py | 21 ++- pyomo/solvers/tests/solvers.py | 2 +- 3 files changed, 73 insertions(+), 101 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 456b370eff1..04440b59f9b 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -86,10 +86,12 @@ def _init(self): self._capabilities.integer = True self._capabilities.sos1 = True self._capabilities.sos2 = True + self._skip_trivial_constraints = True # Dictionary used exclusively for SCIP, as we want the constraint expressions self._pyomo_var_to_solver_var_expr_map = ComponentMap() self._pyomo_con_to_solver_con_expr_map = dict() + self._pyomo_con_to_solver_expr_map = dict() def _apply_solver(self): StaleFlagManager.mark_all_as_stale() @@ -239,6 +241,7 @@ def _scip_lb_ub_from_var(self, var): ub = value(var.ub) else: ub = self._solver_model.infinity() + return lb, ub def _add_var(self, var): @@ -327,7 +330,10 @@ def _add_constraint(self, con): ) elif con.has_lb() and con.has_ub(): scip_cons = self._solver_model.addCons( - value(con.lower) <= (scip_expr <= value(con.upper)), name=conname + value(con.lower) <= scip_expr, name=conname + ) + self._solver_model.chgRhs( + scip_cons, value(con.upper) - value(con.body.constant) ) elif con.has_lb(): scip_cons = self._solver_model.addCons( @@ -346,6 +352,7 @@ def _add_constraint(self, con): self._referenced_variables[var] += 1 self._vars_referenced_by_con[con] = referenced_vars self._pyomo_con_to_solver_con_expr_map[con] = scip_cons + self._pyomo_con_to_solver_expr_map[con] = scip_expr self._pyomo_con_to_solver_con_map[con] = scip_cons.name self._solver_con_to_pyomo_con_map[conname] = con @@ -440,8 +447,6 @@ def _set_objective(self, obj): self._objective = obj self._vars_referenced_by_obj = referenced_vars - self._needs_updated = True - def _get_solver_solution_status(self, scip, soln): """ """ # Get the status of the SCIP Model currently @@ -569,24 +574,17 @@ def _get_solver_solution_status(self, scip, soln): def _postsolve(self): # the only suffixes that we extract from SCIP are - # constraint duals, constraint slacks, and variable - # reduced-costs. scan through the solver suffix list + # constraint slacks. constraint duals and variable + # reduced-costs were removed as in SCIP they contain + # too many caveats. scan through the solver suffix list # and throw an exception if the user has specified # any others. - extract_duals = False extract_slacks = False - extract_reduced_costs = False for suffix in self._suffixes: flag = False - if re.match(suffix, "dual"): - extract_duals = True - flag = True if re.match(suffix, "slack"): extract_slacks = True flag = True - if re.match(suffix, "rc"): - extract_reduced_costs = True - flag = True if not flag: raise RuntimeError( f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" @@ -599,14 +597,6 @@ def _postsolve(self): n_int_vars = sum([scip_var.vtype() == "INTEGER" for scip_var in scip_vars]) n_con_vars = sum([scip_var.vtype() == "CONTINUOUS" for scip_var in scip_vars]) - if n_bin_vars + n_int_vars > 0: - if extract_reduced_costs: - logger.warning("Cannot get reduced costs for MIP.") - if extract_duals: - logger.warning("Cannot get duals for MIP.") - extract_reduced_costs = False - extract_duals = False - self.results = SolverResults() soln = Solution() @@ -667,6 +657,7 @@ def _postsolve(self): This code in this if statement is only needed for backwards compatibility. It is more efficient to set _save_results to False and use load_vars, load_duals, etc. """ + if scip.getNSols() > 0: soln_variables = soln.variable soln_constraints = soln.constraint @@ -683,42 +674,35 @@ def _postsolve(self): if self._referenced_variables[pyomo_var] > 0: soln_variables[name] = {"Value": val} - if extract_reduced_costs: - vals = [scip.getVarRedcost(scip_var) for scip_var in scip_vars] - for scip_var, val, name in zip(scip_vars, vals, scip_var_names): - pyomo_var = self._solver_var_to_pyomo_var_map[name] - if self._referenced_variables[pyomo_var] > 0: - soln_variables[name]["Rc"] = val - - if extract_duals or extract_slacks: - scip_cons = scip.getConss() - con_names = [cons.name for cons in scip_cons] - assert set(self._solver_con_to_pyomo_con_map.keys()) == set( - con_names - ) - for name in con_names: - soln_constraints[name] = {} - - if extract_duals: - vals = [scip.getDualSolVal(con) for con in scip_cons] - for val, name in zip(vals, con_names): - soln_constraints[name]["Dual"] = val - if extract_slacks: - vals = [scip.getSlack(con, scip_sol) for con in scip_cons] - for val, name in zip(vals, con_names): - soln_constraints[name]["Slack"] = val + scip_cons = list(self._pyomo_con_to_solver_con_expr_map.values()) + con_names = [cons.name for cons in scip_cons] + if set(self._solver_con_to_pyomo_con_map.keys()) != set(con_names): + raise AssertionError( + f"{set(self._solver_con_to_pyomo_con_map.keys())}, {set(con_names)}" + ) + for cons in scip_cons: + if cons.getConshdlrName() in ["linear", "nonlinear"]: + soln_constraints[cons.name] = {} + pyomo_con = self._solver_con_to_pyomo_con_map[cons.name] + scip_expr = self._pyomo_con_to_solver_expr_map[pyomo_con] + activity = scip_sol[scip_expr] + if pyomo_con.has_lb(): + lhs = value(pyomo_con.lower) + else: + lhs = -1e20 + if pyomo_con.has_ub(): + rhs = value(pyomo_con.upper) + else: + rhs = 1e20 + soln_constraints[cons.name]["Slack"] = min( + activity - lhs, rhs - activity + ) elif self._load_solutions: if scip.getNSols() > 0: self.load_vars() - if extract_reduced_costs: - self._load_rc() - - if extract_duals: - self._load_duals() - if extract_slacks: self._load_slacks() @@ -773,65 +757,36 @@ def _load_vars(self, vars_to_load=None): var.set_value(val, skip_validation=True) def _load_rc(self, vars_to_load=None): - if not hasattr(self._pyomo_model, "rc"): - self._pyomo_model.rc = Suffix(direction=Suffix.IMPORT) - var_map = self._pyomo_var_to_solver_var_expr_map - ref_vars = self._referenced_variables - rc = self._pyomo_model.rc - if vars_to_load is None: - vars_to_load = var_map.keys() - - scip_vars_to_load = [var_map[pyomo_var] for pyomo_var in vars_to_load] - vals = [ - self._solver_model.getVarRedcost(scip_var) for scip_var in scip_vars_to_load - ] - - for var, val in zip(vars_to_load, vals): - if ref_vars[var] > 0: - rc[var] = val + raise NotImplementedError( + "SCIP via Pyomo does not support reduced cost loading." + ) def _load_duals(self, cons_to_load=None): - if not hasattr(self._pyomo_model, "dual"): - self._pyomo_model.dual = Suffix(direction=Suffix.IMPORT) - con_map = self._pyomo_con_to_solver_con_map - reverse_con_map = self._solver_con_to_pyomo_con_map - dual = self._pyomo_model.dual - scip_cons = self._solver_model.getConss() - - if cons_to_load is None: - con_names = [con.name for con in scip_cons] - vals = [self._solver_model.getDualSolVal(con) for con in scip_cons] - else: - con_names = set([con_map[pyomo_con] for pyomo_con in cons_to_load]) - scip_cons_to_load = [con for con in scip_cons if con.name in con_names] - vals = [self._solver_model.getDualSolVal(con) for con in scip_cons_to_load] - - for i, con_name in enumerate(con_names): - pyomo_con = reverse_con_map[con_name] - dual[pyomo_con] = vals[i] + raise NotImplementedError( + "SCIP via Pyomo does not support dual solution loading" + ) def _load_slacks(self, cons_to_load=None): if not hasattr(self._pyomo_model, "slack"): self._pyomo_model.slack = Suffix(direction=Suffix.IMPORT) - con_map = self._pyomo_con_to_solver_con_map - reverse_con_map = self._solver_con_to_pyomo_con_map slack = self._pyomo_model.slack - scip_cons = self._solver_model.getConss() scip_sol = self._solver_model.getBestSol() if cons_to_load is None: - con_names = [con.name for con in scip_cons] - vals = [self._solver_model.getSlack(con, scip_sol) for con in scip_cons] + scip_cons = list(self._pyomo_con_to_solver_con_expr_map.values()) else: - con_names = set([con_map[pyomo_con] for pyomo_con in cons_to_load]) - scip_cons_to_load = [con for con in scip_cons if con.name in con_names] - vals = [ - self._solver_model.getSlack(con, scip_sol) for con in scip_cons_to_load + scip_cons = [ + self._pyomo_con_to_solver_con_expr_map[pyomo_cons] + for pyomo_cons in cons_to_load ] - - for i, con_name in enumerate(con_names): - pyomo_con = reverse_con_map[con_name] - slack[pyomo_con] = vals[i] + for cons in scip_cons: + if cons.getConshdlrName() in ["linear", "nonlinear"]: + pyomo_con = self._solver_con_to_pyomo_con_map[cons.name] + scip_expr = self._pyomo_con_to_solver_expr_map[pyomo_con] + activity = scip_sol[scip_expr] + rhs = self._solver_model.getRhs(cons) + lhs = self._solver_model.getLhs(cons) + slack[pyomo_con] = min(activity - lhs, rhs - activity) def load_duals(self, cons_to_load=None): """ diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index 572a1b638e0..880380ced1f 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -8,7 +8,6 @@ # rights in this software. # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ - from pyomo.solvers.plugins.solvers.scip_direct import SCIPDirect from pyomo.solvers.plugins.solvers.persistent_solver import PersistentSolver from pyomo.opt.base import SolverFactory @@ -50,16 +49,34 @@ def _remove_constraint(self, solver_conname): con = self._solver_con_to_pyomo_con_map[solver_conname] scip_con = self._pyomo_con_to_solver_con_expr_map[con] self._solver_model.delCons(scip_con) - + for var in self._vars_reference_by_con[con]: + self._references_vars[var] -= 1 + del self._vars_reference_by_con[con] + del self._pyomo_con_to_solver_con_map[con] + del self._pyomo_con_to_solver_con_expr_map[con] + del self._pyomo_con_to_solver_expr_map[con] + del self._solver_con_to_pyomo_con_map[solver_conname] + + def _remove_sos_constraint(self, solver_sos_conname): con = self._solver_con_to_pyomo_con_map[solver_sos_conname] scip_con = self._pyomo_con_to_solver_con_expr_map[con] self._solver_model.delCons(scip_con) + for var in self._vars_reference_by_con[con]: + self._references_vars[var] -= 1 + del self._vars_reference_by_con[con] + del self._pyomo_con_to_solver_con_map[con] + del self._pyomo_con_to_solver_con_expr_map[con] + del self._solver_con_to_pyomo_con_map[solver_conname] def _remove_var(self, solver_varname): var = self._solver_var_to_pyomo_var_map[solver_varname] scip_var = self._pyomo_var_to_solver_var_expr_map[var] self._solver_model.delVar(scip_var) + del self._pyomo_var_to_solver_var_expr_map[var] + del self._pyomo_var_to_solver_var_map[var] + del self._solver_var_to_pyomo_var_map[scip_var.name] + del self._referenced_variables[var] def _warm_start(self): SCIPDirect._warm_start(self) diff --git a/pyomo/solvers/tests/solvers.py b/pyomo/solvers/tests/solvers.py index 1a5c1671f19..b66c1ca5af5 100644 --- a/pyomo/solvers/tests/solvers.py +++ b/pyomo/solvers/tests/solvers.py @@ -388,7 +388,7 @@ def test_solver_cases(*args): name="scip_persistent", io="python", capabilities=_scip_persistent_capabilities, - import_suffixes=["slack", "dual", "rc"], + import_suffixes=["slack"], ) # From 9e5d9442ea0900d36629a3f0677eb6c6ce8d7f19 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Fri, 19 Apr 2024 18:48:34 +0200 Subject: [PATCH 017/104] Add safe con.body.constant check --- pyomo/solvers/plugins/solvers/scip_direct.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 04440b59f9b..1b5e81db302 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -332,9 +332,10 @@ def _add_constraint(self, con): scip_cons = self._solver_model.addCons( value(con.lower) <= scip_expr, name=conname ) - self._solver_model.chgRhs( - scip_cons, value(con.upper) - value(con.body.constant) - ) + rhs = value(con.upper) + if hasattr(con.body, "constant"): + rhs -= value(con.body.constant) + self._solver_model.chgRhs(scip_cons, rhs) elif con.has_lb(): scip_cons = self._solver_model.addCons( value(con.lower) <= scip_expr, name=conname From f90dfade88dafd2d150409efd0a216c34578c89d Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 29 Apr 2024 10:33:26 +0200 Subject: [PATCH 018/104] Remove slack loading for SCIP --- pyomo/solvers/plugins/solvers/scip_direct.py | 72 +++---------------- .../plugins/solvers/scip_persistent.py | 15 ---- 2 files changed, 8 insertions(+), 79 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 1b5e81db302..57cfc213f3d 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -91,7 +91,6 @@ def _init(self): # Dictionary used exclusively for SCIP, as we want the constraint expressions self._pyomo_var_to_solver_var_expr_map = ComponentMap() self._pyomo_con_to_solver_con_expr_map = dict() - self._pyomo_con_to_solver_expr_map = dict() def _apply_solver(self): StaleFlagManager.mark_all_as_stale() @@ -353,7 +352,6 @@ def _add_constraint(self, con): self._referenced_variables[var] += 1 self._vars_referenced_by_con[con] = referenced_vars self._pyomo_con_to_solver_con_expr_map[con] = scip_cons - self._pyomo_con_to_solver_expr_map[con] = scip_expr self._pyomo_con_to_solver_con_map[con] = scip_cons.name self._solver_con_to_pyomo_con_map[conname] = con @@ -574,22 +572,17 @@ def _get_solver_solution_status(self, scip, soln): return soln def _postsolve(self): - # the only suffixes that we extract from SCIP are - # constraint slacks. constraint duals and variable + # Constraint duals and variable # reduced-costs were removed as in SCIP they contain - # too many caveats. scan through the solver suffix list + # too many caveats. Slacks were removed as later + # planned interfaces do not intend to support. + # Scan through the solver suffix list # and throw an exception if the user has specified # any others. - extract_slacks = False for suffix in self._suffixes: - flag = False - if re.match(suffix, "slack"): - extract_slacks = True - flag = True - if not flag: - raise RuntimeError( - f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" - ) + raise RuntimeError( + f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" + ) scip = self._solver_model status = scip.getStatus() @@ -661,8 +654,6 @@ def _postsolve(self): if scip.getNSols() > 0: soln_variables = soln.variable - soln_constraints = soln.constraint - scip_sol = scip.getBestSol() scip_vars = scip.getVars() scip_var_names = [scip_var.name for scip_var in scip_vars] @@ -675,38 +666,10 @@ def _postsolve(self): if self._referenced_variables[pyomo_var] > 0: soln_variables[name] = {"Value": val} - if extract_slacks: - scip_cons = list(self._pyomo_con_to_solver_con_expr_map.values()) - con_names = [cons.name for cons in scip_cons] - if set(self._solver_con_to_pyomo_con_map.keys()) != set(con_names): - raise AssertionError( - f"{set(self._solver_con_to_pyomo_con_map.keys())}, {set(con_names)}" - ) - for cons in scip_cons: - if cons.getConshdlrName() in ["linear", "nonlinear"]: - soln_constraints[cons.name] = {} - pyomo_con = self._solver_con_to_pyomo_con_map[cons.name] - scip_expr = self._pyomo_con_to_solver_expr_map[pyomo_con] - activity = scip_sol[scip_expr] - if pyomo_con.has_lb(): - lhs = value(pyomo_con.lower) - else: - lhs = -1e20 - if pyomo_con.has_ub(): - rhs = value(pyomo_con.upper) - else: - rhs = 1e20 - soln_constraints[cons.name]["Slack"] = min( - activity - lhs, rhs - activity - ) - elif self._load_solutions: if scip.getNSols() > 0: self.load_vars() - if extract_slacks: - self._load_slacks() - self.results.solution.insert(soln) # finally, clean any temporary files registered with the temp file @@ -768,26 +731,7 @@ def _load_duals(self, cons_to_load=None): ) def _load_slacks(self, cons_to_load=None): - if not hasattr(self._pyomo_model, "slack"): - self._pyomo_model.slack = Suffix(direction=Suffix.IMPORT) - slack = self._pyomo_model.slack - scip_sol = self._solver_model.getBestSol() - - if cons_to_load is None: - scip_cons = list(self._pyomo_con_to_solver_con_expr_map.values()) - else: - scip_cons = [ - self._pyomo_con_to_solver_con_expr_map[pyomo_cons] - for pyomo_cons in cons_to_load - ] - for cons in scip_cons: - if cons.getConshdlrName() in ["linear", "nonlinear"]: - pyomo_con = self._solver_con_to_pyomo_con_map[cons.name] - scip_expr = self._pyomo_con_to_solver_expr_map[pyomo_con] - activity = scip_sol[scip_expr] - rhs = self._solver_model.getRhs(cons) - lhs = self._solver_model.getLhs(cons) - slack[pyomo_con] = min(activity - lhs, rhs - activity) + raise NotImplementedError("SCIP via Pyomo does not support slack loading") def load_duals(self, cons_to_load=None): """ diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index 880380ced1f..e3fe9e37b5d 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -49,34 +49,19 @@ def _remove_constraint(self, solver_conname): con = self._solver_con_to_pyomo_con_map[solver_conname] scip_con = self._pyomo_con_to_solver_con_expr_map[con] self._solver_model.delCons(scip_con) - for var in self._vars_reference_by_con[con]: - self._references_vars[var] -= 1 - del self._vars_reference_by_con[con] - del self._pyomo_con_to_solver_con_map[con] del self._pyomo_con_to_solver_con_expr_map[con] - del self._pyomo_con_to_solver_expr_map[con] - del self._solver_con_to_pyomo_con_map[solver_conname] - def _remove_sos_constraint(self, solver_sos_conname): con = self._solver_con_to_pyomo_con_map[solver_sos_conname] scip_con = self._pyomo_con_to_solver_con_expr_map[con] self._solver_model.delCons(scip_con) - for var in self._vars_reference_by_con[con]: - self._references_vars[var] -= 1 - del self._vars_reference_by_con[con] - del self._pyomo_con_to_solver_con_map[con] del self._pyomo_con_to_solver_con_expr_map[con] - del self._solver_con_to_pyomo_con_map[solver_conname] def _remove_var(self, solver_varname): var = self._solver_var_to_pyomo_var_map[solver_varname] scip_var = self._pyomo_var_to_solver_var_expr_map[var] self._solver_model.delVar(scip_var) del self._pyomo_var_to_solver_var_expr_map[var] - del self._pyomo_var_to_solver_var_map[var] - del self._solver_var_to_pyomo_var_map[scip_var.name] - del self._referenced_variables[var] def _warm_start(self): SCIPDirect._warm_start(self) From f703d1f71128a95d509aa9ea0b08d12de2dcb41a Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 29 Apr 2024 10:38:10 +0200 Subject: [PATCH 019/104] Remove dual loading test for SCIP --- pyomo/solvers/tests/checks/test_SCIPDirect.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/pyomo/solvers/tests/checks/test_SCIPDirect.py b/pyomo/solvers/tests/checks/test_SCIPDirect.py index cc9e114fed1..5863a54bdcb 100644 --- a/pyomo/solvers/tests/checks/test_SCIPDirect.py +++ b/pyomo/solvers/tests/checks/test_SCIPDirect.py @@ -86,25 +86,6 @@ def test_optimal_lp(self): self.assertEqual(results.solution.status, SolutionStatus.optimal) - def test_get_duals_lp(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var(within=NonNegativeReals) - model.Y = Var(within=NonNegativeReals) - - model.C1 = Constraint(expr=2 * model.X + model.Y >= 8) - model.C2 = Constraint(expr=model.X + 3 * model.Y >= 6) - - model.O = Objective(expr=model.X + model.Y) - - results = opt.solve(model, suffixes=["dual"], load_solutions=False) - - model.dual = Suffix(direction=Suffix.IMPORT) - model.solutions.load_from(results) - - self.assertAlmostEqual(model.dual[model.C1], 0.4) - self.assertAlmostEqual(model.dual[model.C2], 0.2) - def test_infeasible_mip(self): with SolverFactory("scip_direct", solver_io="python") as opt: model = ConcreteModel() From 5c02d32009990b8054440f0a6049bdf934247a79 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 29 Apr 2024 10:42:00 +0200 Subject: [PATCH 020/104] Remove slack for suffix in tests --- pyomo/solvers/plugins/solvers/scip_direct.py | 2 -- pyomo/solvers/tests/solvers.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 57cfc213f3d..a965e66362e 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -10,7 +10,6 @@ # ___________________________________________________________________________ import logging -import re import sys from pyomo.common.collections import ComponentSet, ComponentMap, Bunch @@ -36,7 +35,6 @@ from pyomo.opt.results.solution import Solution, SolutionStatus from pyomo.opt.results.solver import TerminationCondition, SolverStatus from pyomo.opt.base import SolverFactory -from pyomo.core.base.suffix import Suffix logger = logging.getLogger("pyomo.solvers") diff --git a/pyomo/solvers/tests/solvers.py b/pyomo/solvers/tests/solvers.py index b66c1ca5af5..ba1530c67cc 100644 --- a/pyomo/solvers/tests/solvers.py +++ b/pyomo/solvers/tests/solvers.py @@ -388,7 +388,7 @@ def test_solver_cases(*args): name="scip_persistent", io="python", capabilities=_scip_persistent_capabilities, - import_suffixes=["slack"], + import_suffixes=[], ) # From 8ebcf88365267e28a5b820eedce12a0d1bf5473c Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Mon, 29 Apr 2024 11:13:28 +0200 Subject: [PATCH 021/104] Remove TODO for nonlinear handling --- pyomo/solvers/plugins/solvers/scip_direct.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index a965e66362e..9061deac6ad 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -161,7 +161,6 @@ def _get_expr_from_pyomo_repn(self, repn, max_degree=None): referenced_vars.add(x) referenced_vars.add(y) - # TODO: Introduce handling on non-linear expressions if repn.nonlinear_expr is not None: def get_nl_expr_recursively(pyomo_expr): From 30d8cc62d5903e9e8dce4f0cabe79f22a5aba495 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Thu, 13 Jun 2024 14:48:43 +0200 Subject: [PATCH 022/104] Skip LP_trivial_constraints for SCIP persistent --- pyomo/solvers/plugins/solvers/scip_direct.py | 6 ++++-- pyomo/solvers/tests/testcases.py | 9 +++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 9061deac6ad..39c3a4fd996 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -67,8 +67,10 @@ def _init(self): self._scip = pyscipopt self._python_api_exists = True - self._version = str(self._scip.Model().version()) - self._version_major = self._version.split(".")[0] + self._version = tuple( + int(k) for k in str(self._scip.Model().version()).split(".") + ) + self._version_major = self._version[0] except ImportError: self._python_api_exists = False except Exception as e: diff --git a/pyomo/solvers/tests/testcases.py b/pyomo/solvers/tests/testcases.py index 6bef40818d9..f586e22b1e1 100644 --- a/pyomo/solvers/tests/testcases.py +++ b/pyomo/solvers/tests/testcases.py @@ -248,6 +248,15 @@ "inside NL files. A ticket has been filed.", ) +# +# SCIP Persistent +# + +ExpectedFailures["scip_persistent", "python", "LP_trivial_constraints"] = ( + lambda v: v <= _trunk_version, + "SCIP does not allow empty constraints with no variables to be added to the Model.", +) + # # BARON # From 30e5e65bfd063b049f950b9f74c0d238187077ca Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Thu, 13 Jun 2024 16:12:24 +0200 Subject: [PATCH 023/104] Add transformation for add_cons with non float/int rhs e.g. np.int --- pyomo/solvers/plugins/solvers/scip_direct.py | 21 ++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 39c3a4fd996..7c26670c2b4 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -318,29 +318,38 @@ def _add_constraint(self, con): if con.has_lb(): if not is_fixed(con.lower): raise ValueError(f"Lower bound of constraint {con} is not constant.") + con_lower = value(con.lower) + if not isinstance(con_lower, (float, int)): + con_lower = float(con_lower) if con.has_ub(): if not is_fixed(con.upper): raise ValueError(f"Upper bound of constraint {con} is not constant.") + con_upper = value(con.upper) + if not isinstance(con_upper, (float, int)): + con_upper = float(con_upper) if con.equality: scip_cons = self._solver_model.addCons( - scip_expr == value(con.lower), name=conname + scip_expr == con_lower, name=conname ) elif con.has_lb() and con.has_ub(): scip_cons = self._solver_model.addCons( - value(con.lower) <= scip_expr, name=conname + con_lower <= scip_expr, name=conname ) - rhs = value(con.upper) + rhs = con_upper if hasattr(con.body, "constant"): - rhs -= value(con.body.constant) + con_constant = value(con.body.constant) + if not isinstance(con_constant, (float, int)): + con_body = float(con_constant) + rhs -= con_constant self._solver_model.chgRhs(scip_cons, rhs) elif con.has_lb(): scip_cons = self._solver_model.addCons( - value(con.lower) <= scip_expr, name=conname + con_lower <= scip_expr, name=conname ) elif con.has_ub(): scip_cons = self._solver_model.addCons( - scip_expr <= value(con.upper), name=conname + scip_expr <= con_upper, name=conname ) else: raise ValueError( From 9104a921c55f9cd170bd1a7e93e1628869de2360 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Thu, 13 Jun 2024 16:35:41 +0200 Subject: [PATCH 024/104] Add warning if type is converted. Tidy up logic --- pyomo/solvers/plugins/solvers/scip_direct.py | 30 ++++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 7c26670c2b4..6ce98d80e27 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -319,38 +319,38 @@ def _add_constraint(self, con): if not is_fixed(con.lower): raise ValueError(f"Lower bound of constraint {con} is not constant.") con_lower = value(con.lower) - if not isinstance(con_lower, (float, int)): + if type(con_lower) != float and type(con_lower) != int: + logger.warning( + f"Constraint {conname} has LHS type {type(value(con.lower))}. " + f"Converting to float as type is not allowed for SCIP." + ) con_lower = float(con_lower) if con.has_ub(): if not is_fixed(con.upper): raise ValueError(f"Upper bound of constraint {con} is not constant.") con_upper = value(con.upper) - if not isinstance(con_upper, (float, int)): + if type(con_upper) != float and type(con_upper) != int: + logger.warning( + f"Constraint {conname} has RHS type {type(value(con.upper))}. " + f"Converting to float as type is not allowed for SCIP." + ) con_upper = float(con_upper) if con.equality: - scip_cons = self._solver_model.addCons( - scip_expr == con_lower, name=conname - ) + scip_cons = self._solver_model.addCons(scip_expr == con_lower, name=conname) elif con.has_lb() and con.has_ub(): - scip_cons = self._solver_model.addCons( - con_lower <= scip_expr, name=conname - ) + scip_cons = self._solver_model.addCons(con_lower <= scip_expr, name=conname) rhs = con_upper if hasattr(con.body, "constant"): con_constant = value(con.body.constant) if not isinstance(con_constant, (float, int)): - con_body = float(con_constant) + con_constant = float(con_constant) rhs -= con_constant self._solver_model.chgRhs(scip_cons, rhs) elif con.has_lb(): - scip_cons = self._solver_model.addCons( - con_lower <= scip_expr, name=conname - ) + scip_cons = self._solver_model.addCons(con_lower <= scip_expr, name=conname) elif con.has_ub(): - scip_cons = self._solver_model.addCons( - scip_expr <= con_upper, name=conname - ) + scip_cons = self._solver_model.addCons(scip_expr <= con_upper, name=conname) else: raise ValueError( f"Constraint does not have a lower or an upper bound: {con} \n" From f3f2d7c0334afbea37e9d8d24d60227f333ee4c1 Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Wed, 20 Nov 2024 17:13:03 +0100 Subject: [PATCH 025/104] Fix num. vars and cons from transformed. Silent warm start fail --- pyomo/solvers/plugins/solvers/scip_direct.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 6ce98d80e27..89dd25b86ee 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -641,10 +641,9 @@ def _postsolve(self): except TypeError: soln.gap = None - # TODO: Should these values be of the transformed or the original problem? - self.results.problem.number_of_constraints = scip.getNConss() + self.results.problem.number_of_constraints = scip.getNConss(transformed=False) # self.results.problem.number_of_nonzeros = None - self.results.problem.number_of_variables = scip.getNVars() + self.results.problem.number_of_variables = scip.getNVars(transformed=False) self.results.problem.number_of_binary_variables = n_bin_vars self.results.problem.number_of_integer_variables = n_int_vars self.results.problem.number_of_continuous_variables = n_con_vars @@ -704,16 +703,13 @@ def _warm_start(self): scip_sol[scip_var] = value(pyomo_var) if partial_sol: self._solver_model.addSol(scip_sol) - del scip_sol else: - feasible = self._solver_model.checkSol(scip_sol) + feasible = self._solver_model.checkSol(scip_sol, printreason=not self._tee) if feasible: self._solver_model.addSol(scip_sol) - del scip_sol else: logger.warning("Warm start solution was not accepted by SCIP") self._solver_model.freeSol(scip_sol) - del scip_sol def _load_vars(self, vars_to_load=None): var_map = self._pyomo_var_to_solver_var_expr_map From 7b18354386df3954cf9fc59e5f0ec9b587e316ed Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Wed, 19 Feb 2025 16:38:06 +0100 Subject: [PATCH 026/104] Add minor changes --- pyomo/solvers/plugins/solvers/scip_direct.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 89dd25b86ee..314fce40da5 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -322,19 +322,13 @@ def _add_constraint(self, con): if type(con_lower) != float and type(con_lower) != int: logger.warning( f"Constraint {conname} has LHS type {type(value(con.lower))}. " - f"Converting to float as type is not allowed for SCIP." + f"Converting to float as SCIP fails otherwise." ) con_lower = float(con_lower) if con.has_ub(): if not is_fixed(con.upper): raise ValueError(f"Upper bound of constraint {con} is not constant.") con_upper = value(con.upper) - if type(con_upper) != float and type(con_upper) != int: - logger.warning( - f"Constraint {conname} has RHS type {type(value(con.upper))}. " - f"Converting to float as type is not allowed for SCIP." - ) - con_upper = float(con_upper) if con.equality: scip_cons = self._solver_model.addCons(scip_expr == con_lower, name=conname) @@ -642,7 +636,6 @@ def _postsolve(self): soln.gap = None self.results.problem.number_of_constraints = scip.getNConss(transformed=False) - # self.results.problem.number_of_nonzeros = None self.results.problem.number_of_variables = scip.getNVars(transformed=False) self.results.problem.number_of_binary_variables = n_bin_vars self.results.problem.number_of_integer_variables = n_int_vars From 27e3d108662f554966819c4ea5db52df1ca38ffc Mon Sep 17 00:00:00 2001 From: Mark Turner Date: Fri, 21 Mar 2025 11:26:23 +0100 Subject: [PATCH 027/104] Change copyright 2024 to 2025 --- pyomo/solvers/plugins/solvers/scip_direct.py | 2 +- pyomo/solvers/plugins/solvers/scip_persistent.py | 2 +- pyomo/solvers/tests/checks/test_SCIPDirect.py | 2 +- pyomo/solvers/tests/checks/test_SCIPPersistent.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/solvers/plugins/solvers/scip_direct.py index 314fce40da5..c862d9047c1 100644 --- a/pyomo/solvers/plugins/solvers/scip_direct.py +++ b/pyomo/solvers/plugins/solvers/scip_direct.py @@ -1,7 +1,7 @@ # ___________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2024 +# Copyright (c) 2008-2025 # National Technology and Engineering Solutions of Sandia, LLC # Under the terms of Contract DE-NA0003525 with National Technology and # Engineering Solutions of Sandia, LLC, the U.S. Government retains certain diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/solvers/plugins/solvers/scip_persistent.py index e3fe9e37b5d..bc64edc28a8 100644 --- a/pyomo/solvers/plugins/solvers/scip_persistent.py +++ b/pyomo/solvers/plugins/solvers/scip_persistent.py @@ -1,7 +1,7 @@ # ___________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2024 +# Copyright (c) 2008-2025 # National Technology and Engineering Solutions of Sandia, LLC # Under the terms of Contract DE-NA0003525 with National Technology and # Engineering Solutions of Sandia, LLC, the U.S. Government retains certain diff --git a/pyomo/solvers/tests/checks/test_SCIPDirect.py b/pyomo/solvers/tests/checks/test_SCIPDirect.py index 5863a54bdcb..186de0eaf58 100644 --- a/pyomo/solvers/tests/checks/test_SCIPDirect.py +++ b/pyomo/solvers/tests/checks/test_SCIPDirect.py @@ -1,7 +1,7 @@ # ___________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2024 +# Copyright (c) 2008-2025 # National Technology and Engineering Solutions of Sandia, LLC # Under the terms of Contract DE-NA0003525 with National Technology and # Engineering Solutions of Sandia, LLC, the U.S. Government retains certain diff --git a/pyomo/solvers/tests/checks/test_SCIPPersistent.py b/pyomo/solvers/tests/checks/test_SCIPPersistent.py index 0cf1aab65f6..61cf7385352 100644 --- a/pyomo/solvers/tests/checks/test_SCIPPersistent.py +++ b/pyomo/solvers/tests/checks/test_SCIPPersistent.py @@ -1,7 +1,7 @@ # ___________________________________________________________________________ # # Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2024 +# Copyright (c) 2008-2025 # National Technology and Engineering Solutions of Sandia, LLC # Under the terms of Contract DE-NA0003525 with National Technology and # Engineering Solutions of Sandia, LLC, the U.S. Government retains certain From f2066ea16e4a92d390f41cd32ee4debc38a1dbd1 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 18 May 2025 09:14:22 -0600 Subject: [PATCH 028/104] working on a model observer --- pyomo/contrib/observer/__init__.py | 0 pyomo/contrib/observer/model_observer.py | 741 +++++++++++++++++++++++ 2 files changed, 741 insertions(+) create mode 100644 pyomo/contrib/observer/__init__.py create mode 100644 pyomo/contrib/observer/model_observer.py diff --git a/pyomo/contrib/observer/__init__.py b/pyomo/contrib/observer/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py new file mode 100644 index 00000000000..b644676f3d2 --- /dev/null +++ b/pyomo/contrib/observer/model_observer.py @@ -0,0 +1,741 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# __________________________________________________________________________ + +import abc +import datetime +from typing import List, Sequence + +from pyomo.common.config import ConfigDict, ConfigValue +from pyomo.core.base.constraint import ConstraintData, Constraint +from pyomo.core.base.sos import SOSConstraintData, SOSConstraint +from pyomo.core.base.var import VarData +from pyomo.core.base.param import ParamData, Param +from pyomo.core.base.objective import ObjectiveData +from pyomo.core.staleflag import StaleFlagManager +from pyomo.common.collections import ComponentMap +from pyomo.common.timing import HierarchicalTimer +from pyomo.contrib.solver.common.results import Results +from pyomo.contrib.solver.common.util import collect_vars_and_named_exprs, get_objective +from pyomo.common.numeric_types import native_numeric_types + + +class AutoUpdateConfig(ConfigDict): + """ + Control which parts of the model are automatically checked and/or updated upon re-solve + """ + + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + if doc is None: + doc = 'Configuration options to detect changes in model between solves' + super().__init__( + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + + self.check_for_new_or_removed_constraints: bool = self.declare( + 'check_for_new_or_removed_constraints', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, new/old constraints will not be automatically detected on subsequent + solves. Use False only when manually updating the solver with opt.add_constraints() + and opt.remove_constraints() or when you are certain constraints are not being + added to/removed from the model.""", + ), + ) + self.check_for_new_or_removed_vars: bool = self.declare( + 'check_for_new_or_removed_vars', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, new/old variables will not be automatically detected on subsequent + solves. Use False only when manually updating the solver with opt.add_variables() and + opt.remove_variables() or when you are certain variables are not being added to / + removed from the model.""", + ), + ) + self.check_for_new_or_removed_params: bool = self.declare( + 'check_for_new_or_removed_params', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, new/old parameters will not be automatically detected on subsequent + solves. Use False only when manually updating the solver with opt.add_parameters() and + opt.remove_parameters() or when you are certain parameters are not being added to / + removed from the model.""", + ), + ) + self.check_for_new_objective: bool = self.declare( + 'check_for_new_objective', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, new/old objectives will not be automatically detected on subsequent + solves. Use False only when manually updating the solver with opt.set_objective() or + when you are certain objectives are not being added to / removed from the model.""", + ), + ) + self.update_constraints: bool = self.declare( + 'update_constraints', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to existing constraints will not be automatically detected on + subsequent solves. This includes changes to the lower, body, and upper attributes of + constraints. Use False only when manually updating the solver with + opt.remove_constraints() and opt.add_constraints() or when you are certain constraints + are not being modified.""", + ), + ) + self.update_vars: bool = self.declare( + 'update_vars', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to existing variables will not be automatically detected on + subsequent solves. This includes changes to the lb, ub, domain, and fixed + attributes of variables. Use False only when manually updating the solver with + opt.update_variables() or when you are certain variables are not being modified.""", + ), + ) + self.update_parameters: bool = self.declare( + 'update_parameters', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to parameter values will not be automatically detected on + subsequent solves. Use False only when manually updating the solver with + opt.update_parameters() or when you are certain parameters are not being modified.""", + ), + ) + self.update_named_expressions: bool = self.declare( + 'update_named_expressions', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to Expressions will not be automatically detected on + subsequent solves. Use False only when manually updating the solver with + opt.remove_constraints() and opt.add_constraints() or when you are certain + Expressions are not being modified.""", + ), + ) + self.update_objective: bool = self.declare( + 'update_objective', + ConfigValue( + domain=bool, + default=True, + description=""" + If False, changes to objectives will not be automatically detected on + subsequent solves. This includes the expr and sense attributes of objectives. Use + False only when manually updating the solver with opt.set_objective() or when you are + certain objectives are not being modified.""", + ), + ) + + +class Observer(abc.ABC): + def __init__(self): + pass + + @abc.abstractmethod + def add_variables(self, variables: List[VarData]): + pass + + @abc.abstractmethod + def add_parameters(self, params: List[ParamData]): + pass + + @abc.abstractmethod + def add_constraints(self, cons: List[ConstraintData]): + pass + + @abc.abstractmethod + def add_sos_constraints(self, cons: List[SOSConstraintData]): + pass + + @abc.abstractmethod + def set_objective(self, obj: ObjectiveData): + pass + + @abc.abstractmethod + def remove_constraints(self, cons: List[ConstraintData]): + pass + + @abc.abstractmethod + def remove_sos_constraints(self, cons: List[SOSConstraintData]): + pass + + @abc.abstractmethod + def remove_variables(self, variables: List[VarData]): + pass + + @abc.abstractmethod + def remove_parameters(self, params: List[ParamData]): + pass + + +class ModelChangeDetector: + def __init__( + self, observers: Sequence[Observer], + treat_fixed_vars_as_params=True, + **kwds, + ): + """ + Parameters + ---------- + observers: Sequence[Observer] + The objects to notify when changes are made to the model + treat_fixed_vars_as_params: bool + This is an advanced option that should only be used in special circumstances. + With the default setting of True, fixed variables will be treated like parameters. + This means that z == x*y will be linear if x or y is fixed and the constraint + can be written to an LP file. If the value of the fixed variable gets changed, we have + to completely reprocess all constraints using that variable. If + treat_fixed_vars_as_params is False, then constraints will be processed as if fixed + variables are not fixed, and the solver will be told the variable is fixed. This means + z == x*y could not be written to an LP file even if x and/or y is fixed. However, + updating the values of fixed variables is much faster this way. + """ + self._observers: List[Observer] = list(observers) + self._model = None + self._active_constraints = {} # maps constraint to (lower, body, upper) + self._vars = {} # maps var id to (var, lb, ub, fixed, domain, value) + self._params = {} # maps param id to param + self._objective = None + self._objective_expr = None + self._objective_sense = None + self._named_expressions = ( + {} + ) # maps constraint to list of tuples (named_expr, named_expr.expr) + self._external_functions = ComponentMap() + self._obj_named_expressions = [] + self._referenced_variables = ( + {} + ) # var_id: [dict[constraints, None], dict[sos constraints, None], None or objective] + self._referenced_params = ( + {} + ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] + self._vars_referenced_by_con = {} + self._vars_referenced_by_obj = [] + self._params_referenced_by_con = {} + self._params_referenced_by_obj = [] + self._expr_types = None + self._treat_fixed_vars_as_params = treat_fixed_vars_as_params + self.config: AutoUpdateConfig = AutoUpdateConfig()(value=kwds, preserve_implicit=True) + + def set_instance(self, model): + saved_config = self.config + self.__init__(observers=self._observers, treat_fixed_vars_as_params=self._treat_fixed_vars_as_params) + self.config = saved_config + self._model = model + self.add_block(model) + if self._objective is None: + self.set_objective(None) + + def _add_variables(self, variables: List[VarData]): + for v in variables: + if id(v) in self._referenced_variables: + raise ValueError(f'Variable {v.name} has already been added') + self._referenced_variables[id(v)] = [{}, {}, None] + self._vars[id(v)] = ( + v, + v._lb, + v._ub, + v.fixed, + v.domain.get_interval(), + v.value, + ) + for obs in self._observers: + obs.add_variables(variables) + + def _add_parameters(self, params: List[ParamData]): + for p in params: + pid = id(p) + if pid in self._referenced_params: + raise ValueError(f'Parameter {p.name} has already been added') + self._referenced_params[pid] = [{}, {}, None] + self._params[id(p)] = (p, p.value) + for obs in self._observers: + obs.add_parameters(params) + + def _check_for_new_vars(self, variables: List[VarData]): + new_vars = {} + for v in variables: + v_id = id(v) + if v_id not in self._referenced_variables: + new_vars[v_id] = v + self._add_variables(list(new_vars.values())) + + def _check_to_remove_vars(self, variables: List[VarData]): + vars_to_remove = {} + for v in variables: + v_id = id(v) + ref_cons, ref_sos, ref_obj = self._referenced_variables[v_id] + if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: + vars_to_remove[v_id] = v + self._remove_variables(list(vars_to_remove.values())) + + def _check_for_new_params(self, params: List[ParamData]): + new_params = {} + for p in params: + pid = id(p) + if pid not in self._referenced_params: + new_params[pid] = p + self._add_parameters(list(new_params.values())) + + def _check_to_remove_params(self, params: List[ParamData]): + params_to_remove = {} + for p in params: + p_id = id(p) + ref_cons, ref_sos, ref_obj = self._referenced_params[p_id] + if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: + params_to_remove[p_id] = p + self._remove_parameters(list(params_to_remove.values())) + + def _add_constraints(self, cons: List[ConstraintData]): + all_fixed_vars = {} + for con in cons: + if con in self._named_expressions: + raise ValueError(f'Constraint {con.name} has already been added') + self._active_constraints[con] = con.expr + tmp = collect_vars_and_named_exprs(con.expr) + named_exprs, variables, fixed_vars, parameters, external_functions = tmp + self._check_for_new_vars(variables) + self._check_for_new_params(parameters) + self._named_expressions[con] = [(e, e.expr) for e in named_exprs] + if len(external_functions) > 0: + self._external_functions[con] = external_functions + self._vars_referenced_by_con[con] = variables + self._params_referenced_by_con[con] = parameters + for v in variables: + self._referenced_variables[id(v)][0][con] = None + for p in parameters: + self._referenced_params[id(p)][0][con] = None + if not self._treat_fixed_vars_as_params: + for v in fixed_vars: + v.unfix() + all_fixed_vars[id(v)] = v + for obs in self._observers: + obs.add_constraints(cons) + for v in all_fixed_vars.values(): + v.fix() + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + all_fixed_vars = {} + for con in cons: + if con in self._vars_referenced_by_con: + raise ValueError(f'Constraint {con.name} has already been added') + self._active_constraints[con] = tuple() + variables = [] + params = [] + for v, p in con.get_items(): + variables.append(v) + if type(p) in native_numeric_types: + continue + if p.is_parameter_type(): + params.append(p) + self._check_for_new_vars(variables) + self._check_for_new_params(params) + self._named_expressions[con] = [] + self._vars_referenced_by_con[con] = variables + self._params_referenced_by_con[con] = params + for v in variables: + self._referenced_variables[id(v)][1][con] = None + for p in params: + self._referenced_params[id(p)][1][con] = None + if not self._treat_fixed_vars_as_params: + for v in variables: + if v.is_fixed(): + v.unfix() + all_fixed_vars[id(v)] = v + for obs in self._observers: + obs.add_sos_constraints(cons) + for v in all_fixed_vars.values(): + v.fix() + + def _set_objective(self, obj: ObjectiveData): + if self._objective is not None: + for v in self._vars_referenced_by_obj: + self._referenced_variables[id(v)][2] = None + self._check_to_remove_vars(self._vars_referenced_by_obj) + self._external_functions.pop(self._objective, None) + if obj is not None: + self._objective = obj + self._objective_expr = obj.expr + self._objective_sense = obj.sense + tmp = collect_vars_and_named_exprs(obj.expr) + named_exprs, variables, fixed_vars, external_functions = tmp + self._check_for_new_vars(variables) + self._obj_named_expressions = [(i, i.expr) for i in named_exprs] + if len(external_functions) > 0: + self._external_functions[obj] = external_functions + self._vars_referenced_by_obj = variables + for v in variables: + self._referenced_variables[id(v)][2] = obj + if not self._treat_fixed_vars_as_params: + for v in fixed_vars: + v.unfix() + for obs in self._observers: + obs.set_objective(obj) + for v in fixed_vars: + v.fix() + else: + self._vars_referenced_by_obj = [] + self._objective = None + self._objective_expr = None + self._objective_sense = None + self._obj_named_expressions = [] + for obs in self._observers: + obs.set_objective(obj) + + def add_block(self, block): + param_dict = {} + for p in block.component_objects(Param, descend_into=True): + if p.mutable: + for _p in p.values(): + param_dict[id(_p)] = _p + self._add_parameters(list(param_dict.values())) + self._add_constraints( + list( + block.component_data_objects(Constraint, descend_into=True, active=True) + ) + ) + self._add_sos_constraints( + list( + block.component_data_objects( + SOSConstraint, descend_into=True, active=True + ) + ) + ) + obj = get_objective(block) + if obj is not None: + self._set_objective(obj) + + def _remove_constraints(self, cons: List[ConstraintData]): + for obs in self._observers: + obs.remove_constraints(cons) + for con in cons: + if con not in self._named_expressions: + raise ValueError( + f'Cannot remove constraint {con.name} - it was not added' + ) + for v in self._vars_referenced_by_con[con]: + self._referenced_variables[id(v)][0].pop(con) + self._check_to_remove_vars(self._vars_referenced_by_con[con]) + del self._active_constraints[con] + del self._named_expressions[con] + self._external_functions.pop(con, None) + del self._vars_referenced_by_con[con] + + def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + for obs in self._observers: + obs.remove_sos_constraints(cons) + for con in cons: + if con not in self._vars_referenced_by_con: + raise ValueError( + f'Cannot remove constraint {con.name} - it was not added' + ) + for v in self._vars_referenced_by_con[con]: + self._referenced_variables[id(v)][1].pop(con) + self._check_to_remove_vars(self._vars_referenced_by_con[con]) + del self._active_constraints[con] + del self._named_expressions[con] + del self._vars_referenced_by_con[con] + + def _remove_variables(self, variables: List[VarData]): + for obs in self._observers: + obs.remove_variables(variables) + for v in variables: + v_id = id(v) + if v_id not in self._referenced_variables: + raise ValueError( + f'Cannot remove variable {v.name} - it has not been added' + ) + cons_using, sos_using, obj_using = self._referenced_variables[v_id] + if cons_using or sos_using or (obj_using is not None): + raise ValueError( + f'Cannot remove variable {v.name} - it is still being used by constraints or the objective' + ) + del self._referenced_variables[v_id] + del self._vars[v_id] + + def _remove_parameters(self, params: List[ParamData]): + for obs in self._observers: + obs.remove_parameters(params) + for p in params: + del self._params[id(p)] + + def _remove_block(self, block): + self._remove_constraints( + list( + block.component_data_objects( + ctype=Constraint, descend_into=True, active=True + ) + ) + ) + self._remove_sos_constraints( + list( + block.component_data_objects( + ctype=SOSConstraint, descend_into=True, active=True + ) + ) + ) + self._remove_parameters( + list( + dict( + (id(p), p) + for p in block.component_data_objects( + ctype=Param, descend_into=True + ) + ).values() + ) + ) + + @abc.abstractmethod + def _update_variables(self, variables: List[VarData]): + pass + + def update_variables(self, variables: List[VarData]): + for v in variables: + self._vars[id(v)] = ( + v, + v._lb, + v._ub, + v.fixed, + v.domain.get_interval(), + v.value, + ) + self._update_variables(variables) + + @abc.abstractmethod + def update_parameters(self): + pass + + def update(self, timer: HierarchicalTimer = None): + if timer is None: + timer = HierarchicalTimer() + config = self._active_config.auto_updates + new_vars = [] + old_vars = [] + new_params = [] + old_params = [] + new_cons = [] + old_cons = [] + old_sos = [] + new_sos = [] + current_cons_dict = {} + current_sos_dict = {} + timer.start('vars') + if config.update_vars: + start_vars = {v_id: v_tuple[0] for v_id, v_tuple in self._vars.items()} + timer.stop('vars') + timer.start('params') + if config.check_for_new_or_removed_params: + current_params_dict = {} + for p in self._model.component_objects(Param, descend_into=True): + if p.mutable: + for _p in p.values(): + current_params_dict[id(_p)] = _p + for p_id, p in current_params_dict.items(): + if p_id not in self._params: + new_params.append(p) + for p_id, p in self._params.items(): + if p_id not in current_params_dict: + old_params.append(p) + timer.stop('params') + timer.start('cons') + if config.check_for_new_or_removed_constraints or config.update_constraints: + current_cons_dict = { + c: None + for c in self._model.component_data_objects( + Constraint, descend_into=True, active=True + ) + } + current_sos_dict = { + c: None + for c in self._model.component_data_objects( + SOSConstraint, descend_into=True, active=True + ) + } + for c in current_cons_dict.keys(): + if c not in self._vars_referenced_by_con: + new_cons.append(c) + for c in current_sos_dict.keys(): + if c not in self._vars_referenced_by_con: + new_sos.append(c) + for c in self._vars_referenced_by_con: + if c not in current_cons_dict and c not in current_sos_dict: + if (c.ctype is Constraint) or ( + c.ctype is None and isinstance(c, ConstraintData) + ): + old_cons.append(c) + else: + assert (c.ctype is SOSConstraint) or ( + c.ctype is None and isinstance(c, SOSConstraintData) + ) + old_sos.append(c) + self.remove_constraints(old_cons) + self.remove_sos_constraints(old_sos) + timer.stop('cons') + timer.start('params') + self.remove_parameters(old_params) + + # sticking this between removal and addition + # is important so that we don't do unnecessary work + if config.update_parameters: + self.update_parameters() + + self.add_parameters(new_params) + timer.stop('params') + timer.start('vars') + self.add_variables(new_vars) + timer.stop('vars') + timer.start('cons') + self.add_constraints(new_cons) + self.add_sos_constraints(new_sos) + new_cons_set = set(new_cons) + new_sos_set = set(new_sos) + cons_to_remove_and_add = {} + need_to_set_objective = False + if config.update_constraints: + for c in current_cons_dict.keys(): + if c not in new_cons_set and c.expr is not self._active_constraints[c]: + cons_to_remove_and_add[c] = None + sos_to_update = [] + for c in current_sos_dict.keys(): + if c not in new_sos_set: + sos_to_update.append(c) + self.remove_sos_constraints(sos_to_update) + self.add_sos_constraints(sos_to_update) + timer.stop('cons') + timer.start('vars') + if config.update_vars: + end_vars = {v_id: v_tuple[0] for v_id, v_tuple in self._vars.items()} + vars_to_check = [v for v_id, v in end_vars.items() if v_id in start_vars] + if config.update_vars: + vars_to_update = [] + for v in vars_to_check: + _v, lb, ub, fixed, domain_interval, value = self._vars[id(v)] + if (fixed != v.fixed) or (fixed and (value != v.value)): + vars_to_update.append(v) + if self._treat_fixed_vars_as_params: + for c in self._referenced_variables[id(v)][0]: + cons_to_remove_and_add[c] = None + if self._referenced_variables[id(v)][2] is not None: + need_to_set_objective = True + elif lb is not v._lb: + vars_to_update.append(v) + elif ub is not v._ub: + vars_to_update.append(v) + elif domain_interval != v.domain.get_interval(): + vars_to_update.append(v) + self.update_variables(vars_to_update) + timer.stop('vars') + timer.start('cons') + cons_to_remove_and_add = list(cons_to_remove_and_add.keys()) + self.remove_constraints(cons_to_remove_and_add) + self.add_constraints(cons_to_remove_and_add) + timer.stop('cons') + timer.start('named expressions') + if config.update_named_expressions: + cons_to_update = [] + for c, expr_list in self._named_expressions.items(): + if c in new_cons_set: + continue + for named_expr, old_expr in expr_list: + if named_expr.expr is not old_expr: + cons_to_update.append(c) + break + self.remove_constraints(cons_to_update) + self.add_constraints(cons_to_update) + for named_expr, old_expr in self._obj_named_expressions: + if named_expr.expr is not old_expr: + need_to_set_objective = True + break + timer.stop('named expressions') + timer.start('objective') + if self._active_config.auto_updates.check_for_new_objective: + pyomo_obj = get_objective(self._model) + if pyomo_obj is not self._objective: + need_to_set_objective = True + else: + pyomo_obj = self._objective + if self._active_config.auto_updates.update_objective: + if pyomo_obj is not None and pyomo_obj.expr is not self._objective_expr: + need_to_set_objective = True + elif pyomo_obj is not None and pyomo_obj.sense is not self._objective_sense: + # we can definitely do something faster here than resetting the whole objective + need_to_set_objective = True + if need_to_set_objective: + self.set_objective(pyomo_obj) + timer.stop('objective') + + # this has to be done after the objective and constraints in case the + # old objective/constraints use old variables + timer.start('vars') + self.remove_variables(old_vars) + timer.stop('vars') + + +class PersistentSolverMixin: + """ + The `solve` method in Gurobi and Highs is exactly the same, so this Mixin + minimizes the duplicate code + """ + + def solve(self, model, **kwds) -> Results: + start_timestamp = datetime.datetime.now(datetime.timezone.utc) + self._active_config = config = self.config(value=kwds, preserve_implicit=True) + StaleFlagManager.mark_all_as_stale() + + if self._last_results_object is not None: + self._last_results_object.solution_loader.invalidate() + if config.timer is None: + config.timer = HierarchicalTimer() + timer = config.timer + + if model is not self._model: + timer.start('set_instance') + self.set_instance(model) + timer.stop('set_instance') + else: + timer.start('update') + self.update(timer=timer) + timer.stop('update') + + res = self._solve() + self._last_results_object = res + + end_timestamp = datetime.datetime.now(datetime.timezone.utc) + res.timing_info.start_timestamp = start_timestamp + res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() + res.timing_info.timer = timer + self._active_config = self.config + + return res From 97aeb31fd2c6fb2fc30349a19644d719af92b4dc Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 30 Jul 2025 06:37:17 -0600 Subject: [PATCH 029/104] working on a model observer --- pyomo/contrib/observer/model_observer.py | 392 ++++++++++-------- pyomo/contrib/observer/tests/__init__.py | 0 .../observer/tests/test_change_detector.py | 91 ++++ 3 files changed, 306 insertions(+), 177 deletions(-) create mode 100644 pyomo/contrib/observer/tests/__init__.py create mode 100644 pyomo/contrib/observer/tests/test_change_detector.py diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index b644676f3d2..b6768f2ac6b 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -11,7 +11,7 @@ import abc import datetime -from typing import List, Sequence +from typing import List, Sequence, Optional from pyomo.common.config import ConfigDict, ConfigValue from pyomo.core.base.constraint import ConstraintData, Constraint @@ -27,6 +27,27 @@ from pyomo.common.numeric_types import native_numeric_types +""" +The ModelChangeDetector is meant to be used to automatically identify changes +in a Pyomo model or block. Here is a list of changes that will be detected. +Note that inactive components (e.g., constraints) are treated as "removed". + - new constraints that have been added to the model + - constraints that have been removed from the model + - new variables that have been detected in new or modified constraints/objectives + - old variables that are no longer used in any constraints/objectives + - new parameters that have been detected in new or modified constraints/objectives + - old parameters that are no longer used in any constraints/objectives + - new objectives that have been added to the model + - objectives that have been removed from the model + - modified constraint expressions (relies on expressions being immutable) + - modified objective expressions (relies on expressions being immutable) + - modified objective sense + - changes to variable bounds, domains, and "fixed" flags + - changes to named expressions (relies on expressions being immutable) + - changes to parameter values and fixed variable values +""" + + class AutoUpdateConfig(ConfigDict): """ Control which parts of the model are automatically checked and/or updated upon re-solve @@ -62,30 +83,6 @@ def __init__( added to/removed from the model.""", ), ) - self.check_for_new_or_removed_vars: bool = self.declare( - 'check_for_new_or_removed_vars', - ConfigValue( - domain=bool, - default=True, - description=""" - If False, new/old variables will not be automatically detected on subsequent - solves. Use False only when manually updating the solver with opt.add_variables() and - opt.remove_variables() or when you are certain variables are not being added to / - removed from the model.""", - ), - ) - self.check_for_new_or_removed_params: bool = self.declare( - 'check_for_new_or_removed_params', - ConfigValue( - domain=bool, - default=True, - description=""" - If False, new/old parameters will not be automatically detected on subsequent - solves. Use False only when manually updating the solver with opt.add_parameters() and - opt.remove_parameters() or when you are certain parameters are not being added to / - removed from the model.""", - ), - ) self.check_for_new_objective: bool = self.declare( 'check_for_new_objective', ConfigValue( @@ -118,19 +115,23 @@ def __init__( description=""" If False, changes to existing variables will not be automatically detected on subsequent solves. This includes changes to the lb, ub, domain, and fixed - attributes of variables. Use False only when manually updating the solver with - opt.update_variables() or when you are certain variables are not being modified.""", + attributes of variables. Use False only when manually updating the observer with + opt.update_variables() or when you are certain variables are not being modified. + Note that changes to values of fixed variables is handled by + update_parameters_and_fixed_vars.""", ), ) - self.update_parameters: bool = self.declare( + self.update_parameters_and_fixed_vars: bool = self.declare( 'update_parameters', ConfigValue( domain=bool, default=True, description=""" - If False, changes to parameter values will not be automatically detected on - subsequent solves. Use False only when manually updating the solver with - opt.update_parameters() or when you are certain parameters are not being modified.""", + If False, changes to parameter values and fixed variable values will + not be automatically detected on subsequent solves. Use False only + when manually updating the observer with + opt.update_parameters_and_fixed_variables() or when you are certain + parameters are not being modified.""", ), ) self.update_named_expressions: bool = self.declare( @@ -199,6 +200,18 @@ def remove_variables(self, variables: List[VarData]): def remove_parameters(self, params: List[ParamData]): pass + @abc.abstractmethod + def update_variables(self, variables: List[VarData]): + pass + + @abc.abstractmethod + def update_parameters_and_fixed_variables( + self, + params: List[ParamData], + variables: List[VarData], + ): + pass + class ModelChangeDetector: def __init__( @@ -224,7 +237,8 @@ def __init__( """ self._observers: List[Observer] = list(observers) self._model = None - self._active_constraints = {} # maps constraint to (lower, body, upper) + self._active_constraints = {} # maps constraint to expression + self._active_sos = {} self._vars = {} # maps var id to (var, lb, ub, fixed, domain, value) self._params = {} # maps param id to param self._objective = None @@ -254,9 +268,7 @@ def set_instance(self, model): self.__init__(observers=self._observers, treat_fixed_vars_as_params=self._treat_fixed_vars_as_params) self.config = saved_config self._model = model - self.add_block(model) - if self._objective is None: - self.set_objective(None) + self._add_block(model) def _add_variables(self, variables: List[VarData]): for v in variables: @@ -321,7 +333,7 @@ def _check_to_remove_params(self, params: List[ParamData]): def _add_constraints(self, cons: List[ConstraintData]): all_fixed_vars = {} for con in cons: - if con in self._named_expressions: + if con in self._active_constraints: raise ValueError(f'Constraint {con.name} has already been added') self._active_constraints[con] = con.expr tmp = collect_vars_and_named_exprs(con.expr) @@ -351,10 +363,11 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): for con in cons: if con in self._vars_referenced_by_con: raise ValueError(f'Constraint {con.name} has already been added') - self._active_constraints[con] = tuple() + sos_items = list(con.get_items()) + self._active_sos[con] = ([i[0] for i in sos_items], [i[1] for i in sos_items]) variables = [] params = [] - for v, p in con.get_items(): + for v, p in sos_items: variables.append(v) if type(p) in native_numeric_types: continue @@ -384,20 +397,25 @@ def _set_objective(self, obj: ObjectiveData): for v in self._vars_referenced_by_obj: self._referenced_variables[id(v)][2] = None self._check_to_remove_vars(self._vars_referenced_by_obj) + self._check_to_remove_params(self._params_referenced_by_obj) self._external_functions.pop(self._objective, None) if obj is not None: self._objective = obj self._objective_expr = obj.expr self._objective_sense = obj.sense tmp = collect_vars_and_named_exprs(obj.expr) - named_exprs, variables, fixed_vars, external_functions = tmp + named_exprs, variables, fixed_vars, parameters, external_functions = tmp self._check_for_new_vars(variables) + self._check_for_new_params(parameters) self._obj_named_expressions = [(i, i.expr) for i in named_exprs] if len(external_functions) > 0: self._external_functions[obj] = external_functions self._vars_referenced_by_obj = variables + self._params_referenced_by_obj = parameters for v in variables: self._referenced_variables[id(v)][2] = obj + for p in parameters: + self._referenced_params[id(p)][2] = obj if not self._treat_fixed_vars_as_params: for v in fixed_vars: v.unfix() @@ -407,6 +425,7 @@ def _set_objective(self, obj: ObjectiveData): v.fix() else: self._vars_referenced_by_obj = [] + self._params_referenced_by_obj = [] self._objective = None self._objective_expr = None self._objective_sense = None @@ -414,13 +433,7 @@ def _set_objective(self, obj: ObjectiveData): for obs in self._observers: obs.set_objective(obj) - def add_block(self, block): - param_dict = {} - for p in block.component_objects(Param, descend_into=True): - if p.mutable: - for _p in p.values(): - param_dict[id(_p)] = _p - self._add_parameters(list(param_dict.values())) + def _add_block(self, block): self._add_constraints( list( block.component_data_objects(Constraint, descend_into=True, active=True) @@ -447,11 +460,15 @@ def _remove_constraints(self, cons: List[ConstraintData]): ) for v in self._vars_referenced_by_con[con]: self._referenced_variables[id(v)][0].pop(con) + for p in self._params_referenced_by_con[con]: + self._referenced_params[id(p)][0].pop(con) self._check_to_remove_vars(self._vars_referenced_by_con[con]) + self._check_to_remove_params(self._params_referenced_by_con[con]) del self._active_constraints[con] del self._named_expressions[con] self._external_functions.pop(con, None) del self._vars_referenced_by_con[con] + del self._params_referenced_by_con[con] def _remove_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: @@ -463,10 +480,14 @@ def _remove_sos_constraints(self, cons: List[SOSConstraintData]): ) for v in self._vars_referenced_by_con[con]: self._referenced_variables[id(v)][1].pop(con) + for p in self._params_referenced_by_con[con]: + self._referenced_params[id(p)][1].pop(con) self._check_to_remove_vars(self._vars_referenced_by_con[con]) - del self._active_constraints[con] + self._check_to_remove_params(self._params_referenced_by_con[con]) + del self._active_sos[con] del self._named_expressions[con] del self._vars_referenced_by_con[con] + del self._params_referenced_by_con[con] def _remove_variables(self, variables: List[VarData]): for obs in self._observers: @@ -489,39 +510,20 @@ def _remove_parameters(self, params: List[ParamData]): for obs in self._observers: obs.remove_parameters(params) for p in params: - del self._params[id(p)] - - def _remove_block(self, block): - self._remove_constraints( - list( - block.component_data_objects( - ctype=Constraint, descend_into=True, active=True + p_id = id(p) + if p_id not in self._referenced_params: + raise ValueError( + f'Cannot remove parameter {p.name} - it has not been added' ) - ) - ) - self._remove_sos_constraints( - list( - block.component_data_objects( - ctype=SOSConstraint, descend_into=True, active=True + cons_using, sos_using, obj_using = self._referenced_params[p_id] + if cons_using or sos_using or (obj_using is not None): + raise ValueError( + f'Cannot remove parameter {p.name} - it is still being used by constraints or the objective' ) - ) - ) - self._remove_parameters( - list( - dict( - (id(p), p) - for p in block.component_data_objects( - ctype=Param, descend_into=True - ) - ).values() - ) - ) + del self._referenced_params[p_id] + del self._params[p_id] - @abc.abstractmethod def _update_variables(self, variables: List[VarData]): - pass - - def update_variables(self, variables: List[VarData]): for v in variables: self._vars[id(v)] = ( v, @@ -531,109 +533,138 @@ def update_variables(self, variables: List[VarData]): v.domain.get_interval(), v.value, ) - self._update_variables(variables) + for obs in self._observers: + obs.update_variables(variables) - @abc.abstractmethod - def update_parameters(self): - pass + def _update_parameters_and_fixed_variables(self, params, variables): + for p in params: + self._params[id(p)] = (p, p.value) + for v in variables: + self._vars[id(v)][5] = v.value + for obs in self._observers: + obs.update_parameters_and_fixed_variables(params, variables) - def update(self, timer: HierarchicalTimer = None): - if timer is None: - timer = HierarchicalTimer() - config = self._active_config.auto_updates - new_vars = [] - old_vars = [] - new_params = [] - old_params = [] + def _check_for_new_or_removed_sos(self): + new_sos = [] + old_sos = [] + current_sos_dict = { + c: None + for c in self._model.component_data_objects( + SOSConstraint, descend_into=True, active=True + ) + } + for c in current_sos_dict.keys(): + if c not in self._active_sos: + new_sos.append(c) + for c in self._active_sos: + if c not in current_sos_dict: + old_sos.append(c) + return new_sos, old_sos + + def _check_for_new_or_removed_constraints(self): new_cons = [] old_cons = [] - old_sos = [] - new_sos = [] - current_cons_dict = {} - current_sos_dict = {} - timer.start('vars') - if config.update_vars: - start_vars = {v_id: v_tuple[0] for v_id, v_tuple in self._vars.items()} - timer.stop('vars') - timer.start('params') - if config.check_for_new_or_removed_params: - current_params_dict = {} - for p in self._model.component_objects(Param, descend_into=True): - if p.mutable: - for _p in p.values(): - current_params_dict[id(_p)] = _p - for p_id, p in current_params_dict.items(): - if p_id not in self._params: - new_params.append(p) - for p_id, p in self._params.items(): - if p_id not in current_params_dict: - old_params.append(p) - timer.stop('params') - timer.start('cons') - if config.check_for_new_or_removed_constraints or config.update_constraints: - current_cons_dict = { - c: None - for c in self._model.component_data_objects( - Constraint, descend_into=True, active=True - ) - } - current_sos_dict = { - c: None - for c in self._model.component_data_objects( - SOSConstraint, descend_into=True, active=True - ) - } - for c in current_cons_dict.keys(): - if c not in self._vars_referenced_by_con: - new_cons.append(c) - for c in current_sos_dict.keys(): - if c not in self._vars_referenced_by_con: - new_sos.append(c) - for c in self._vars_referenced_by_con: - if c not in current_cons_dict and c not in current_sos_dict: - if (c.ctype is Constraint) or ( - c.ctype is None and isinstance(c, ConstraintData) - ): - old_cons.append(c) - else: - assert (c.ctype is SOSConstraint) or ( - c.ctype is None and isinstance(c, SOSConstraintData) - ) - old_sos.append(c) - self.remove_constraints(old_cons) - self.remove_sos_constraints(old_sos) - timer.stop('cons') - timer.start('params') - self.remove_parameters(old_params) + current_cons_dict = { + c: None + for c in self._model.component_data_objects( + Constraint, descend_into=True, active=True + ) + } + for c in current_cons_dict.keys(): + if c not in self._active_constraints: + new_cons.append(c) + for c in self._active_constraints: + if c not in current_cons_dict: + old_cons.append(c) + return new_cons, old_cons + + def _check_for_modified_sos(self): + sos_to_update = [] + for c, (old_vlist, old_plist) in self._active_sos.items(): + sos_items = list(c.get_items()) + new_vlist = [i[0] for i in sos_items] + new_plist = [i[1] for i in sos_items] + if len(old_vlist) != len(new_vlist): + sos_to_update.append(c) + elif len(old_plist) != len(new_plist): + sos_to_update.append(c) + else: + needs_update = False + for v1, v2 in zip(old_vlist, new_vlist): + if v1 is not v2: + needs_update = True + break + for p1, p2 in zip(old_plist, new_plist): + if p1 is not p2: + needs_update = True + if needs_update: + break + if needs_update: + sos_to_update.append(c) + return sos_to_update + + def _check_for_modified_constraints(self): + cons_to_update = [] + for c, expr in self._active_constraints.items(): + if c.expr is not expr: + cons_to_update.append(c) + return cons_to_update + + def _check_for_var_changes(self): + vars_to_update = [] + cons_to_update = {} + update_obj = False + for vid, (v, _lb, _ub, _fixed, _domain_interval, _value) in self._vars.items(): + if v.fixed != _fixed: + vars_to_update.append(v) + if self._treat_fixed_vars_as_params: + for c in self._referenced_variables[vid][0]: + cons_to_update[c] = None + + elif v._lb is not _lb: + vars_to_update.append(v) + elif v._ub is not _ub: + vars_to_update.append(v) + + + def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): + if timer is None: + timer = HierarchicalTimer() + config: AutoUpdateConfig = self.config(value=kwds, preserve_implicit=True) + + added_cons = set() + added_sos = set() + + if config.check_for_new_or_removed_constraints: + timer.start('sos') + new_sos, old_sos = self._check_for_new_or_removed_sos() + self._add_sos_constraints(new_sos) + self._remove_sos_constraints(old_sos) + added_sos.update(new_sos) + timer.stop('cons') + timer.start('cons') + new_cons, old_cons = self._check_for_new_or_removed_constraints() + self._add_constraints(new_cons) + self._remove_constraints(old_cons) + added_cons.update(new_cons) + timer.stop('cons') - # sticking this between removal and addition - # is important so that we don't do unnecessary work - if config.update_parameters: - self.update_parameters() + if config.update_constraints: + timer.start('cons') + cons_to_update = self._check_for_modified_constraints() + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) + added_cons.update(cons_to_update) + timer.stop('cons') + timer.start('sos') + sos_to_update = self._check_for_modified_sos() + self._remove_sos_constraints(sos_to_update) + self._add_sos_constraints(sos_to_update) + added_sos.update(sos_to_update) + timer.stop('sos') - self.add_parameters(new_params) - timer.stop('params') - timer.start('vars') - self.add_variables(new_vars) - timer.stop('vars') - timer.start('cons') - self.add_constraints(new_cons) - self.add_sos_constraints(new_sos) - new_cons_set = set(new_cons) - new_sos_set = set(new_sos) - cons_to_remove_and_add = {} need_to_set_objective = False - if config.update_constraints: - for c in current_cons_dict.keys(): - if c not in new_cons_set and c.expr is not self._active_constraints[c]: - cons_to_remove_and_add[c] = None - sos_to_update = [] - for c in current_sos_dict.keys(): - if c not in new_sos_set: - sos_to_update.append(c) - self.remove_sos_constraints(sos_to_update) - self.add_sos_constraints(sos_to_update) - timer.stop('cons') + timer.start('vars') if config.update_vars: end_vars = {v_id: v_tuple[0] for v_id, v_tuple in self._vars.items()} @@ -696,11 +727,18 @@ def update(self, timer: HierarchicalTimer = None): self.set_objective(pyomo_obj) timer.stop('objective') - # this has to be done after the objective and constraints in case the - # old objective/constraints use old variables - timer.start('vars') - self.remove_variables(old_vars) - timer.stop('vars') + if config.update_parameters: + timer.start('params') + modified_params = [] + for pid, (p, old_val) in self._params.items(): + if p.value != old_val: + modified_params.append(p) + modified_vars = [] + for vid, (v, _lb, _ub, _fixed, _domain_interval, _val) in self._vars.items(): + if _fixed and _val != v.value: + modified_vars.append(v) + self._update_parameters_and_fixed_variables(modified_params, modified_vars) + timer.stop('params') class PersistentSolverMixin: diff --git a/pyomo/contrib/observer/tests/__init__.py b/pyomo/contrib/observer/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py new file mode 100644 index 00000000000..46faacae9bb --- /dev/null +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -0,0 +1,91 @@ +from pyomo.core.base.constraint import ConstraintData +from pyomo.core.base.objective import ObjectiveData +from pyomo.core.base.param import ParamData +from pyomo.core.base.sos import SOSConstraintData +from pyomo.core.base.var import VarData +import pyomo.environ as pe +from pyomo.common import unittest +from typing import List +from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig +from pyomo.common.collections import ComponentMap +import logging + + +logger = logging.getLogger(__name__) + + +class ObserverChecker(Observer): + def __init__(self): + super().__init__() + self.counts = ComponentMap() + """ + counts is be a mapping from component (e.g., variable) to another + mapping from string ('add', 'remove', 'update', or 'value') to an int that + indicates the number of times the corresponding method has been called + """ + + def check(self, expected): + unittest.assertStructuredAlmostEqual( + first=expected, + second=self.counts, + places=7, + ) + + def _process(self, comps, key): + for c in comps: + if c not in self.counts: + self.counts[c] = {'add': 0, 'remove': 0, 'update': 0, 'value': 0} + self.counts[c][key] += 1 + + def add_variables(self, variables: List[VarData]): + self._process(variables, 'add') + + def add_parameters(self, params: List[ParamData]): + self._process(params, 'add') + + def add_constraints(self, cons: List[ConstraintData]): + self._process(cons, 'add') + + def add_sos_constraints(self, cons: List[SOSConstraintData]): + self._process(cons, 'add') + + def set_objective(self, obj: ObjectiveData): + self._process([obj], 'add') + + def remove_constraints(self, cons: List[ConstraintData]): + self._process(cons, 'remove') + + def remove_sos_constraints(self, cons: List[SOSConstraintData]): + self._process(cons, 'remove') + + def remove_variables(self, variables: List[VarData]): + self._process(variables, 'remove') + + def remove_parameters(self, params: List[ParamData]): + self._process(params, 'remove') + + def update_variables(self, variables: List[VarData]): + self._process(variables, 'update') + + def update_parameters_and_fixed_variables(self, params: List[ParamData], variables: List[VarData]): + self._process(params, 'value') + self._process(variables, 'value') + + +class TestChangeDetector(unittest.TestCase): + def test_basics(self): + m = pe.ConcreteModel() + m.x = pe.Var() + m.y = pe.Var() + + obs = ObserverChecker() + detector = ModelChangeDetector([obs]) + + detector.set_instance(m) + + expected = ComponentMap() + + obs.check(expected) + + def test_vars_and_params_elsewhere(self): + pass \ No newline at end of file From a69d5e69ea9729a4c1ae0396719fd173065d55a9 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 6 Aug 2025 06:20:19 -0600 Subject: [PATCH 030/104] working on model change detector --- pyomo/contrib/observer/component_collector.py | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 pyomo/contrib/observer/component_collector.py diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py new file mode 100644 index 00000000000..149c4038bc6 --- /dev/null +++ b/pyomo/contrib/observer/component_collector.py @@ -0,0 +1,77 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +from pyomo.core.expr.visitor import StreamBasedExpressionVisitor +from pyomo.core.expr.numeric_expr import ExternalFunctionExpression, NPV_ExternalFunctionExpression +from pyomo.core.base.var import VarData, ScalarVar +from pyomo.core.base.param import ParamData, ScalarParam +from pyomo.core.base.expression import ExpressionData, ScalarExpression + + +def handle_var(node, collector): + collector.variables[id(node)] = node + return None + + +def handle_param(node, collector): + collector.params[id(node)] = node + return None + + +def handle_named_expression(node, collector): + collector.named_expressions[id(node)] = node + return None + + +def handle_external_function(node, collector): + collector.external_functions[id(node)] = node + return None + + +collector_handlers = { + VarData: handle_var, + ScalarVar: handle_var, + ParamData: handle_param, + ScalarParam: handle_param, + ExpressionData: handle_named_expression, + ScalarExpression: handle_named_expression, + ExternalFunctionExpression: handle_external_function, + NPV_ExternalFunctionExpression: handle_external_function, +} + + +class _ComponentFromExprCollector(StreamBasedExpressionVisitor): + def __init__(self): + self.named_expressions = {} + self.variables = {} + self.params = {} + self.external_functions = {} + + def exitNode(self, node, data): + nt = type(node) + if nt in collector_handlers: + return collector_handlers[nt](node, self) + else: + return None + + +_visitor = _ComponentFromExprCollector() + + +def collect_components_from_expr(expr): + _visitor.__init__() + _visitor.walk_expression(expr) + return ( + list(_visitor.named_expressions.values()), + list(_visitor.variables.values()), + list(_visitor.params.values()), + list(_visitor.external_functions.values()), + ) From 9763e9a2f594e74a26bd5003682b963812d0acb3 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 7 Aug 2025 08:26:14 -0600 Subject: [PATCH 031/104] observer --- pyomo/contrib/observer/component_collector.py | 3 +- pyomo/contrib/observer/model_observer.py | 313 ++++++++---------- .../observer/tests/test_change_detector.py | 118 ++++++- 3 files changed, 250 insertions(+), 184 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index 149c4038bc6..5cbbdaf31bd 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -49,11 +49,12 @@ def handle_external_function(node, collector): class _ComponentFromExprCollector(StreamBasedExpressionVisitor): - def __init__(self): + def __init__(self, **kwds): self.named_expressions = {} self.variables = {} self.params = {} self.external_functions = {} + super().__init__(**kwds) def exitNode(self, node, data): nt = type(node) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index b6768f2ac6b..422eb1da574 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -23,7 +23,8 @@ from pyomo.common.collections import ComponentMap from pyomo.common.timing import HierarchicalTimer from pyomo.contrib.solver.common.results import Results -from pyomo.contrib.solver.common.util import collect_vars_and_named_exprs, get_objective +from pyomo.contrib.solver.common.util import get_objective +from .component_collector import collect_components_from_expr from pyomo.common.numeric_types import native_numeric_types @@ -121,7 +122,7 @@ def __init__( update_parameters_and_fixed_vars.""", ), ) - self.update_parameters_and_fixed_vars: bool = self.declare( + self.update_parameters: bool = self.declare( 'update_parameters', ConfigValue( domain=bool, @@ -181,7 +182,7 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): pass @abc.abstractmethod - def set_objective(self, obj: ObjectiveData): + def set_objective(self, obj: Optional[ObjectiveData]): pass @abc.abstractmethod @@ -205,18 +206,13 @@ def update_variables(self, variables: List[VarData]): pass @abc.abstractmethod - def update_parameters_and_fixed_variables( - self, - params: List[ParamData], - variables: List[VarData], - ): + def update_parameters(self, params: List[ParamData]): pass class ModelChangeDetector: def __init__( self, observers: Sequence[Observer], - treat_fixed_vars_as_params=True, **kwds, ): """ @@ -224,16 +220,6 @@ def __init__( ---------- observers: Sequence[Observer] The objects to notify when changes are made to the model - treat_fixed_vars_as_params: bool - This is an advanced option that should only be used in special circumstances. - With the default setting of True, fixed variables will be treated like parameters. - This means that z == x*y will be linear if x or y is fixed and the constraint - can be written to an LP file. If the value of the fixed variable gets changed, we have - to completely reprocess all constraints using that variable. If - treat_fixed_vars_as_params is False, then constraints will be processed as if fixed - variables are not fixed, and the solver will be told the variable is fixed. This means - z == x*y could not be written to an LP file even if x and/or y is fixed. However, - updating the values of fixed variables is much faster this way. """ self._observers: List[Observer] = list(observers) self._model = None @@ -260,12 +246,11 @@ def __init__( self._params_referenced_by_con = {} self._params_referenced_by_obj = [] self._expr_types = None - self._treat_fixed_vars_as_params = treat_fixed_vars_as_params self.config: AutoUpdateConfig = AutoUpdateConfig()(value=kwds, preserve_implicit=True) def set_instance(self, model): saved_config = self.config - self.__init__(observers=self._observers, treat_fixed_vars_as_params=self._treat_fixed_vars_as_params) + self.__init__(observers=self._observers) self.config = saved_config self._model = model self._add_block(model) @@ -331,37 +316,38 @@ def _check_to_remove_params(self, params: List[ParamData]): self._remove_parameters(list(params_to_remove.values())) def _add_constraints(self, cons: List[ConstraintData]): - all_fixed_vars = {} + vars_to_check = [] + params_to_check = [] for con in cons: if con in self._active_constraints: raise ValueError(f'Constraint {con.name} has already been added') self._active_constraints[con] = con.expr - tmp = collect_vars_and_named_exprs(con.expr) - named_exprs, variables, fixed_vars, parameters, external_functions = tmp - self._check_for_new_vars(variables) - self._check_for_new_params(parameters) + tmp = collect_components_from_expr(con.expr) + named_exprs, variables, parameters, external_functions = tmp + vars_to_check.extend(variables) + params_to_check.extend(parameters) self._named_expressions[con] = [(e, e.expr) for e in named_exprs] if len(external_functions) > 0: self._external_functions[con] = external_functions self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = parameters + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) + for con in cons: + variables = self._vars_referenced_by_con[con] + parameters = self._params_referenced_by_con[con] for v in variables: self._referenced_variables[id(v)][0][con] = None for p in parameters: self._referenced_params[id(p)][0][con] = None - if not self._treat_fixed_vars_as_params: - for v in fixed_vars: - v.unfix() - all_fixed_vars[id(v)] = v for obs in self._observers: obs.add_constraints(cons) - for v in all_fixed_vars.values(): - v.fix() def _add_sos_constraints(self, cons: List[SOSConstraintData]): - all_fixed_vars = {} + vars_to_check = [] + params_to_check = [] for con in cons: - if con in self._vars_referenced_by_con: + if con in self._active_sos: raise ValueError(f'Constraint {con.name} has already been added') sos_items = list(con.get_items()) self._active_sos[con] = ([i[0] for i in sos_items], [i[1] for i in sos_items]) @@ -373,8 +359,8 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): continue if p.is_parameter_type(): params.append(p) - self._check_for_new_vars(variables) - self._check_for_new_params(params) + vars_to_check.extend(variables) + params_to_check.extend(params) self._named_expressions[con] = [] self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params @@ -382,29 +368,28 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): self._referenced_variables[id(v)][1][con] = None for p in params: self._referenced_params[id(p)][1][con] = None - if not self._treat_fixed_vars_as_params: - for v in variables: - if v.is_fixed(): - v.unfix() - all_fixed_vars[id(v)] = v + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) for obs in self._observers: obs.add_sos_constraints(cons) - for v in all_fixed_vars.values(): - v.fix() - def _set_objective(self, obj: ObjectiveData): + def _set_objective(self, obj: Optional[ObjectiveData]): + vars_to_remove_check = [] + params_to_remove_check = [] if self._objective is not None: for v in self._vars_referenced_by_obj: self._referenced_variables[id(v)][2] = None - self._check_to_remove_vars(self._vars_referenced_by_obj) - self._check_to_remove_params(self._params_referenced_by_obj) + for p in self._params_referenced_by_obj: + self._referenced_params[id(p)][2] = None + vars_to_remove_check.extend(self._vars_referenced_by_obj) + params_to_remove_check.extend(self._params_referenced_by_obj) self._external_functions.pop(self._objective, None) if obj is not None: self._objective = obj self._objective_expr = obj.expr self._objective_sense = obj.sense - tmp = collect_vars_and_named_exprs(obj.expr) - named_exprs, variables, fixed_vars, parameters, external_functions = tmp + tmp = collect_components_from_expr(obj.expr) + named_exprs, variables, parameters, external_functions = tmp self._check_for_new_vars(variables) self._check_for_new_params(parameters) self._obj_named_expressions = [(i, i.expr) for i in named_exprs] @@ -416,13 +401,6 @@ def _set_objective(self, obj: ObjectiveData): self._referenced_variables[id(v)][2] = obj for p in parameters: self._referenced_params[id(p)][2] = obj - if not self._treat_fixed_vars_as_params: - for v in fixed_vars: - v.unfix() - for obs in self._observers: - obs.set_objective(obj) - for v in fixed_vars: - v.fix() else: self._vars_referenced_by_obj = [] self._params_referenced_by_obj = [] @@ -430,8 +408,10 @@ def _set_objective(self, obj: ObjectiveData): self._objective_expr = None self._objective_sense = None self._obj_named_expressions = [] - for obs in self._observers: - obs.set_objective(obj) + for obs in self._observers: + obs.set_objective(obj) + self._check_to_remove_vars(vars_to_remove_check) + self._check_to_remove_params(params_to_remove_check) def _add_block(self, block): self._add_constraints( @@ -447,14 +427,15 @@ def _add_block(self, block): ) ) obj = get_objective(block) - if obj is not None: - self._set_objective(obj) + self._set_objective(obj) def _remove_constraints(self, cons: List[ConstraintData]): for obs in self._observers: obs.remove_constraints(cons) + vars_to_check = [] + params_to_check = [] for con in cons: - if con not in self._named_expressions: + if con not in self._active_constraints: raise ValueError( f'Cannot remove constraint {con.name} - it was not added' ) @@ -462,19 +443,23 @@ def _remove_constraints(self, cons: List[ConstraintData]): self._referenced_variables[id(v)][0].pop(con) for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][0].pop(con) - self._check_to_remove_vars(self._vars_referenced_by_con[con]) - self._check_to_remove_params(self._params_referenced_by_con[con]) + vars_to_check.extend(self._vars_referenced_by_con[con]) + params_to_check.extend(self._params_referenced_by_con[con]) del self._active_constraints[con] del self._named_expressions[con] self._external_functions.pop(con, None) del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] + self._check_to_remove_vars(vars_to_check) + self._check_to_remove_params(params_to_check) def _remove_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.remove_sos_constraints(cons) + vars_to_check = [] + params_to_check = [] for con in cons: - if con not in self._vars_referenced_by_con: + if con not in self._active_sos: raise ValueError( f'Cannot remove constraint {con.name} - it was not added' ) @@ -482,12 +467,14 @@ def _remove_sos_constraints(self, cons: List[SOSConstraintData]): self._referenced_variables[id(v)][1].pop(con) for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][1].pop(con) - self._check_to_remove_vars(self._vars_referenced_by_con[con]) - self._check_to_remove_params(self._params_referenced_by_con[con]) + vars_to_check.extend(self._vars_referenced_by_con[con]) + params_to_check.extend(self._params_referenced_by_con[con]) del self._active_sos[con] del self._named_expressions[con] del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] + self._check_to_remove_vars(vars_to_check) + self._check_to_remove_params(params_to_check) def _remove_variables(self, variables: List[VarData]): for obs in self._observers: @@ -536,13 +523,11 @@ def _update_variables(self, variables: List[VarData]): for obs in self._observers: obs.update_variables(variables) - def _update_parameters_and_fixed_variables(self, params, variables): + def _update_parameters(self, params): for p in params: self._params[id(p)] = (p, p.value) - for v in variables: - self._vars[id(v)][5] = v.value for obs in self._observers: - obs.update_parameters_and_fixed_variables(params, variables) + obs.update_parameters(params) def _check_for_new_or_removed_sos(self): new_sos = [] @@ -617,15 +602,60 @@ def _check_for_var_changes(self): for vid, (v, _lb, _ub, _fixed, _domain_interval, _value) in self._vars.items(): if v.fixed != _fixed: vars_to_update.append(v) - if self._treat_fixed_vars_as_params: - for c in self._referenced_variables[vid][0]: - cons_to_update[c] = None - + for c in self._referenced_variables[vid][0]: + cons_to_update[c] = None + if self._referenced_variables[vid][2] is not None: + update_obj = True elif v._lb is not _lb: vars_to_update.append(v) elif v._ub is not _ub: vars_to_update.append(v) - + elif _domain_interval != v.domain.get_interval(): + vars_to_update.append(v) + elif v.value != _value: + vars_to_update.append(v) + cons_to_update = list(cons_to_update.keys()) + return vars_to_update, cons_to_update, update_obj + + def _check_for_param_changes(self): + params_to_update = [] + for pid, (p, val) in self._params.items(): + if p.value != val: + params_to_update.append(p) + return params_to_update + + def _check_for_named_expression_changes(self): + cons_to_update = [] + for con, ne_list in self._named_expressions.items(): + for named_expr, old_expr in ne_list: + if named_expr.expr is not old_expr: + cons_to_update.append(con) + break + update_obj = False + ne_list = self._obj_named_expressions + for named_expr, old_expr in ne_list: + if named_expr.expr is not old_expr: + update_obj = True + break + return cons_to_update, update_obj + + def _check_for_new_objective(self): + update_obj = False + new_obj = get_objective(self._model) + if new_obj is not self._objective: + update_obj = True + return new_obj, update_obj + + def _check_for_objective_changes(self): + update_obj = False + if self._objective is None: + return update_obj + if self._objective.expr is not self._objective_expr: + update_obj = True + elif self._objective.sense != self._objective_sense: + # we can definitely do something faster here than resetting the whole objective + update_obj = True + return update_obj def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if timer is None: @@ -641,7 +671,7 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): self._add_sos_constraints(new_sos) self._remove_sos_constraints(old_sos) added_sos.update(new_sos) - timer.stop('cons') + timer.stop('sos') timer.start('cons') new_cons, old_cons = self._check_for_new_or_removed_constraints() self._add_constraints(new_cons) @@ -665,115 +695,46 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): need_to_set_objective = False - timer.start('vars') if config.update_vars: - end_vars = {v_id: v_tuple[0] for v_id, v_tuple in self._vars.items()} - vars_to_check = [v for v_id, v in end_vars.items() if v_id in start_vars] - if config.update_vars: - vars_to_update = [] - for v in vars_to_check: - _v, lb, ub, fixed, domain_interval, value = self._vars[id(v)] - if (fixed != v.fixed) or (fixed and (value != v.value)): - vars_to_update.append(v) - if self._treat_fixed_vars_as_params: - for c in self._referenced_variables[id(v)][0]: - cons_to_remove_and_add[c] = None - if self._referenced_variables[id(v)][2] is not None: - need_to_set_objective = True - elif lb is not v._lb: - vars_to_update.append(v) - elif ub is not v._ub: - vars_to_update.append(v) - elif domain_interval != v.domain.get_interval(): - vars_to_update.append(v) - self.update_variables(vars_to_update) - timer.stop('vars') - timer.start('cons') - cons_to_remove_and_add = list(cons_to_remove_and_add.keys()) - self.remove_constraints(cons_to_remove_and_add) - self.add_constraints(cons_to_remove_and_add) - timer.stop('cons') - timer.start('named expressions') + timer.start('vars') + vars_to_update, cons_to_update, update_obj = self._check_for_var_changes() + self._update_variables(vars_to_update) + cons_to_update = [i for i in cons_to_update if i not in added_cons] + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) + added_cons.update(cons_to_update) + if update_obj: + need_to_set_objective = True + timer.stop('vars') + if config.update_named_expressions: - cons_to_update = [] - for c, expr_list in self._named_expressions.items(): - if c in new_cons_set: - continue - for named_expr, old_expr in expr_list: - if named_expr.expr is not old_expr: - cons_to_update.append(c) - break - self.remove_constraints(cons_to_update) - self.add_constraints(cons_to_update) - for named_expr, old_expr in self._obj_named_expressions: - if named_expr.expr is not old_expr: - need_to_set_objective = True - break - timer.stop('named expressions') - timer.start('objective') - if self._active_config.auto_updates.check_for_new_objective: - pyomo_obj = get_objective(self._model) - if pyomo_obj is not self._objective: + timer.start('named expressions') + cons_to_update, update_obj = self._check_for_named_expression_changes() + cons_to_update = [i for i in cons_to_update if i not in added_cons] + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) + added_cons.update(cons_to_update) + if update_obj: need_to_set_objective = True - else: - pyomo_obj = self._objective - if self._active_config.auto_updates.update_objective: - if pyomo_obj is not None and pyomo_obj.expr is not self._objective_expr: + timer.stop('named expressions') + + timer.start('objective') + new_obj = self._objective + if config.check_for_new_objective: + new_obj, update_obj = self._check_for_new_objective() + if update_obj: need_to_set_objective = True - elif pyomo_obj is not None and pyomo_obj.sense is not self._objective_sense: - # we can definitely do something faster here than resetting the whole objective + if config.update_objective: + update_obj = self._check_for_objective_changes() + if update_obj: need_to_set_objective = True + if need_to_set_objective: - self.set_objective(pyomo_obj) + self._set_objective(new_obj) timer.stop('objective') if config.update_parameters: timer.start('params') - modified_params = [] - for pid, (p, old_val) in self._params.items(): - if p.value != old_val: - modified_params.append(p) - modified_vars = [] - for vid, (v, _lb, _ub, _fixed, _domain_interval, _val) in self._vars.items(): - if _fixed and _val != v.value: - modified_vars.append(v) - self._update_parameters_and_fixed_variables(modified_params, modified_vars) + params_to_update = self._check_for_param_changes() + self._update_parameters(params_to_update) timer.stop('params') - - -class PersistentSolverMixin: - """ - The `solve` method in Gurobi and Highs is exactly the same, so this Mixin - minimizes the duplicate code - """ - - def solve(self, model, **kwds) -> Results: - start_timestamp = datetime.datetime.now(datetime.timezone.utc) - self._active_config = config = self.config(value=kwds, preserve_implicit=True) - StaleFlagManager.mark_all_as_stale() - - if self._last_results_object is not None: - self._last_results_object.solution_loader.invalidate() - if config.timer is None: - config.timer = HierarchicalTimer() - timer = config.timer - - if model is not self._model: - timer.start('set_instance') - self.set_instance(model) - timer.stop('set_instance') - else: - timer.start('update') - self.update(timer=timer) - timer.stop('update') - - res = self._solve() - self._last_results_object = res - - end_timestamp = datetime.datetime.now(datetime.timezone.utc) - res.timing_info.start_timestamp = start_timestamp - res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() - res.timing_info.timer = timer - self._active_config = self.config - - return res diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 46faacae9bb..4ed0fff8e45 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -14,13 +14,18 @@ logger = logging.getLogger(__name__) +def make_count_dict(): + d = {'add': 0, 'remove': 0, 'update': 0, 'set': 0} + return d + + class ObserverChecker(Observer): def __init__(self): super().__init__() self.counts = ComponentMap() """ counts is be a mapping from component (e.g., variable) to another - mapping from string ('add', 'remove', 'update', or 'value') to an int that + mapping from string ('add', 'remove', 'update', or 'set') to an int that indicates the number of times the corresponding method has been called """ @@ -34,57 +39,156 @@ def check(self, expected): def _process(self, comps, key): for c in comps: if c not in self.counts: - self.counts[c] = {'add': 0, 'remove': 0, 'update': 0, 'value': 0} + self.counts[c] = make_count_dict() self.counts[c][key] += 1 def add_variables(self, variables: List[VarData]): + for v in variables: + assert v.is_variable_type() self._process(variables, 'add') def add_parameters(self, params: List[ParamData]): + for p in params: + assert p.is_parameter_type() self._process(params, 'add') def add_constraints(self, cons: List[ConstraintData]): + for c in cons: + assert isinstance(c, ConstraintData) self._process(cons, 'add') def add_sos_constraints(self, cons: List[SOSConstraintData]): + for c in cons: + assert isinstance(c, SOSConstraintData) self._process(cons, 'add') def set_objective(self, obj: ObjectiveData): - self._process([obj], 'add') + assert obj is None or isinstance(obj, ObjectiveData) + self._process([obj], 'set') def remove_constraints(self, cons: List[ConstraintData]): + for c in cons: + assert isinstance(c, ConstraintData) self._process(cons, 'remove') def remove_sos_constraints(self, cons: List[SOSConstraintData]): + for c in cons: + assert isinstance(c, SOSConstraintData) self._process(cons, 'remove') def remove_variables(self, variables: List[VarData]): + for v in variables: + assert v.is_variable_type() self._process(variables, 'remove') def remove_parameters(self, params: List[ParamData]): + for p in params: + assert p.is_parameter_type() self._process(params, 'remove') def update_variables(self, variables: List[VarData]): + for v in variables: + assert v.is_variable_type() self._process(variables, 'update') - def update_parameters_and_fixed_variables(self, params: List[ParamData], variables: List[VarData]): - self._process(params, 'value') - self._process(variables, 'value') + def update_parameters(self, params: List[ParamData]): + for p in params: + assert p.is_parameter_type() + self._process(params, 'update') class TestChangeDetector(unittest.TestCase): - def test_basics(self): + def test_objective(self): m = pe.ConcreteModel() m.x = pe.Var() m.y = pe.Var() + m.p = pe.Param(mutable=True, initialize=1) obs = ObserverChecker() detector = ModelChangeDetector([obs]) + expected = ComponentMap() + expected[None] = make_count_dict() + expected[None]['set'] += 1 + detector.set_instance(m) + obs.check(expected) + + m.obj = pe.Objective(expr=m.x**2 + m.p*m.y**2) + detector.update() + expected[m.obj] = make_count_dict() + expected[m.obj]['set'] += 1 + expected[m.x] = make_count_dict() + expected[m.x]['add'] += 1 + expected[m.y] = make_count_dict() + expected[m.y]['add'] += 1 + expected[m.p] = make_count_dict() + expected[m.p]['add'] += 1 + obs.check(expected) + + m.y.setlb(0) + detector.update() + expected[m.y]['update'] += 1 + obs.check(expected) + + m.x.fix(2) + detector.update() + expected[m.x]['update'] += 1 + expected[m.obj]['set'] += 1 + obs.check(expected) + + m.x.unfix() + detector.update() + expected[m.x]['update'] += 1 + expected[m.obj]['set'] += 1 + obs.check(expected) + + m.p.value = 2 + detector.update() + expected[m.p]['update'] += 1 + obs.check(expected) + + m.obj.expr = m.x**2 + m.y**2 + detector.update() + expected[m.p]['remove'] += 1 + expected[m.obj]['set'] += 1 + obs.check(expected) + + del m.obj + m.obj = pe.Objective(expr=m.p*m.x) + detector.update() + expected[m.p]['add'] += 1 + expected[m.y]['remove'] += 1 + # remember, m.obj is a different object now + expected[m.obj] = make_count_dict() + expected[m.obj]['set'] += 1 + + def test_constraints(self): + m = pe.ConcreteModel() + m.x = pe.Var() + m.y = pe.Var() + m.p = pe.Param(mutable=True, initialize=1) + + obs = ObserverChecker() + detector = ModelChangeDetector([obs]) expected = ComponentMap() + expected[None] = make_count_dict() + expected[None]['set'] += 1 + + detector.set_instance(m) + obs.check(expected) + m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p)**2) + detector.update() + expected[m.x] = make_count_dict() + expected[m.y] = make_count_dict() + expected[m.p] = make_count_dict() + expected[m.x]['add'] += 1 + expected[m.y]['add'] += 1 + expected[m.p]['add'] += 1 + expected[m.c1] = make_count_dict() + expected[m.c1]['add'] += 1 obs.check(expected) def test_vars_and_params_elsewhere(self): From ff635b88c03e6a8ba22148cf32dfe7061c066954 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 8 Aug 2025 07:31:55 -0600 Subject: [PATCH 032/104] working on a model observer --- .../observer/tests/test_change_detector.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 4ed0fff8e45..efda8a181d9 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -42,6 +42,12 @@ def _process(self, comps, key): self.counts[c] = make_count_dict() self.counts[c][key] += 1 + def pprint(self): + for k, d in self.counts.items(): + print(f'{k}:') + for a, v in d.items(): + print(f' {a}: {v}') + def add_variables(self, variables: List[VarData]): for v in variables: assert v.is_variable_type() @@ -179,6 +185,7 @@ def test_constraints(self): detector.set_instance(m) obs.check(expected) + m.obj = pe.Objective(expr=m.y) m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p)**2) detector.update() expected[m.x] = make_count_dict() @@ -189,6 +196,27 @@ def test_constraints(self): expected[m.p]['add'] += 1 expected[m.c1] = make_count_dict() expected[m.c1]['add'] += 1 + expected[m.obj] = make_count_dict() + expected[m.obj]['set'] += 1 + obs.check(expected) + + # now fix a variable and make sure the + # constraint gets removed and added + m.x.fix(1) + obs.pprint() + detector.update() + obs.pprint() + expected[m.c1]['remove'] += 1 + expected[m.c1]['add'] += 1 + # because x and p are only used in the + # one constraint, they get removed when + # the constraint is removed and then + # added again when the constraint is added + expected[m.x]['update'] += 1 + expected[m.x]['remove'] += 1 + expected[m.x]['add'] += 1 + expected[m.p]['remove'] += 1 + expected[m.p]['add'] += 1 obs.check(expected) def test_vars_and_params_elsewhere(self): From b57ab07cde98cc387f8ee0c4f1724f209b05084a Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 9 Aug 2025 07:06:26 -0600 Subject: [PATCH 033/104] updating solution loader --- .../contrib/solver/common/solution_loader.py | 106 +++++++++++++++--- pyomo/contrib/solver/solvers/highs.py | 4 +- 2 files changed, 92 insertions(+), 18 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index 911d8bee50d..065c00185f6 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -9,7 +9,7 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -from typing import Sequence, Dict, Optional, Mapping, NoReturn +from typing import Sequence, Dict, Optional, Mapping, List, Any from pyomo.core.base.constraint import ConstraintData from pyomo.core.base.var import VarData @@ -23,24 +23,75 @@ class SolutionLoaderBase: Intent of this class and its children is to load the solution back into the model. """ - def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None) -> NoReturn: + def get_solution_ids(self) -> List[Any]: """ - Load the solution of the primal variables into the value attribute of the variables. + If there are multiple solutions available, this will return a + list of the solution ids which can then be used with other + methods like `load_soltuion`. If only one solution is + available, this will return [None]. If no solutions + are available, this will return None + + Returns + ------- + solutions_ids: List[Any] + The identifiers for multiple solutions + """ + return NotImplemented + + def get_number_of_solutions(self) -> int: + """ + Returns + ------- + num_solutions: int + Indicates the number of solutions found + """ + return NotImplemented + + def load_solution(self, solution_id=None): + """ + Load the solution (everything that can be) back into the model + + Parameters + ---------- + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be loaded. If None, the default solution will be used. + """ + # this should load everything it can + self.load_vars(solution_id=solution_id) + self.load_import_suffixes(solution_id=solution_id) + + def load_vars( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, + ) -> None: + """ + Load the solution of the primal variables into the value attribute + of the variables. Parameters ---------- vars_to_load: list - The minimum set of variables whose solution should be loaded. If vars_to_load - is None, then the solution to all primal variables will be loaded. Even if - vars_to_load is specified, the values of other variables may also be - loaded depending on the interface. + The minimum set of variables whose solution should be loaded. If + vars_to_load is None, then the solution to all primal variables + will be loaded. Even if vars_to_load is specified, the values of + other variables may also be loaded depending on the interface. + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be loaded. If None, the default solution will be used. """ - for var, val in self.get_primals(vars_to_load=vars_to_load).items(): + for var, val in self.get_vars( + vars_to_load=vars_to_load, + solution_id=solution_id + ).items(): var.set_value(val, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) - def get_primals( - self, vars_to_load: Optional[Sequence[VarData]] = None + def get_vars( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: """ Returns a ComponentMap mapping variable to var value. @@ -50,6 +101,9 @@ def get_primals( vars_to_load: list A list of the variables whose solution value should be retrieved. If vars_to_load is None, then the values for all variables will be retrieved. + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be retrieved. If None, the default solution will be used. Returns ------- @@ -57,11 +111,13 @@ def get_primals( Maps variables to solution values """ raise NotImplementedError( - f"Derived class {self.__class__.__name__} failed to implement required method 'get_primals'." + f"Derived class {self.__class__.__name__} failed to implement required method 'get_vars'." ) def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None + self, + cons_to_load: Optional[Sequence[ConstraintData]] = None, + solution_id=None, ) -> Dict[ConstraintData, float]: """ Returns a dictionary mapping constraint to dual value. @@ -71,16 +127,21 @@ def get_duals( cons_to_load: list A list of the constraints whose duals should be retrieved. If cons_to_load is None, then the duals for all constraints will be retrieved. + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be retrieved. If None, the default solution will be used. Returns ------- duals: dict Maps constraints to dual values """ - raise NotImplementedError(f'{type(self)} does not support the get_duals method') + return NotImplemented def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: """ Returns a ComponentMap mapping variable to reduced cost. @@ -90,15 +151,26 @@ def get_reduced_costs( vars_to_load: list A list of the variables whose reduced cost should be retrieved. If vars_to_load is None, then the reduced costs for all variables will be loaded. + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be retrieved. If None, the default solution will be used. Returns ------- reduced_costs: ComponentMap Maps variables to reduced costs """ - raise NotImplementedError( - f'{type(self)} does not support the get_reduced_costs method' - ) + return NotImplemented + + def load_import_suffixes(self, solution_id=None): + """ + Parameters + ---------- + solution_id: Optional[Any] + If there are multiple solutions, this specifies which solution + should be loaded. If None, the default solution will be used. + """ + return NotImplemented class PersistentSolutionLoader(SolutionLoaderBase): diff --git a/pyomo/contrib/solver/solvers/highs.py b/pyomo/contrib/solver/solvers/highs.py index 2fdac4942c8..6eb4afa828a 100644 --- a/pyomo/contrib/solver/solvers/highs.py +++ b/pyomo/contrib/solver/solvers/highs.py @@ -306,7 +306,9 @@ def _solve(self): self._solver_model.run() timer.stop('optimize') - return self._postsolve() + res = self._postsolve() + res.solver_log = ostreams[0].getvalue() + return res def _process_domain_and_bounds(self, var_id): _v, _lb, _ub, _fixed, _domain_interval, _value = self._vars[var_id] From 070811d91c992691f16bbf7b97d9de2310fc7398 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 9 Aug 2025 07:20:11 -0600 Subject: [PATCH 034/104] refactoring gurobi interfaces --- .../solver/solvers/gurobi_persistent.py | 156 ++++++++++++------ 1 file changed, 108 insertions(+), 48 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi_persistent.py index ea3693c1c70..899b7915e80 100644 --- a/pyomo/contrib/solver/solvers/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi_persistent.py @@ -30,7 +30,7 @@ from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types from pyomo.repn import generate_standard_repn from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression -from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability +from pyomo.contrib.solver.common.base import PersistentSolverBase, SolverBase, Availability from pyomo.contrib.solver.common.results import ( Results, TerminationCondition, @@ -233,44 +233,15 @@ def __init__(self): self.var2 = None -class GurobiPersistent( - GurobiSolverMixin, - PersistentSolverMixin, - PersistentSolverUtils, - PersistentSolverBase, -): - """ - Interface to Gurobi persistent - """ - +class GurobiBase(SolverBase): CONFIG = GurobiConfig() _gurobipy_available = gurobipy_available def __init__(self, **kwds): - treat_fixed_vars_as_params = kwds.pop('treat_fixed_vars_as_params', True) - PersistentSolverBase.__init__(self, **kwds) - PersistentSolverUtils.__init__( - self, treat_fixed_vars_as_params=treat_fixed_vars_as_params - ) + super().__init__(**kwds) self._register_env_client() self._solver_model = None - self._symbol_map = SymbolMap() - self._labeler = None - self._pyomo_var_to_solver_var_map = {} - self._pyomo_con_to_solver_con_map = {} - self._solver_con_to_pyomo_con_map = {} - self._pyomo_sos_to_solver_sos_map = {} - self._range_constraints = OrderedSet() - self._mutable_helpers = {} - self._mutable_bounds = {} - self._mutable_quadratic_helpers = {} - self._mutable_objective = None - self._needs_updated = True self._callback = None - self._callback_func = None - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._last_results_object: Optional[Results] = None def release_license(self): self._reinit() @@ -280,12 +251,10 @@ def __del__(self): if not python_is_shutting_down(): self._release_env_client() - @property - def symbol_map(self): - return self._symbol_map + def _mipstart(self): + raise NotImplementedError('should be implemented by derived classes') - def _solve(self): - config = self._active_config + def _solve(self, config): timer = config.timer ostreams = [io.StringIO()] + config.tee @@ -304,13 +273,7 @@ def _solve(self): self._solver_model.setParam('MIPGapAbs', config.abs_gap) if config.use_mipstart: - for ( - pyomo_var_id, - gurobi_var, - ) in self._pyomo_var_to_solver_var_map.items(): - pyomo_var = self._vars[pyomo_var_id][0] - if pyomo_var.is_integer() and pyomo_var.value is not None: - self.set_var_attr(pyomo_var, 'Start', pyomo_var.value) + self._mipstart() for key, option in options.items(): self._solver_model.setParam(key, option) @@ -319,18 +282,99 @@ def _solve(self): self._solver_model.optimize(self._callback) timer.stop('optimize') - self._needs_updated = False res = self._postsolve(timer) res.solver_config = config res.solver_name = 'Gurobi' res.solver_version = self.version() res.solver_log = ostreams[0].getvalue() return res + + +class GurobiDirect(GurobiBase): + def __init__(self, **kwds): + super().__init__(**kwds) + + +class GurobiQuadraticBase(GurobiBase): + def __init__(self, **kwds): + super().__init__(**kwds) + self._vars = {} # from id(v) to v + self._symbol_map = SymbolMap() + self._labeler = None + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + + @property + def symbol_map(self): + return self._symbol_map + + def _mipstart(self): + for ( + pyomo_var_id, + gurobi_var, + ) in self._pyomo_var_to_solver_var_map.items(): + pyomo_var = self._vars[pyomo_var_id] + if pyomo_var.is_integer() and pyomo_var.value is not None: + gurobi_var.setAttr('Start', pyomo_var.value) + + def _proces_domain_and_bounds(self, var): + lb, ub, step = var.domain.get_interval() + if lb is None: + lb = -gurobipy.GRB.INFINITY + if ub is None: + ub = gurobipy.GRB.INFINITY + if step == 0: + vtype = gurobipy.GRB.CONTINUOUS + elif step == 1: + if lb == 0 and ub == 1: + vtype = gurobipy.GRB.BINARY + else: + vtype = gurobipy.GRB.INTEGER + else: + raise ValueError( + f'Unrecognized domain step: {step} (should be either 0 or 1)' + ) + if var.fixed: + lb = var.value + ub = lb + else: + lb = max(lb, value(var._lb)) + ub = min(ub, value(var._ub)) + return lb, ub, vtype + + +class GurobiDirectQuadratic(GurobiQuadraticBase): + def __init__(self, **kwds): + super().__init__(**kwds) + + +class GurobiPersistentQuadratic(GurobiQuadraticBase): + def __init__(self, **kwds): + super().__init__(**kwds) + self._solver_con_to_pyomo_con_map = {} + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._callback_func = None + self._last_results_object: Optional[Results] = None + + def _solve(self, config): + super()._solve(config) + self._needs_updated = False def _process_domain_and_bounds( self, var, var_id, mutable_lbs, mutable_ubs, ndx, gurobipy_var ): - _v, _lb, _ub, _fixed, _domain_interval, _value = self._vars[id(var)] + _lb = var._lb + _ub = var._ub + _fixed = var.fixed + _domain_interval = var.domain.get_interval() + _value = var.value lb, ub, step = _domain_interval if lb is None: lb = -gurobipy.GRB.INFINITY @@ -372,6 +416,24 @@ def _process_domain_and_bounds( return lb, ub, vtype + +class _GurobiPersistent( + GurobiSolverMixin, + PersistentSolverMixin, + PersistentSolverUtils, + PersistentSolverBase, +): + """ + Interface to Gurobi persistent + """ + + + def __init__(self, **kwds): + PersistentSolverBase.__init__(self, **kwds) + PersistentSolverUtils.__init__( + self, treat_fixed_vars_as_params=treat_fixed_vars_as_params + ) + def _add_variables(self, variables: List[VarData]): var_names = [] vtypes = [] @@ -522,7 +584,6 @@ def _add_constraints(self, cons: List[ConstraintData]): gurobipy_con = self._solver_model.addRange( gurobi_expr, lhs_val, rhs_val, name=conname ) - self._range_constraints.add(con) if not is_constant(lhs_expr) or not is_constant(rhs_expr): mutable_range_constant = _MutableRangeConstant() mutable_range_constant.lhs_expr = lhs_expr @@ -654,7 +715,6 @@ def _remove_constraints(self, cons: List[ConstraintData]): self._symbol_map.removeSymbol(con) del self._pyomo_con_to_solver_con_map[con] del self._solver_con_to_pyomo_con_map[id(solver_con)] - self._range_constraints.discard(con) self._mutable_helpers.pop(con, None) self._mutable_quadratic_helpers.pop(con, None) self._needs_updated = True From d70dbb52a28aeb3500bdd430919f43f94c7a862c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 10 Aug 2025 15:36:39 -0600 Subject: [PATCH 035/104] revert_gurobi_persistent --- .../solver/solvers/gurobi_persistent.py | 156 ++++++------------ 1 file changed, 48 insertions(+), 108 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi_persistent.py index 899b7915e80..ea3693c1c70 100644 --- a/pyomo/contrib/solver/solvers/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi_persistent.py @@ -30,7 +30,7 @@ from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types from pyomo.repn import generate_standard_repn from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression -from pyomo.contrib.solver.common.base import PersistentSolverBase, SolverBase, Availability +from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability from pyomo.contrib.solver.common.results import ( Results, TerminationCondition, @@ -233,15 +233,44 @@ def __init__(self): self.var2 = None -class GurobiBase(SolverBase): +class GurobiPersistent( + GurobiSolverMixin, + PersistentSolverMixin, + PersistentSolverUtils, + PersistentSolverBase, +): + """ + Interface to Gurobi persistent + """ + CONFIG = GurobiConfig() _gurobipy_available = gurobipy_available def __init__(self, **kwds): - super().__init__(**kwds) + treat_fixed_vars_as_params = kwds.pop('treat_fixed_vars_as_params', True) + PersistentSolverBase.__init__(self, **kwds) + PersistentSolverUtils.__init__( + self, treat_fixed_vars_as_params=treat_fixed_vars_as_params + ) self._register_env_client() self._solver_model = None + self._symbol_map = SymbolMap() + self._labeler = None + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._solver_con_to_pyomo_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + self._range_constraints = OrderedSet() + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True self._callback = None + self._callback_func = None + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._last_results_object: Optional[Results] = None def release_license(self): self._reinit() @@ -251,10 +280,12 @@ def __del__(self): if not python_is_shutting_down(): self._release_env_client() - def _mipstart(self): - raise NotImplementedError('should be implemented by derived classes') + @property + def symbol_map(self): + return self._symbol_map - def _solve(self, config): + def _solve(self): + config = self._active_config timer = config.timer ostreams = [io.StringIO()] + config.tee @@ -273,7 +304,13 @@ def _solve(self, config): self._solver_model.setParam('MIPGapAbs', config.abs_gap) if config.use_mipstart: - self._mipstart() + for ( + pyomo_var_id, + gurobi_var, + ) in self._pyomo_var_to_solver_var_map.items(): + pyomo_var = self._vars[pyomo_var_id][0] + if pyomo_var.is_integer() and pyomo_var.value is not None: + self.set_var_attr(pyomo_var, 'Start', pyomo_var.value) for key, option in options.items(): self._solver_model.setParam(key, option) @@ -282,99 +319,18 @@ def _solve(self, config): self._solver_model.optimize(self._callback) timer.stop('optimize') + self._needs_updated = False res = self._postsolve(timer) res.solver_config = config res.solver_name = 'Gurobi' res.solver_version = self.version() res.solver_log = ostreams[0].getvalue() return res - - -class GurobiDirect(GurobiBase): - def __init__(self, **kwds): - super().__init__(**kwds) - - -class GurobiQuadraticBase(GurobiBase): - def __init__(self, **kwds): - super().__init__(**kwds) - self._vars = {} # from id(v) to v - self._symbol_map = SymbolMap() - self._labeler = None - self._pyomo_var_to_solver_var_map = {} - self._pyomo_con_to_solver_con_map = {} - self._pyomo_sos_to_solver_sos_map = {} - - @property - def symbol_map(self): - return self._symbol_map - - def _mipstart(self): - for ( - pyomo_var_id, - gurobi_var, - ) in self._pyomo_var_to_solver_var_map.items(): - pyomo_var = self._vars[pyomo_var_id] - if pyomo_var.is_integer() and pyomo_var.value is not None: - gurobi_var.setAttr('Start', pyomo_var.value) - - def _proces_domain_and_bounds(self, var): - lb, ub, step = var.domain.get_interval() - if lb is None: - lb = -gurobipy.GRB.INFINITY - if ub is None: - ub = gurobipy.GRB.INFINITY - if step == 0: - vtype = gurobipy.GRB.CONTINUOUS - elif step == 1: - if lb == 0 and ub == 1: - vtype = gurobipy.GRB.BINARY - else: - vtype = gurobipy.GRB.INTEGER - else: - raise ValueError( - f'Unrecognized domain step: {step} (should be either 0 or 1)' - ) - if var.fixed: - lb = var.value - ub = lb - else: - lb = max(lb, value(var._lb)) - ub = min(ub, value(var._ub)) - return lb, ub, vtype - - -class GurobiDirectQuadratic(GurobiQuadraticBase): - def __init__(self, **kwds): - super().__init__(**kwds) - - -class GurobiPersistentQuadratic(GurobiQuadraticBase): - def __init__(self, **kwds): - super().__init__(**kwds) - self._solver_con_to_pyomo_con_map = {} - self._mutable_helpers = {} - self._mutable_bounds = {} - self._mutable_quadratic_helpers = {} - self._mutable_objective = None - self._needs_updated = True - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._callback_func = None - self._last_results_object: Optional[Results] = None - - def _solve(self, config): - super()._solve(config) - self._needs_updated = False def _process_domain_and_bounds( self, var, var_id, mutable_lbs, mutable_ubs, ndx, gurobipy_var ): - _lb = var._lb - _ub = var._ub - _fixed = var.fixed - _domain_interval = var.domain.get_interval() - _value = var.value + _v, _lb, _ub, _fixed, _domain_interval, _value = self._vars[id(var)] lb, ub, step = _domain_interval if lb is None: lb = -gurobipy.GRB.INFINITY @@ -416,24 +372,6 @@ def _process_domain_and_bounds( return lb, ub, vtype - -class _GurobiPersistent( - GurobiSolverMixin, - PersistentSolverMixin, - PersistentSolverUtils, - PersistentSolverBase, -): - """ - Interface to Gurobi persistent - """ - - - def __init__(self, **kwds): - PersistentSolverBase.__init__(self, **kwds) - PersistentSolverUtils.__init__( - self, treat_fixed_vars_as_params=treat_fixed_vars_as_params - ) - def _add_variables(self, variables: List[VarData]): var_names = [] vtypes = [] @@ -584,6 +522,7 @@ def _add_constraints(self, cons: List[ConstraintData]): gurobipy_con = self._solver_model.addRange( gurobi_expr, lhs_val, rhs_val, name=conname ) + self._range_constraints.add(con) if not is_constant(lhs_expr) or not is_constant(rhs_expr): mutable_range_constant = _MutableRangeConstant() mutable_range_constant.lhs_expr = lhs_expr @@ -715,6 +654,7 @@ def _remove_constraints(self, cons: List[ConstraintData]): self._symbol_map.removeSymbol(con) del self._pyomo_con_to_solver_con_map[con] del self._solver_con_to_pyomo_con_map[id(solver_con)] + self._range_constraints.discard(con) self._mutable_helpers.pop(con, None) self._mutable_quadratic_helpers.pop(con, None) self._needs_updated = True From 5b1d3f9cfb551598599a8f2ecf99313517bf06f7 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 10 Aug 2025 20:43:45 -0600 Subject: [PATCH 036/104] refactoring gurobi interfaces --- .../contrib/solver/solvers/gurobi/__init__.py | 0 .../solver/solvers/gurobi/gurobi_direct.py | 201 +++++++++++ .../solvers/gurobi/gurobi_direct_base.py | 328 ++++++++++++++++++ 3 files changed, 529 insertions(+) create mode 100644 pyomo/contrib/solver/solvers/gurobi/__init__.py create mode 100644 pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py create mode 100644 pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py diff --git a/pyomo/contrib/solver/solvers/gurobi/__init__.py b/pyomo/contrib/solver/solvers/gurobi/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py new file mode 100644 index 00000000000..5c36372ef72 --- /dev/null +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -0,0 +1,201 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import datetime +import io +import math +import operator +import os + +from pyomo.common.collections import ComponentMap, ComponentSet +from pyomo.common.config import ConfigValue +from pyomo.common.dependencies import attempt_import +from pyomo.common.enums import ObjectiveSense +from pyomo.common.errors import MouseTrap, ApplicationError +from pyomo.common.shutdown import python_is_shutting_down +from pyomo.common.tee import capture_output, TeeStream +from pyomo.common.timing import HierarchicalTimer +from pyomo.core.staleflag import StaleFlagManager + +from pyomo.contrib.solver.common.base import SolverBase, Availability +from pyomo.contrib.solver.common.config import BranchAndBoundConfig +from pyomo.contrib.solver.common.util import ( + NoFeasibleSolutionError, + NoOptimalSolutionError, + NoDualsError, + NoReducedCostsError, + NoSolutionError, + IncompatibleModelError, +) +from pyomo.contrib.solver.common.results import ( + Results, + SolutionStatus, + TerminationCondition, +) +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +import logging +from .gurobi_direct_base import GurobiDirectBase, gurobipy + + +class GurobiDirectSolutionLoader(SolutionLoaderBase): + def __init__(self, grb_model, grb_cons, grb_vars, pyo_cons, pyo_vars): + self._grb_model = grb_model + self._grb_cons = grb_cons + self._grb_vars = grb_vars + self._pyo_cons = pyo_cons + self._pyo_vars = pyo_vars + GurobiDirectBase._register_env_client() + + def __del__(self): + if python_is_shutting_down(): + return + # Free the associated model + if self._grb_model is not None: + self._grb_cons = None + self._grb_vars = None + self._pyo_cons = None + self._pyo_vars = None + # explicitly release the model + self._grb_model.dispose() + self._grb_model = None + # Release the gurobi license if this is the last reference to + # the environment (either through a results object or solver + # interface) + GurobiDirectBase._release_env_client() + + def load_vars(self, vars_to_load=None, solution_number=0): + assert solution_number == 0 + if self._grb_model.SolCount == 0: + raise NoSolutionError() + + iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) + if vars_to_load: + vars_to_load = ComponentSet(vars_to_load) + iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) + for p_var, g_var in iterator: + p_var.set_value(g_var, skip_validation=True) + StaleFlagManager.mark_all_as_stale(delayed=True) + + def get_primals(self, vars_to_load=None, solution_number=0): + assert solution_number == 0 + if self._grb_model.SolCount == 0: + raise NoSolutionError() + + iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) + if vars_to_load: + vars_to_load = ComponentSet(vars_to_load) + iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) + return ComponentMap(iterator) + + def get_duals(self, cons_to_load=None): + if self._grb_model.Status != gurobipy.GRB.OPTIMAL: + raise NoDualsError() + + def dedup(_iter): + last = None + for con_info_dual in _iter: + if not con_info_dual[1] and con_info_dual[0][0] is last: + continue + last = con_info_dual[0][0] + yield con_info_dual + + iterator = dedup(zip(self._pyo_cons, self._grb_cons.getAttr('Pi').tolist())) + if cons_to_load: + cons_to_load = set(cons_to_load) + iterator = filter( + lambda con_info_dual: con_info_dual[0][0] in cons_to_load, iterator + ) + return {con_info[0]: dual for con_info, dual in iterator} + + def get_reduced_costs(self, vars_to_load=None): + if self._grb_model.Status != gurobipy.GRB.OPTIMAL: + raise NoReducedCostsError() + + iterator = zip(self._pyo_vars, self._grb_vars.getAttr('Rc').tolist()) + if vars_to_load: + vars_to_load = ComponentSet(vars_to_load) + iterator = filter(lambda var_rc: var_rc[0] in vars_to_load, iterator) + return ComponentMap(iterator) + + +class GurobiDirect(GurobiDirectBase): + def __init__(self, **kwds): + super().__init__(**kwds) + self._gurobi_vars = None + self._pyomo_vars = None + + def _pyomo_gurobi_var_iter(self): + return zip(self._pyomo_vars, self._gurobi_vars.tolist()) + + def _create_solver_model(self, pyomo_model, config): + timer = config.timer + + timer.start('compile_model') + repn = LinearStandardFormCompiler().write( + pyomo_model, mixed_form=True, set_sense=None + ) + timer.stop('compile_model') + + if len(repn.objectives) > 1: + raise IncompatibleModelError( + f"The {self.__class__.__name__} solver only supports models " + f"with zero or one objectives (received {len(repn.objectives)})." + ) + + timer.start('prepare_matrices') + inf = float('inf') + ninf = -inf + bounds = list(map(operator.attrgetter('bounds'), repn.columns)) + lb = [ninf if _b is None else _b for _b in map(operator.itemgetter(0), bounds)] + ub = [inf if _b is None else _b for _b in map(operator.itemgetter(1), bounds)] + CON = gurobipy.GRB.CONTINUOUS + BIN = gurobipy.GRB.BINARY + INT = gurobipy.GRB.INTEGER + vtype = [ + ( + CON + if v.is_continuous() + else BIN if v.is_binary() else INT if v.is_integer() else '?' + ) + for v in repn.columns + ] + sense_type = list('=<>') # Note: ordering matches 0, 1, -1 + sense = [sense_type[r[1]] for r in repn.rows] + timer.stop('prepare_matrices') + + gurobi_model = gurobipy.Model(env=self.env()) + + timer.start('transfer_model') + x = gurobi_model.addMVar( + len(repn.columns), + lb=lb, + ub=ub, + obj=repn.c.todense()[0] if repn.c.shape[0] else 0, + vtype=vtype, + ) + A = gurobi_model.addMConstr(repn.A, x, sense, repn.rhs) + if repn.c.shape[0]: + gurobi_model.setAttr('ObjCon', repn.c_offset[0]) + gurobi_model.setAttr('ModelSense', int(repn.objectives[0].sense)) + # Note: calling gurobi_model.update() here is not + # necessary (it will happen as part of optimize()): + # gurobi_model.update() + timer.stop('transfer_model') + + self._pyomo_vars = repn.columns + self._gurobi_vars = x + + solution_loader = GurobiDirectSolutionLoader( + gurobi_model, A, x, repn.rows, repn.columns + ) + has_obj = len(repn.objectives) > 0 + + return gurobi_model, solution_loader, has_obj diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py new file mode 100644 index 00000000000..01c91b8b2ed --- /dev/null +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -0,0 +1,328 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import datetime +import io +import math +import operator +import os + +from pyomo.common.collections import ComponentMap, ComponentSet +from pyomo.common.config import ConfigValue +from pyomo.common.dependencies import attempt_import +from pyomo.common.enums import ObjectiveSense +from pyomo.common.errors import MouseTrap, ApplicationError +from pyomo.common.shutdown import python_is_shutting_down +from pyomo.common.tee import capture_output, TeeStream +from pyomo.common.timing import HierarchicalTimer +from pyomo.core.staleflag import StaleFlagManager + +from pyomo.contrib.solver.common.base import SolverBase, Availability +from pyomo.contrib.solver.common.config import BranchAndBoundConfig +from pyomo.contrib.solver.common.util import ( + NoFeasibleSolutionError, + NoOptimalSolutionError, + NoDualsError, + NoReducedCostsError, + NoSolutionError, + IncompatibleModelError, +) +from pyomo.contrib.solver.common.results import ( + Results, + SolutionStatus, + TerminationCondition, +) +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +import logging + + +logger = logging.getLogger(__name__) + + +gurobipy, gurobipy_available = attempt_import('gurobipy') + + +class GurobiConfig(BranchAndBoundConfig): + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + BranchAndBoundConfig.__init__( + self, + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + self.use_mipstart: bool = self.declare( + 'use_mipstart', + ConfigValue( + default=False, + domain=bool, + description="If True, the current values of the integer variables " + "will be passed to Gurobi.", + ), + ) + + +class GurobiDirectBase(SolverBase): + + _num_gurobipy_env_clients = 0 + _gurobipy_env = None + _available = None + _gurobipy_available = gurobipy_available + _tc_map = None + + CONFIG = GurobiConfig() + + def __init__(self, **kwds): + super().__init__(**kwds) + self._register_env_client() + + def __del__(self): + if not python_is_shutting_down(): + self._release_env_client() + + def available(self): + if self._available is None: + # this triggers the deferred import, and for the persistent + # interface, may update the _available flag + # + # Note that we set the _available flag on the *most derived + # class* and not on the instance, or on the base class. That + # allows different derived interfaces to have different + # availability (e.g., persistent has a minimum version + # requirement that the direct interface doesn't - is that true?) + if not self._gurobipy_available: + if self._available is None: + self.__class__._available = Availability.NotFound + else: + self.__class__._available = self._check_license() + return self._available + + @staticmethod + def release_license(): + if GurobiDirectBase._gurobipy_env is None: + return + if GurobiDirectBase._num_gurobipy_env_clients: + logger.warning( + "Call to GurobiDirectBase.release_license() with %s remaining " + "environment clients." % (GurobiDirectBase._num_gurobipy_env_clients,) + ) + GurobiDirectBase._gurobipy_env.close() + GurobiDirectBase._gurobipy_env = None + + @staticmethod + def env(): + if GurobiDirectBase._gurobipy_env is None: + with capture_output(capture_fd=True): + GurobiDirectBase._gurobipy_env = gurobipy.Env() + return GurobiDirectBase._gurobipy_env + + @staticmethod + def _register_env_client(): + GurobiDirectBase._num_gurobipy_env_clients += 1 + + @staticmethod + def _release_env_client(): + GurobiDirectBase._num_gurobipy_env_clients -= 1 + if GurobiDirectBase._num_gurobipy_env_clients <= 0: + # Note that _num_gurobipy_env_clients should never be <0, + # but if it is, release_license will issue a warning (that + # we want to know about) + GurobiDirectBase.release_license() + + def _check_license(self): + try: + model = gurobipy.Model(env=self.env()) + except gurobipy.GurobiError: + return Availability.BadLicense + + model.setParam('OutputFlag', 0) + try: + model.addVars(range(2001)) + model.optimize() + return Availability.FullLicense + except gurobipy.GurobiError: + return Availability.LimitedLicense + finally: + model.dispose() + + def version(self): + version = ( + gurobipy.GRB.VERSION_MAJOR, + gurobipy.GRB.VERSION_MINOR, + gurobipy.GRB.VERSION_TECHNICAL, + ) + return version + + def _create_solver_model(self, pyomo_model, config): + # should return gurobi_model, solution_loader, has_objective + raise NotImplementedError('should be implemented by derived classes') + + def _pyomo_gurobi_var_iter(self): + # generator of tuples (pyomo_var, gurobi_var) + raise NotImplementedError('should be implemented by derived classes') + + def _mipstart(self): + for pyomo_var, gurobi_var in self._pyomo_gurobi_var_iter(): + if pyomo_var.is_integer() and pyomo_var.value is not None: + gurobi_var.setAttr('Start', pyomo_var.value) + + def solve(self, model, **kwds) -> Results: + start_timestamp = datetime.datetime.now(datetime.timezone.utc) + config: GurobiConfig = self.config( + value=kwds, + preserve_implicit=True, + ) + if not self.available(): + c = self.__class__ + raise ApplicationError( + f'Solver {c.__module__}.{c.__qualname__} is not available ' + f'({self.available()}).' + ) + if config.timer is None: + config.timer = HierarchicalTimer() + timer = config.timer + + StaleFlagManager.mark_all_as_stale() + ostreams = [io.StringIO()] + config.tee + + orig_cwd = os.getcwd() + try: + if config.working_dir: + os.chdir(config.working_dir) + with capture_output(TeeStream(*ostreams), capture_fd=False): + gurobi_model, solution_loader, has_obj = self._create_solver_model(model, config) + options = config.solver_options + + gurobi_model.setParam('LogToConsole', 1) + + if config.threads is not None: + gurobi_model.setParam('Threads', config.threads) + if config.time_limit is not None: + gurobi_model.setParam('TimeLimit', config.time_limit) + if config.rel_gap is not None: + gurobi_model.setParam('MIPGap', config.rel_gap) + if config.abs_gap is not None: + gurobi_model.setParam('MIPGapAbs', config.abs_gap) + + if config.use_mipstart: + self._mipstart() + + for key, option in options.items(): + gurobi_model.setParam(key, option) + + timer.start('optimize') + gurobi_model.optimize() + timer.stop('optimize') + finally: + os.chdir(orig_cwd) + + res = self._postsolve( + grb_model=gurobi_model, + config=config, + ) + + res.solution_loader = solution_loader + res.solver_log = ostreams[0].getvalue() + end_timestamp = datetime.datetime.now(datetime.timezone.utc) + res.timing_info.start_timestamp = start_timestamp + res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() + res.timing_info.timer = timer + return res + + def _get_tc_map(self): + if GurobiDirectBase._tc_map is None: + grb = gurobipy.GRB + tc = TerminationCondition + GurobiDirectBase._tc_map = { + grb.LOADED: tc.unknown, # problem is loaded, but no solution + grb.OPTIMAL: tc.convergenceCriteriaSatisfied, + grb.INFEASIBLE: tc.provenInfeasible, + grb.INF_OR_UNBD: tc.infeasibleOrUnbounded, + grb.UNBOUNDED: tc.unbounded, + grb.CUTOFF: tc.objectiveLimit, + grb.ITERATION_LIMIT: tc.iterationLimit, + grb.NODE_LIMIT: tc.iterationLimit, + grb.TIME_LIMIT: tc.maxTimeLimit, + grb.SOLUTION_LIMIT: tc.unknown, + grb.INTERRUPTED: tc.interrupted, + grb.NUMERIC: tc.unknown, + grb.SUBOPTIMAL: tc.unknown, + grb.USER_OBJ_LIMIT: tc.objectiveLimit, + } + return GurobiDirectBase._tc_map + + def _postsolve(self, grb_model, config, has_obj): + status = grb_model.Status + + results = Results() + results.timing_info.gurobi_time = grb_model.Runtime + + if grb_model.SolCount > 0: + if status == gurobipy.GRB.OPTIMAL: + results.solution_status = SolutionStatus.optimal + else: + results.solution_status = SolutionStatus.feasible + else: + results.solution_status = SolutionStatus.noSolution + + results.termination_condition = self._get_tc_map().get( + status, TerminationCondition.unknown + ) + + if ( + results.termination_condition + != TerminationCondition.convergenceCriteriaSatisfied + and config.raise_exception_on_nonoptimal_result + ): + raise NoOptimalSolutionError() + + if has_obj: + try: + if math.isfinite(grb_model.ObjVal): + results.incumbent_objective = grb_model.ObjVal + else: + results.incumbent_objective = None + except (gurobipy.GurobiError, AttributeError): + results.incumbent_objective = None + try: + results.objective_bound = grb_model.ObjBound + except (gurobipy.GurobiError, AttributeError): + if grb_model.ModelSense == ObjectiveSense.minimize: + results.objective_bound = -math.inf + else: + results.objective_bound = math.inf + else: + results.incumbent_objective = None + results.objective_bound = None + + results.iteration_count = grb_model.getAttr('IterCount') + + config.timer.start('load solution') + if config.load_solutions: + if grb_model.SolCount > 0: + results.solution_loader.load_vars() + else: + raise NoFeasibleSolutionError() + config.timer.stop('load solution') + + results.solver_config = config + results.solver_name = self.name + results.solver_version = self.version() + + return results From 4818130badff45f8873326b32710bfba2e87e7fd Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 11 Aug 2025 06:23:20 -0600 Subject: [PATCH 037/104] refactoring gurobi interfaces --- .../solver/solvers/gurobi/gurobi_direct.py | 7 +- .../solvers/gurobi/gurobi_direct_base.py | 72 ++-- .../solvers/gurobi/gurobi_persistent.py | 335 ++++++++++++++++++ 3 files changed, 381 insertions(+), 33 deletions(-) create mode 100644 pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index 5c36372ef72..f4a33e2cc54 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -24,6 +24,7 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer from pyomo.core.staleflag import StaleFlagManager +from pyomo.repn.plugins.standard_form import LinearStandardFormCompiler from pyomo.contrib.solver.common.base import SolverBase, Availability from pyomo.contrib.solver.common.config import BranchAndBoundConfig @@ -127,6 +128,8 @@ def get_reduced_costs(self, vars_to_load=None): class GurobiDirect(GurobiDirectBase): + _minimum_version = (9, 0, 0) + def __init__(self, **kwds): super().__init__(**kwds) self._gurobi_vars = None @@ -135,8 +138,8 @@ def __init__(self, **kwds): def _pyomo_gurobi_var_iter(self): return zip(self._pyomo_vars, self._gurobi_vars.tolist()) - def _create_solver_model(self, pyomo_model, config): - timer = config.timer + def _create_solver_model(self, pyomo_model): + timer = self.config.timer timer.start('compile_model') repn = LinearStandardFormCompiler().write( diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 01c91b8b2ed..b314a39b49a 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -85,12 +85,14 @@ class GurobiDirectBase(SolverBase): _available = None _gurobipy_available = gurobipy_available _tc_map = None + _minimum_version = (0, 0, 0) CONFIG = GurobiConfig() def __init__(self, **kwds): super().__init__(**kwds) self._register_env_client() + self._callback = None def __del__(self): if not python_is_shutting_down(): @@ -111,6 +113,8 @@ def available(self): self.__class__._available = Availability.NotFound else: self.__class__._available = self._check_license() + if self.version() < self._minimum_version: + self.__class__._available = Availability.BadVersion return self._available @staticmethod @@ -169,7 +173,7 @@ def version(self): ) return version - def _create_solver_model(self, pyomo_model, config): + def _create_solver_model(self, pyomo_model): # should return gurobi_model, solution_loader, has_objective raise NotImplementedError('should be implemented by derived classes') @@ -184,29 +188,30 @@ def _mipstart(self): def solve(self, model, **kwds) -> Results: start_timestamp = datetime.datetime.now(datetime.timezone.utc) - config: GurobiConfig = self.config( - value=kwds, - preserve_implicit=True, - ) - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) - if config.timer is None: - config.timer = HierarchicalTimer() - timer = config.timer - - StaleFlagManager.mark_all_as_stale() - ostreams = [io.StringIO()] + config.tee - + orig_config = self.config orig_cwd = os.getcwd() try: + self.config = config = self.config( + value=kwds, + preserve_implicit=True, + ) + if not self.available(): + c = self.__class__ + raise ApplicationError( + f'Solver {c.__module__}.{c.__qualname__} is not available ' + f'({self.available()}).' + ) + if config.timer is None: + config.timer = HierarchicalTimer() + timer = config.timer + + StaleFlagManager.mark_all_as_stale() + ostreams = [io.StringIO()] + config.tee + if config.working_dir: os.chdir(config.working_dir) with capture_output(TeeStream(*ostreams), capture_fd=False): - gurobi_model, solution_loader, has_obj = self._create_solver_model(model, config) + gurobi_model, solution_loader, has_obj = self._create_solver_model(model) options = config.solver_options gurobi_model.setParam('LogToConsole', 1) @@ -227,15 +232,16 @@ def solve(self, model, **kwds) -> Results: gurobi_model.setParam(key, option) timer.start('optimize') - gurobi_model.optimize() + gurobi_model.optimize(self._callback) timer.stop('optimize') + + res = self._postsolve( + grb_model=gurobi_model, + has_obj=has_obj, + ) finally: os.chdir(orig_cwd) - - res = self._postsolve( - grb_model=gurobi_model, - config=config, - ) + self.config = orig_config res.solution_loader = solution_loader res.solver_log = ostreams[0].getvalue() @@ -267,7 +273,7 @@ def _get_tc_map(self): } return GurobiDirectBase._tc_map - def _postsolve(self, grb_model, config, has_obj): + def _postsolve(self, grb_model, has_obj): status = grb_model.Status results = Results() @@ -288,7 +294,7 @@ def _postsolve(self, grb_model, config, has_obj): if ( results.termination_condition != TerminationCondition.convergenceCriteriaSatisfied - and config.raise_exception_on_nonoptimal_result + and self.config.raise_exception_on_nonoptimal_result ): raise NoOptimalSolutionError() @@ -313,15 +319,19 @@ def _postsolve(self, grb_model, config, has_obj): results.iteration_count = grb_model.getAttr('IterCount') - config.timer.start('load solution') - if config.load_solutions: + self.config.timer.start('load solution') + if self.config.load_solutions: if grb_model.SolCount > 0: results.solution_loader.load_vars() else: raise NoFeasibleSolutionError() - config.timer.stop('load solution') + self.config.timer.stop('load solution') - results.solver_config = config + # self.config gets copied a the beginning of + # solve and restored at the end, so modifying + # results.solver_config will not actually + # modify self.config + results.solver_config = self.config results.solver_name = self.name results.solver_version = self.version() diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py new file mode 100644 index 00000000000..fa269c1d3c5 --- /dev/null +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -0,0 +1,335 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + +import io +import logging +import math +from typing import List, Optional +from collections.abc import Iterable + +from pyomo.common.collections import ComponentSet, ComponentMap, OrderedSet +from pyomo.common.dependencies import attempt_import +from pyomo.common.errors import ApplicationError +from pyomo.common.tee import capture_output, TeeStream +from pyomo.common.timing import HierarchicalTimer +from pyomo.common.shutdown import python_is_shutting_down +from pyomo.core.kernel.objective import minimize, maximize +from pyomo.core.base import SymbolMap, NumericLabeler, TextLabeler +from pyomo.core.base.var import VarData +from pyomo.core.base.constraint import ConstraintData, Constraint +from pyomo.core.base.sos import SOSConstraintData, SOSConstraint +from pyomo.core.base.param import ParamData +from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types +from pyomo.repn import generate_standard_repn +from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression +from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability +from pyomo.contrib.solver.common.results import ( + Results, + TerminationCondition, + SolutionStatus, +) +from pyomo.contrib.solver.common.config import PersistentBranchAndBoundConfig +from pyomo.contrib.solver.solvers.gurobi_direct import ( + GurobiConfigMixin, + GurobiSolverMixin, +) +from pyomo.contrib.solver.common.util import ( + NoFeasibleSolutionError, + NoOptimalSolutionError, + NoDualsError, + NoReducedCostsError, + NoSolutionError, + IncompatibleModelError, +) +from pyomo.contrib.solver.common.persistent import ( + PersistentSolverUtils, + PersistentSolverMixin, +) +from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader +from pyomo.core.staleflag import StaleFlagManager +from .gurobi_direct_base import GurobiConfig, GurobiDirectBase, gurobipy +from pyomo.contrib.solver.common.util import get_objective +from pyomo.repn.quadratic import QuadraticRepn, QuadraticRepnVisitor + + +logger = logging.getLogger(__name__) + + +class GurobiSolutionLoader(PersistentSolutionLoader): + def load_vars(self, vars_to_load=None, solution_number=0): + self._assert_solution_still_valid() + self._solver._load_vars( + vars_to_load=vars_to_load, solution_number=solution_number + ) + + def get_primals(self, vars_to_load=None, solution_number=0): + self._assert_solution_still_valid() + return self._solver._get_primals( + vars_to_load=vars_to_load, solution_number=solution_number + ) + + +class _MutableLowerBound: + def __init__(self, var_id, expr, var_map): + self.var_id = var_id + self.expr = expr + self.var_map = var_map + + def update(self): + self.var_map[self.var_id].setAttr('lb', value(self.expr)) + + +class _MutableUpperBound: + def __init__(self, var_id, expr, var_map): + self.var_id = var_id + self.expr = expr + self.var_map = var_map + + def update(self): + self.var_map[self.var_id].setAttr('ub', value(self.expr)) + + +class _MutableLinearCoefficient: + def __init__(self): + self.expr = None + self.var = None + self.con = None + self.gurobi_model = None + + def update(self): + self.gurobi_model.chgCoeff(self.con, self.var, value(self.expr)) + + +class _MutableRangeConstant: + def __init__(self): + self.lhs_expr = None + self.rhs_expr = None + self.con = None + self.slack_name = None + self.gurobi_model = None + + def update(self): + rhs_val = value(self.rhs_expr) + lhs_val = value(self.lhs_expr) + self.con.rhs = rhs_val + slack = self.gurobi_model.getVarByName(self.slack_name) + slack.ub = rhs_val - lhs_val + + +class _MutableConstant: + def __init__(self): + self.expr = None + self.con = None + + def update(self): + self.con.rhs = value(self.expr) + + +class _MutableQuadraticConstraint: + def __init__( + self, gurobi_model, gurobi_con, constant, linear_coefs, quadratic_coefs + ): + self.con = gurobi_con + self.gurobi_model = gurobi_model + self.constant = constant + self.last_constant_value = value(self.constant.expr) + self.linear_coefs = linear_coefs + self.last_linear_coef_values = [value(i.expr) for i in self.linear_coefs] + self.quadratic_coefs = quadratic_coefs + self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] + + def get_updated_expression(self): + gurobi_expr = self.gurobi_model.getQCRow(self.con) + for ndx, coef in enumerate(self.linear_coefs): + current_coef_value = value(coef.expr) + incremental_coef_value = ( + current_coef_value - self.last_linear_coef_values[ndx] + ) + gurobi_expr += incremental_coef_value * coef.var + self.last_linear_coef_values[ndx] = current_coef_value + for ndx, coef in enumerate(self.quadratic_coefs): + current_coef_value = value(coef.expr) + incremental_coef_value = ( + current_coef_value - self.last_quadratic_coef_values[ndx] + ) + gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 + self.last_quadratic_coef_values[ndx] = current_coef_value + return gurobi_expr + + def get_updated_rhs(self): + return value(self.constant.expr) + + +class _MutableObjective: + def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): + self.gurobi_model = gurobi_model + self.constant = constant + self.linear_coefs = linear_coefs + self.quadratic_coefs = quadratic_coefs + self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] + + def get_updated_expression(self): + for ndx, coef in enumerate(self.linear_coefs): + coef.var.obj = value(coef.expr) + self.gurobi_model.ObjCon = value(self.constant.expr) + + gurobi_expr = None + for ndx, coef in enumerate(self.quadratic_coefs): + if value(coef.expr) != self.last_quadratic_coef_values[ndx]: + if gurobi_expr is None: + self.gurobi_model.update() + gurobi_expr = self.gurobi_model.getObjective() + current_coef_value = value(coef.expr) + incremental_coef_value = ( + current_coef_value - self.last_quadratic_coef_values[ndx] + ) + gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 + self.last_quadratic_coef_values[ndx] = current_coef_value + return gurobi_expr + + +class _MutableQuadraticCoefficient: + def __init__(self): + self.expr = None + self.var1 = None + self.var2 = None + + +class GurobiDirectQuadratic(GurobiDirectBase): + _minimum_version = (7, 0, 0) + + def __init__(self, **kwds): + super().__init__(**kwds) + self._solver_model = None + self._vars = {} # from id(v) to v + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + + def _create_solver_model(self, pyomo_model): + self._clear() + self._solver_model = gurobipy.Model(env=self.env()) + cons = list(pyomo_model.component_data_objects(Constraint, descend_into=True, active=True)) + self._add_constraints(cons) + sos = list(pyomo_model.component_data_objects(SOSConstraint, descend_into=True, active=True)) + self._add_sos_constraints(sos) + obj = get_objective(pyomo_model) + self._set_objective(obj) + + def _clear(self): + self._solver_model = None + self._vars = {} + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._pyomo_sos_to_solver_sos_map = {} + + def _pyomo_gurobi_var_iter(self): + for vid, v in self._vars.items(): + yield v, self._pyomo_var_to_solver_var_map[vid] + + def _process_domain_and_bounds(self, var): + lb, ub, step = var.domain.get_interval() + if lb is None: + lb = -gurobipy.GRB.INFINITY + if ub is None: + ub = gurobipy.GRB.INFINITY + if step == 0: + vtype = gurobipy.GRB.CONTINUOUS + elif step == 1: + if lb == 0 and ub == 1: + vtype = gurobipy.GRB.BINARY + else: + vtype = gurobipy.GRB.INTEGER + else: + raise ValueError( + f'Unrecognized domain: {var.domain}' + ) + if var.fixed: + lb = var.value + ub = lb + else: + lb = max(lb, value(var._lb)) + ub = min(ub, value(var._ub)) + return lb, ub, vtype + + def _add_variables(self, variables: List[VarData]): + vtypes = [] + lbs = [] + ubs = [] + for ndx, var in enumerate(variables): + self._vars[id(var)] = var + lb, ub, vtype = self._process_domain_and_bounds(var) + vtypes.append(vtype) + lbs.append(lb) + ubs.append(ub) + + gurobi_vars = self._solver_model.addVars( + len(variables), lb=lbs, ub=ubs, vtype=vtypes + ) + + for pyomo_var, gurobi_var in zip(variables, gurobi_vars): + self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var + + def _get_expr_from_pyomo_expr(self, expr): + repn = generate_standard_repn(expr, quadratic=True, compute_values=True) + + if repn.nonlinear_expr is not None: + raise IncompatibleModelError( + f'GurobiDirectQuadratic only supports linear and quadratic expressions: {expr}.' + ) + + if len(repn.linear_vars) > 0: + missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] + self._add_variables(missing_vars) + new_expr = gurobipy.LinExpr( + repn.linear_coefs, + [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars], + ) + else: + new_expr = 0.0 + + for coef, v in zip(repn.quadratic_coefs, repn.quadratic_vars): + x, y = v + gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] + gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] + new_expr += coef * gurobi_x * gurobi_y + + return new_expr, repn.constant + + def _add_constraints(self, cons: List[ConstraintData]): + gurobi_expr_list = [] + for con in cons: + lb, body, ub = con.to_bounded_expression(evaluate_bounds=True) + gurobi_expr, repn_constant = self._get_expr_from_pyomo_expr(body) + if lb is None and ub is None: + raise ValueError( + "Constraint does not have a lower " + f"or an upper bound: {con} \n" + ) + elif lb is None: + gurobi_expr_list.append(gurobi_expr <= ub - repn_constant) + elif ub is None: + gurobi_expr_list.append(lb - repn_constant <= gurobi_expr) + elif lb == ub: + gurobi_expr_list.append(gurobi_expr == lb - repn_constant) + else: + gurobi_expr_list.append(gurobi_expr == [lb-repn_constant, ub-repn_constant]) + + gurobi_cons = self._solver_model.addConstrs(gurobi_expr_list) + self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) + self._pyomo_con_to_solver_con_map[con] = gurobipy_con + self._solver_con_to_pyomo_con_map[id(gurobipy_con)] = con + self._constraints_added_since_update.update(cons) + self._needs_updated = True + + +class GurobiPersistentQuadratic(GurobiDirectQuadratic): + _minimum_version = (7, 0, 0) From db0fda4310d498ff7d225df06a035a377356b7bb Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Tue, 12 Aug 2025 07:31:39 -0600 Subject: [PATCH 038/104] Apply black --- pyomo/contrib/observer/component_collector.py | 5 +++- pyomo/contrib/observer/model_observer.py | 24 ++++++++-------- .../observer/tests/test_change_detector.py | 28 ++++++++++--------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index 5cbbdaf31bd..d52ec46086c 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -10,7 +10,10 @@ # ___________________________________________________________________________ from pyomo.core.expr.visitor import StreamBasedExpressionVisitor -from pyomo.core.expr.numeric_expr import ExternalFunctionExpression, NPV_ExternalFunctionExpression +from pyomo.core.expr.numeric_expr import ( + ExternalFunctionExpression, + NPV_ExternalFunctionExpression, +) from pyomo.core.base.var import VarData, ScalarVar from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.expression import ExpressionData, ScalarExpression diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 422eb1da574..39b832cc266 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -211,10 +211,7 @@ def update_parameters(self, params: List[ParamData]): class ModelChangeDetector: - def __init__( - self, observers: Sequence[Observer], - **kwds, - ): + def __init__(self, observers: Sequence[Observer], **kwds): """ Parameters ---------- @@ -240,13 +237,15 @@ def __init__( ) # var_id: [dict[constraints, None], dict[sos constraints, None], None or objective] self._referenced_params = ( {} - ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] + ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] self._vars_referenced_by_con = {} self._vars_referenced_by_obj = [] self._params_referenced_by_con = {} self._params_referenced_by_obj = [] self._expr_types = None - self.config: AutoUpdateConfig = AutoUpdateConfig()(value=kwds, preserve_implicit=True) + self.config: AutoUpdateConfig = AutoUpdateConfig()( + value=kwds, preserve_implicit=True + ) def set_instance(self, model): saved_config = self.config @@ -350,7 +349,10 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): if con in self._active_sos: raise ValueError(f'Constraint {con.name} has already been added') sos_items = list(con.get_items()) - self._active_sos[con] = ([i[0] for i in sos_items], [i[1] for i in sos_items]) + self._active_sos[con] = ( + [i[0] for i in sos_items], + [i[1] for i in sos_items], + ) variables = [] params = [] for v, p in sos_items: @@ -616,14 +618,14 @@ def _check_for_var_changes(self): vars_to_update.append(v) cons_to_update = list(cons_to_update.keys()) return vars_to_update, cons_to_update, update_obj - + def _check_for_param_changes(self): params_to_update = [] for pid, (p, val) in self._params.items(): if p.value != val: params_to_update.append(p) return params_to_update - + def _check_for_named_expression_changes(self): cons_to_update = [] for con, ne_list in self._named_expressions.items(): @@ -644,7 +646,7 @@ def _check_for_new_objective(self): new_obj = get_objective(self._model) if new_obj is not self._objective: update_obj = True - return new_obj, update_obj + return new_obj, update_obj def _check_for_objective_changes(self): update_obj = False @@ -717,7 +719,7 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if update_obj: need_to_set_objective = True timer.stop('named expressions') - + timer.start('objective') new_obj = self._objective if config.check_for_new_objective: diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index efda8a181d9..29e0de01eb9 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -6,7 +6,11 @@ import pyomo.environ as pe from pyomo.common import unittest from typing import List -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig +from pyomo.contrib.observer.model_observer import ( + Observer, + ModelChangeDetector, + AutoUpdateConfig, +) from pyomo.common.collections import ComponentMap import logging @@ -31,11 +35,9 @@ def __init__(self): def check(self, expected): unittest.assertStructuredAlmostEqual( - first=expected, - second=self.counts, - places=7, + first=expected, second=self.counts, places=7 ) - + def _process(self, comps, key): for c in comps: if c not in self.counts: @@ -120,7 +122,7 @@ def test_objective(self): detector.set_instance(m) obs.check(expected) - m.obj = pe.Objective(expr=m.x**2 + m.p*m.y**2) + m.obj = pe.Objective(expr=m.x**2 + m.p * m.y**2) detector.update() expected[m.obj] = make_count_dict() expected[m.obj]['set'] += 1 @@ -131,7 +133,7 @@ def test_objective(self): expected[m.p] = make_count_dict() expected[m.p]['add'] += 1 obs.check(expected) - + m.y.setlb(0) detector.update() expected[m.y]['update'] += 1 @@ -161,7 +163,7 @@ def test_objective(self): obs.check(expected) del m.obj - m.obj = pe.Objective(expr=m.p*m.x) + m.obj = pe.Objective(expr=m.p * m.x) detector.update() expected[m.p]['add'] += 1 expected[m.y]['remove'] += 1 @@ -186,7 +188,7 @@ def test_constraints(self): obs.check(expected) m.obj = pe.Objective(expr=m.y) - m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p)**2) + m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p) ** 2) detector.update() expected[m.x] = make_count_dict() expected[m.y] = make_count_dict() @@ -208,9 +210,9 @@ def test_constraints(self): obs.pprint() expected[m.c1]['remove'] += 1 expected[m.c1]['add'] += 1 - # because x and p are only used in the - # one constraint, they get removed when - # the constraint is removed and then + # because x and p are only used in the + # one constraint, they get removed when + # the constraint is removed and then # added again when the constraint is added expected[m.x]['update'] += 1 expected[m.x]['remove'] += 1 @@ -220,4 +222,4 @@ def test_constraints(self): obs.check(expected) def test_vars_and_params_elsewhere(self): - pass \ No newline at end of file + pass From 42c8cc8423ea87b2e839c8b820b6e3bb643934b2 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 09:54:25 -0600 Subject: [PATCH 039/104] adding copyright statements --- pyomo/contrib/observer/__init__.py | 10 ++++++++++ pyomo/contrib/observer/tests/__init__.py | 10 ++++++++++ pyomo/contrib/observer/tests/test_change_detector.py | 11 +++++++++++ 3 files changed, 31 insertions(+) diff --git a/pyomo/contrib/observer/__init__.py b/pyomo/contrib/observer/__init__.py index e69de29bb2d..6eb9ea8b81d 100644 --- a/pyomo/contrib/observer/__init__.py +++ b/pyomo/contrib/observer/__init__.py @@ -0,0 +1,10 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ diff --git a/pyomo/contrib/observer/tests/__init__.py b/pyomo/contrib/observer/tests/__init__.py index e69de29bb2d..6eb9ea8b81d 100644 --- a/pyomo/contrib/observer/tests/__init__.py +++ b/pyomo/contrib/observer/tests/__init__.py @@ -0,0 +1,10 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 29e0de01eb9..dd7951342da 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -1,3 +1,14 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# ___________________________________________________________________________ + from pyomo.core.base.constraint import ConstraintData from pyomo.core.base.objective import ObjectiveData from pyomo.core.base.param import ParamData From 7998fda55861cde77931d14d58b27444d9f3a501 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 10:04:55 -0600 Subject: [PATCH 040/104] refactoring gurobi interfaces --- pyomo/contrib/solver/plugins.py | 8 +- .../solvers/gurobi/gurobi_direct_base.py | 132 ++++++++++- .../solvers/gurobi/gurobi_persistent.py | 220 ++++++++++++++++-- .../solver/tests/solvers/test_solvers.py | 26 ++- 4 files changed, 350 insertions(+), 36 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 86c05f2bd70..7630c614aa2 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -13,7 +13,8 @@ from .common.factory import SolverFactory from .solvers.ipopt import Ipopt from .solvers.gurobi_persistent import GurobiPersistent -from .solvers.gurobi_direct import GurobiDirect +from .solvers.gurobi.gurobi_direct import GurobiDirect +from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic from .solvers.highs import Highs @@ -31,6 +32,11 @@ def load(): legacy_name='gurobi_direct_v2', doc='Direct (scipy-based) interface to Gurobi', )(GurobiDirect) + SolverFactory.register( + name='gurobi_direct_quadratic', + legacy_name='gurobi_direct_quadratic_v2', + doc='Direct interface to Gurobi', + )(GurobiDirect) SolverFactory.register( name='highs', legacy_name='highs', doc='Persistent interface to HiGHS' )(Highs) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index b314a39b49a..d26dbf54c83 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -78,6 +78,122 @@ def __init__( ) +def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_number): + """ + solver_model: gurobipy.Model + var_map: Dict[int, gurobipy.Var] + Maps the id of the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + solution_number: int + """ + if ( + solver_model.getAttr('NumIntVars') == 0 + and solver_model.getAttr('NumBinVars') == 0 + ): + raise ValueError( + 'Cannot obtain suboptimal solutions for a continuous model' + ) + original_solution_number = solver_model.getParamInfo('SolutionNumber')[2] + solver_model.setParam('SolutionNumber', solution_number) + gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + vals = solver_model.getAttr("Xn", gurobi_vars_to_load) + res = ComponentMap() + for var, val in zip(vars_to_load, vals): + res[var] = val + solver_model.setParam('SolutionNumber', original_solution_number) + return res + + +def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): + """ + solver_model: gurobipy.Model + var_map: Dict[int, gurobipy.Var] + Maps the id of the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + solution_number: int + """ + for v, val in _get_primals( + solver_model=solver_model, + var_map=var_map, + vars_to_load=vars_to_load, + solution_number=solution_number, + ).items(): + v.set_value(val, skip_validation=True) + StaleFlagManager.mark_all_as_stale(delayed=True) + + +def _get_primals(solver_model, var_map, vars_to_load, solution_number=0): + """ + solver_model: gurobipy.Model + var_map: Dict[int, gurobipy.Var] + Maps the id of the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + solution_number: int + """ + if solver_model.SolCount == 0: + raise NoSolutionError() + + if solution_number != 0: + return _load_suboptimal_mip_solution( + solver_model=solver_model, + var_map=var_map, + vars_to_load=vars_to_load, + solution_number=solution_number, + ) + + gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + vals = solver_model.getAttr("X", gurobi_vars_to_load) + + res = ComponentMap() + for var, val in zip(vars_to_load, vals): + res[var] = val + return res + + +def _get_reduced_costs(solver_model, var_map, vars_to_load): + """ + solver_model: gurobipy.Model + var_map: Dict[int, gurobipy.Var] + Maps the id of the pyomo variable to the gurobipy variable + vars_to_load: List[VarData] + """ + if solver_model.Status != gurobipy.GRB.OPTIMAL: + raise NoReducedCostsError() + + res = ComponentMap() + gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] + vals = solver_model.getAttr("Rc", gurobi_vars_to_load) + + for var, val in zip(vars_to_load, vals): + res[var] = val + + return res + + +def _get_duals(solver_model, con_map, linear_cons_to_load, quadratic_cons_to_load): + """ + solver_model: gurobipy.Model + con_map: Dict[ConstraintData, gurobipy.Constr] + Maps the pyomo constraint to the gurobipy constraint + linear_cons_to_load: List[ConstraintData] + quadratic_cons_to_load: List[ConstraintData] + """ + if solver_model.Status != gurobipy.GRB.OPTIMAL: + raise NoDualsError() + + linear_gurobi_cons = [con_map[c] for c in linear_cons_to_load] + quadratic_gurobi_cons = [con_map[c] for c in quadratic_cons_to_load] + linear_vals = solver_model.getAttr("Pi", linear_gurobi_cons) + quadratic_vals = solver_model.getAttr("QCPi", quadratic_gurobi_cons) + + duals = {} + for c, val in zip(linear_cons_to_load, linear_vals): + duals[c] = val + for c, val in zip(quadratic_cons_to_load, quadratic_vals): + duals[c] = val + return duals + + class GurobiDirectBase(SolverBase): _num_gurobipy_env_clients = 0 @@ -191,10 +307,15 @@ def solve(self, model, **kwds) -> Results: orig_config = self.config orig_cwd = os.getcwd() try: - self.config = config = self.config( + config = self.config( value=kwds, preserve_implicit=True, ) + + # hack to work around legacy solver wrapper __setattr__ + # otherwise, this would just be self.config = config + object.__setattr__(self, 'config', config) + if not self.available(): c = self.__class__ raise ApplicationError( @@ -237,13 +358,17 @@ def solve(self, model, **kwds) -> Results: res = self._postsolve( grb_model=gurobi_model, + solution_loader=solution_loader, has_obj=has_obj, ) finally: os.chdir(orig_cwd) + + # hack to work around legacy solver wrapper __setattr__ + # otherwise, this would just be self.config = orig_config + object.__setattr__(self, 'config', orig_config) self.config = orig_config - res.solution_loader = solution_loader res.solver_log = ostreams[0].getvalue() end_timestamp = datetime.datetime.now(datetime.timezone.utc) res.timing_info.start_timestamp = start_timestamp @@ -273,10 +398,11 @@ def _get_tc_map(self): } return GurobiDirectBase._tc_map - def _postsolve(self, grb_model, has_obj): + def _postsolve(self, grb_model, solution_loader, has_obj): status = grb_model.Status results = Results() + results.solution_loader = solution_loader results.timing_info.gurobi_time = grb_model.Runtime if grb_model.SolCount > 0: diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index fa269c1d3c5..3ae6e86526c 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -12,7 +12,7 @@ import io import logging import math -from typing import List, Optional +from typing import Dict, List, NoReturn, Optional, Sequence, Mapping from collections.abc import Iterable from pyomo.common.collections import ComponentSet, ComponentMap, OrderedSet @@ -53,9 +53,18 @@ PersistentSolverUtils, PersistentSolverMixin, ) -from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader +from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader, SolutionLoaderBase from pyomo.core.staleflag import StaleFlagManager -from .gurobi_direct_base import GurobiConfig, GurobiDirectBase, gurobipy +from .gurobi_direct_base import ( + GurobiConfig, + GurobiDirectBase, + gurobipy, + _load_suboptimal_mip_solution, + _load_vars, + _get_primals, + _get_duals, + _get_reduced_costs, +) from pyomo.contrib.solver.common.util import get_objective from pyomo.repn.quadratic import QuadraticRepn, QuadraticRepnVisitor @@ -63,7 +72,87 @@ logger = logging.getLogger(__name__) -class GurobiSolutionLoader(PersistentSolutionLoader): +class GurobiDirectQuadraticSolutionLoader(SolutionLoaderBase): + def __init__( + self, + solver_model, + var_id_map, + var_map, + con_map, + linear_cons, + quadratic_cons, + ) -> None: + super().__init__() + self._solver_model = solver_model + self._vars = var_id_map + self._var_map = var_map + self._con_map = con_map + self._linear_cons = linear_cons + self._quadratic_cons = quadratic_cons + + def load_vars( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=0, + ) -> None: + if vars_to_load is None: + vars_to_load = list(self._vars.values()) + _load_vars( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + solution_number=solution_id, + ) + + def get_primals( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=0, + ) -> Mapping[VarData, float]: + if vars_to_load is None: + vars_to_load = list(self._vars.values()) + return _get_primals( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + solution_number=solution_id, + ) + + def get_reduced_costs( + self, + vars_to_load: Optional[Sequence[VarData]] = None, + ) -> Mapping[VarData, float]: + if vars_to_load is None: + vars_to_load = list(self._vars.values()) + return _get_reduced_costs( + solver_model=self._solver_model, + var_map=self._var_map, + vars_to_load=vars_to_load, + ) + + def get_duals( + self, + cons_to_load: Optional[Sequence[ConstraintData]] = None, + ) -> Dict[ConstraintData, float]: + if cons_to_load is None: + cons_to_load = list(self._con_map.keys()) + linear_cons_to_load = [] + quadratic_cons_to_load = [] + for c in cons_to_load: + if c in self._linear_cons: + linear_cons_to_load.append(c) + else: + assert c in self._quadratic_cons + quadratic_cons_to_load.append(c) + return _get_duals( + solver_model=self._solver_model, + con_map=self._con_map, + linear_cons_to_load=linear_cons_to_load, + quadratic_cons_to_load=quadratic_cons_to_load, + ) + + +class GurobiPersistentSolutionLoader(PersistentSolutionLoader): def load_vars(self, vars_to_load=None, solution_number=0): self._assert_solution_still_valid() self._solver._load_vars( @@ -212,23 +301,50 @@ def __init__(self, **kwds): self._vars = {} # from id(v) to v self._pyomo_var_to_solver_var_map = {} self._pyomo_con_to_solver_con_map = {} + self._linear_cons = set() + self._quadratic_cons = set() self._pyomo_sos_to_solver_sos_map = {} def _create_solver_model(self, pyomo_model): + timer = self.config.timer + timer.start('create gurobipy model') self._clear() self._solver_model = gurobipy.Model(env=self.env()) + timer.start('collect constraints') cons = list(pyomo_model.component_data_objects(Constraint, descend_into=True, active=True)) + timer.stop('collect constraints') + timer.start('translate constraints') self._add_constraints(cons) + timer.stop('translate constraints') + timer.start('sos') sos = list(pyomo_model.component_data_objects(SOSConstraint, descend_into=True, active=True)) self._add_sos_constraints(sos) + timer.stop('sos') + timer.start('get objective') obj = get_objective(pyomo_model) + timer.stop('get objective') + timer.start('translate objective') self._set_objective(obj) + timer.stop('translate objective') + has_obj = obj is not None + solution_loader = GurobiDirectQuadraticSolutionLoader( + solver_model=self._solver_model, + var_id_map=self._vars, + var_map=self._pyomo_var_to_solver_var_map, + con_map=self._pyomo_con_to_solver_con_map, + linear_cons=self._linear_cons, + quadratic_cons=self._quadratic_cons, + ) + timer.stop('create gurobipy model') + return self._solver_model, solution_loader, has_obj def _clear(self): self._solver_model = None self._vars = {} self._pyomo_var_to_solver_var_map = {} self._pyomo_con_to_solver_con_map = {} + self._linear_cons = set() + self._quadratic_cons = set() self._pyomo_sos_to_solver_sos_map = {} def _pyomo_gurobi_var_iter(self): @@ -256,8 +372,10 @@ def _process_domain_and_bounds(self, var): lb = var.value ub = lb else: - lb = max(lb, value(var._lb)) - ub = min(ub, value(var._ub)) + if var._lb is not None: + lb = max(lb, value(var._lb)) + if var._ub is not None: + ub = min(ub, value(var._ub)) return lb, ub, vtype def _add_variables(self, variables: List[VarData]): @@ -273,14 +391,12 @@ def _add_variables(self, variables: List[VarData]): gurobi_vars = self._solver_model.addVars( len(variables), lb=lbs, ub=ubs, vtype=vtypes - ) + ).values() for pyomo_var, gurobi_var in zip(variables, gurobi_vars): self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var - def _get_expr_from_pyomo_expr(self, expr): - repn = generate_standard_repn(expr, quadratic=True, compute_values=True) - + def _get_expr_from_pyomo_repn(self, repn): if repn.nonlinear_expr is not None: raise IncompatibleModelError( f'GurobiDirectQuadratic only supports linear and quadratic expressions: {expr}.' @@ -289,18 +405,26 @@ def _get_expr_from_pyomo_expr(self, expr): if len(repn.linear_vars) > 0: missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] self._add_variables(missing_vars) + vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] new_expr = gurobipy.LinExpr( repn.linear_coefs, - [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars], + vlist, ) else: new_expr = 0.0 - for coef, v in zip(repn.quadratic_coefs, repn.quadratic_vars): - x, y = v - gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] - gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] - new_expr += coef * gurobi_x * gurobi_y + if len(repn.quadratic_vars) > 0: + missing_vars = {} + for x, y in repn.quadratic_vars: + for v in [x, y]: + vid = id(v) + if vid not in self._vars: + missing_vars[vid] = v + self._add_variables(list(missing_vars.values())) + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] + gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] + new_expr += coef * gurobi_x * gurobi_y return new_expr, repn.constant @@ -308,26 +432,72 @@ def _add_constraints(self, cons: List[ConstraintData]): gurobi_expr_list = [] for con in cons: lb, body, ub = con.to_bounded_expression(evaluate_bounds=True) - gurobi_expr, repn_constant = self._get_expr_from_pyomo_expr(body) + repn = generate_standard_repn(body, quadratic=True, compute_values=True) + if len(repn.quadratic_vars) > 0: + self._quadratic_cons.add(con) + else: + self._linear_cons.add(con) + gurobi_expr, repn_constant = self._get_expr_from_pyomo_repn(repn) if lb is None and ub is None: raise ValueError( "Constraint does not have a lower " f"or an upper bound: {con} \n" ) elif lb is None: - gurobi_expr_list.append(gurobi_expr <= ub - repn_constant) + gurobi_expr_list.append(gurobi_expr <= float(ub - repn_constant)) elif ub is None: - gurobi_expr_list.append(lb - repn_constant <= gurobi_expr) + gurobi_expr_list.append(float(lb - repn_constant) <= gurobi_expr) elif lb == ub: - gurobi_expr_list.append(gurobi_expr == lb - repn_constant) + gurobi_expr_list.append(gurobi_expr == float(lb - repn_constant)) else: - gurobi_expr_list.append(gurobi_expr == [lb-repn_constant, ub-repn_constant]) + gurobi_expr_list.append(gurobi_expr == [float(lb-repn_constant), float(ub-repn_constant)]) - gurobi_cons = self._solver_model.addConstrs(gurobi_expr_list) + gurobi_cons = self._solver_model.addConstrs((gurobi_expr_list[i] for i in range(len(gurobi_expr_list)))).values() self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) - self._pyomo_con_to_solver_con_map[con] = gurobipy_con - self._solver_con_to_pyomo_con_map[id(gurobipy_con)] = con - self._constraints_added_since_update.update(cons) + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + for con in cons: + level = con.level + if level == 1: + sos_type = gurobipy.GRB.SOS_TYPE1 + elif level == 2: + sos_type = gurobipy.GRB.SOS_TYPE2 + else: + raise ValueError( + f"Solver does not support SOS level {level} constraints" + ) + + gurobi_vars = [] + weights = [] + + missing_vars = {id(v): v for v, w in con.get_items() if id(v) not in self._vars} + self._add_variables(list(missing_vars.values())) + + for v, w in con.get_items(): + v_id = id(v) + gurobi_vars.append(self._pyomo_var_to_solver_var_map[v_id]) + weights.append(w) + + gurobipy_con = self._solver_model.addSOS(sos_type, gurobi_vars, weights) + self._pyomo_sos_to_solver_sos_map[con] = gurobipy_con + + def _set_objective(self, obj): + if obj is None: + sense = gurobipy.GRB.MINIMIZE + gurobi_expr = 0 + repn_constant = 0 + else: + if obj.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + elif obj.sense == maximize: + sense = gurobipy.GRB.MAXIMIZE + else: + raise ValueError(f'Objective sense is not recognized: {obj.sense}') + + repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=True) + gurobi_expr, repn_constant = self._get_expr_from_pyomo_repn(repn) + + self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) self._needs_updated = True diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 5ab36554061..748e0127151 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -30,8 +30,9 @@ from pyomo.contrib.solver.common.base import SolverBase from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.solvers.ipopt import Ipopt -from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent -from pyomo.contrib.solver.solvers.gurobi_direct import GurobiDirect +# from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent +from pyomo.contrib.solver.solvers.gurobi.gurobi_direct import GurobiDirect +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -47,20 +48,31 @@ raise unittest.SkipTest('Parameterized is not available.') all_solvers = [ - ('gurobi_persistent', GurobiPersistent), + # ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ('highs', Highs), ] mip_solvers = [ - ('gurobi_persistent', GurobiPersistent), + # ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), ] -nlp_solvers = [('ipopt', Ipopt)] -qcp_solvers = [('gurobi_persistent', GurobiPersistent), ('ipopt', Ipopt)] +nlp_solvers = [ + ('ipopt', Ipopt), +] +qcp_solvers = [ + # ('gurobi_persistent', GurobiPersistent), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), + ('ipopt', Ipopt), +] qp_solvers = qcp_solvers + [("highs", Highs)] -miqcqp_solvers = [('gurobi_persistent', GurobiPersistent)] +miqcqp_solvers = [ + # ('gurobi_persistent', GurobiPersistent), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), +] nl_solvers = [('ipopt', Ipopt)] nl_solvers_set = {i[0] for i in nl_solvers} From 909be8815c6c585a5c2c700b9444599868c26598 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:02:49 -0600 Subject: [PATCH 041/104] refactoring gurobi interfaces --- pyomo/contrib/observer/model_observer.py | 69 +- pyomo/contrib/solver/plugins.py | 3 +- .../solvers/gurobi/gurobi_persistent.py | 934 +++++++++++++++++- .../solver/tests/solvers/test_solvers.py | 11 +- 4 files changed, 925 insertions(+), 92 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 422eb1da574..8f7238c2ee9 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -162,9 +162,6 @@ def __init__( class Observer(abc.ABC): - def __init__(self): - pass - @abc.abstractmethod def add_variables(self, variables: List[VarData]): pass @@ -255,7 +252,7 @@ def set_instance(self, model): self._model = model self._add_block(model) - def _add_variables(self, variables: List[VarData]): + def add_variables(self, variables: List[VarData]): for v in variables: if id(v) in self._referenced_variables: raise ValueError(f'Variable {v.name} has already been added') @@ -271,7 +268,7 @@ def _add_variables(self, variables: List[VarData]): for obs in self._observers: obs.add_variables(variables) - def _add_parameters(self, params: List[ParamData]): + def add_parameters(self, params: List[ParamData]): for p in params: pid = id(p) if pid in self._referenced_params: @@ -287,7 +284,7 @@ def _check_for_new_vars(self, variables: List[VarData]): v_id = id(v) if v_id not in self._referenced_variables: new_vars[v_id] = v - self._add_variables(list(new_vars.values())) + self.add_variables(list(new_vars.values())) def _check_to_remove_vars(self, variables: List[VarData]): vars_to_remove = {} @@ -296,7 +293,7 @@ def _check_to_remove_vars(self, variables: List[VarData]): ref_cons, ref_sos, ref_obj = self._referenced_variables[v_id] if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: vars_to_remove[v_id] = v - self._remove_variables(list(vars_to_remove.values())) + self.remove_variables(list(vars_to_remove.values())) def _check_for_new_params(self, params: List[ParamData]): new_params = {} @@ -304,7 +301,7 @@ def _check_for_new_params(self, params: List[ParamData]): pid = id(p) if pid not in self._referenced_params: new_params[pid] = p - self._add_parameters(list(new_params.values())) + self.add_parameters(list(new_params.values())) def _check_to_remove_params(self, params: List[ParamData]): params_to_remove = {} @@ -313,9 +310,9 @@ def _check_to_remove_params(self, params: List[ParamData]): ref_cons, ref_sos, ref_obj = self._referenced_params[p_id] if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: params_to_remove[p_id] = p - self._remove_parameters(list(params_to_remove.values())) + self.remove_parameters(list(params_to_remove.values())) - def _add_constraints(self, cons: List[ConstraintData]): + def add_constraints(self, cons: List[ConstraintData]): vars_to_check = [] params_to_check = [] for con in cons: @@ -343,7 +340,7 @@ def _add_constraints(self, cons: List[ConstraintData]): for obs in self._observers: obs.add_constraints(cons) - def _add_sos_constraints(self, cons: List[SOSConstraintData]): + def add_sos_constraints(self, cons: List[SOSConstraintData]): vars_to_check = [] params_to_check = [] for con in cons: @@ -373,7 +370,7 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.add_sos_constraints(cons) - def _set_objective(self, obj: Optional[ObjectiveData]): + def set_objective(self, obj: Optional[ObjectiveData]): vars_to_remove_check = [] params_to_remove_check = [] if self._objective is not None: @@ -414,12 +411,12 @@ def _set_objective(self, obj: Optional[ObjectiveData]): self._check_to_remove_params(params_to_remove_check) def _add_block(self, block): - self._add_constraints( + self.add_constraints( list( block.component_data_objects(Constraint, descend_into=True, active=True) ) ) - self._add_sos_constraints( + self.add_sos_constraints( list( block.component_data_objects( SOSConstraint, descend_into=True, active=True @@ -427,9 +424,9 @@ def _add_block(self, block): ) ) obj = get_objective(block) - self._set_objective(obj) + self.set_objective(obj) - def _remove_constraints(self, cons: List[ConstraintData]): + def remove_constraints(self, cons: List[ConstraintData]): for obs in self._observers: obs.remove_constraints(cons) vars_to_check = [] @@ -453,7 +450,7 @@ def _remove_constraints(self, cons: List[ConstraintData]): self._check_to_remove_vars(vars_to_check) self._check_to_remove_params(params_to_check) - def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + def remove_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.remove_sos_constraints(cons) vars_to_check = [] @@ -476,7 +473,7 @@ def _remove_sos_constraints(self, cons: List[SOSConstraintData]): self._check_to_remove_vars(vars_to_check) self._check_to_remove_params(params_to_check) - def _remove_variables(self, variables: List[VarData]): + def remove_variables(self, variables: List[VarData]): for obs in self._observers: obs.remove_variables(variables) for v in variables: @@ -493,7 +490,7 @@ def _remove_variables(self, variables: List[VarData]): del self._referenced_variables[v_id] del self._vars[v_id] - def _remove_parameters(self, params: List[ParamData]): + def remove_parameters(self, params: List[ParamData]): for obs in self._observers: obs.remove_parameters(params) for p in params: @@ -510,7 +507,7 @@ def _remove_parameters(self, params: List[ParamData]): del self._referenced_params[p_id] del self._params[p_id] - def _update_variables(self, variables: List[VarData]): + def update_variables(self, variables: List[VarData]): for v in variables: self._vars[id(v)] = ( v, @@ -523,7 +520,7 @@ def _update_variables(self, variables: List[VarData]): for obs in self._observers: obs.update_variables(variables) - def _update_parameters(self, params): + def update_parameters(self, params): for p in params: self._params[id(p)] = (p, p.value) for obs in self._observers: @@ -668,28 +665,28 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if config.check_for_new_or_removed_constraints: timer.start('sos') new_sos, old_sos = self._check_for_new_or_removed_sos() - self._add_sos_constraints(new_sos) - self._remove_sos_constraints(old_sos) + self.add_sos_constraints(new_sos) + self.remove_sos_constraints(old_sos) added_sos.update(new_sos) timer.stop('sos') timer.start('cons') new_cons, old_cons = self._check_for_new_or_removed_constraints() - self._add_constraints(new_cons) - self._remove_constraints(old_cons) + self.add_constraints(new_cons) + self.remove_constraints(old_cons) added_cons.update(new_cons) timer.stop('cons') if config.update_constraints: timer.start('cons') cons_to_update = self._check_for_modified_constraints() - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + self.remove_constraints(cons_to_update) + self.add_constraints(cons_to_update) added_cons.update(cons_to_update) timer.stop('cons') timer.start('sos') sos_to_update = self._check_for_modified_sos() - self._remove_sos_constraints(sos_to_update) - self._add_sos_constraints(sos_to_update) + self.remove_sos_constraints(sos_to_update) + self.add_sos_constraints(sos_to_update) added_sos.update(sos_to_update) timer.stop('sos') @@ -698,10 +695,10 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if config.update_vars: timer.start('vars') vars_to_update, cons_to_update, update_obj = self._check_for_var_changes() - self._update_variables(vars_to_update) + self.update_variables(vars_to_update) cons_to_update = [i for i in cons_to_update if i not in added_cons] - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + self.remove_constraints(cons_to_update) + self.add_constraints(cons_to_update) added_cons.update(cons_to_update) if update_obj: need_to_set_objective = True @@ -711,8 +708,8 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): timer.start('named expressions') cons_to_update, update_obj = self._check_for_named_expression_changes() cons_to_update = [i for i in cons_to_update if i not in added_cons] - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + self.remove_constraints(cons_to_update) + self.add_constraints(cons_to_update) added_cons.update(cons_to_update) if update_obj: need_to_set_objective = True @@ -730,11 +727,11 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): need_to_set_objective = True if need_to_set_objective: - self._set_objective(new_obj) + self.set_objective(new_obj) timer.stop('objective') if config.update_parameters: timer.start('params') params_to_update = self._check_for_param_changes() - self._update_parameters(params_to_update) + self.update_parameters(params_to_update) timer.stop('params') diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 7630c614aa2..f29c4f61c4e 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -12,9 +12,8 @@ from .common.factory import SolverFactory from .solvers.ipopt import Ipopt -from .solvers.gurobi_persistent import GurobiPersistent from .solvers.gurobi.gurobi_direct import GurobiDirect -from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic +from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent from .solvers.highs import Highs diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 3ae6e86526c..844502ca476 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -9,6 +9,7 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +from __future__ import annotations import io import logging import math @@ -21,6 +22,7 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer from pyomo.common.shutdown import python_is_shutting_down +from pyomo.core.base.objective import ObjectiveData from pyomo.core.kernel.objective import minimize, maximize from pyomo.core.base import SymbolMap, NumericLabeler, TextLabeler from pyomo.core.base.var import VarData @@ -67,6 +69,7 @@ ) from pyomo.contrib.solver.common.util import get_objective from pyomo.repn.quadratic import QuadraticRepn, QuadraticRepnVisitor +from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector logger = logging.getLogger(__name__) @@ -152,18 +155,33 @@ def get_duals( ) -class GurobiPersistentSolutionLoader(PersistentSolutionLoader): - def load_vars(self, vars_to_load=None, solution_number=0): +class GurobiPersistentSolutionLoader(GurobiDirectQuadraticSolutionLoader): + def __init__(self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons) -> None: + super().__init__(solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons) + self._valid = True + + def invalidate(self): + self._valid = False + + def _assert_solution_still_valid(self): + if not self._valid: + raise RuntimeError('The results in the solver are no longer valid.') + + def load_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=0) -> None: self._assert_solution_still_valid() - self._solver._load_vars( - vars_to_load=vars_to_load, solution_number=solution_number - ) + return super().load_vars(vars_to_load, solution_id) + + def get_primals(self, vars_to_load: Sequence[VarData] | None = None, solution_id=0) -> Mapping[VarData, float]: + self._assert_solution_still_valid() + return super().get_primals(vars_to_load, solution_id) - def get_primals(self, vars_to_load=None, solution_number=0): + def get_duals(self, cons_to_load: Sequence[ConstraintData] | None = None) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() - return self._solver._get_primals( - vars_to_load=vars_to_load, solution_number=solution_number - ) + return super().get_duals(cons_to_load) + + def get_reduced_costs(self, vars_to_load: Sequence[VarData] | None = None) -> Mapping[VarData, float]: + self._assert_solution_still_valid() + return super().get_reduced_costs(vars_to_load) class _MutableLowerBound: @@ -187,46 +205,61 @@ def update(self): class _MutableLinearCoefficient: - def __init__(self): - self.expr = None - self.var = None - self.con = None - self.gurobi_model = None + def __init__(self, expr, pyomo_con, con_map, pyomo_var_id, var_map, gurobi_model): + self.expr = expr + self.pyomo_con = pyomo_con + self.pyomo_var_id = pyomo_var_id + self.con_map = con_map + self.var_map = var_map + self.gurobi_model = gurobi_model + + @property + def gurobi_var(self): + return self.var_map[self.pyomo_var_id] + + @property + def gurobi_con(self): + return self.con_map[self.pyomo_con] def update(self): - self.gurobi_model.chgCoeff(self.con, self.var, value(self.expr)) + self.gurobi_model.chgCoeff(self.gurobi_con, self.gurobi_var, value(self.expr)) class _MutableRangeConstant: - def __init__(self): - self.lhs_expr = None - self.rhs_expr = None - self.con = None - self.slack_name = None - self.gurobi_model = None + def __init__(self, lhs_expr, rhs_expr, pyomo_con, con_map, slack_name, gurobi_model): + self.lhs_expr = lhs_expr + self.rhs_expr = rhs_expr + self.pyomo_con = pyomo_con + self.con_map = con_map + self.slack_name = slack_name + self.gurobi_model = gurobi_model def update(self): rhs_val = value(self.rhs_expr) lhs_val = value(self.lhs_expr) - self.con.rhs = rhs_val + con = self.con_map[self.pyomo_con] + con.rhs = rhs_val slack = self.gurobi_model.getVarByName(self.slack_name) slack.ub = rhs_val - lhs_val class _MutableConstant: - def __init__(self): - self.expr = None - self.con = None + def __init__(self, expr, pyomo_con, con_map): + self.expr = expr + self.pyomo_con = pyomo_con + self.con_map = con_map def update(self): - self.con.rhs = value(self.expr) + con = self.con_map[self.pyomo_con] + con.rhs = value(self.expr) class _MutableQuadraticConstraint: def __init__( - self, gurobi_model, gurobi_con, constant, linear_coefs, quadratic_coefs + self, gurobi_model, pyomo_con, con_map, constant, linear_coefs, quadratic_coefs ): - self.con = gurobi_con + self.pyomo_con = pyomo_con + self.con_map = con_map self.gurobi_model = gurobi_model self.constant = constant self.last_constant_value = value(self.constant.expr) @@ -235,8 +268,12 @@ def __init__( self.quadratic_coefs = quadratic_coefs self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] + @property + def gurobi_con(self): + return self.con_map[self.pyomo_con] + def get_updated_expression(self): - gurobi_expr = self.gurobi_model.getQCRow(self.con) + gurobi_expr = self.gurobi_model.getQCRow(self.gurobi_con) for ndx, coef in enumerate(self.linear_coefs): current_coef_value = value(coef.expr) incremental_coef_value = ( @@ -260,14 +297,14 @@ def get_updated_rhs(self): class _MutableObjective: def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): self.gurobi_model = gurobi_model - self.constant = constant - self.linear_coefs = linear_coefs - self.quadratic_coefs = quadratic_coefs - self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] + self.constant: _MutableConstant = constant + self.linear_coefs: List[_MutableLinearCoefficient] = linear_coefs + self.quadratic_coefs: List[_MutableQuadraticCoefficient] = quadratic_coefs + self.last_quadratic_coef_values: List[float] = [value(i.expr) for i in self.quadratic_coefs] def get_updated_expression(self): for ndx, coef in enumerate(self.linear_coefs): - coef.var.obj = value(coef.expr) + coef.gurobi_var.obj = value(coef.expr) self.gurobi_model.ObjCon = value(self.constant.expr) gurobi_expr = None @@ -286,10 +323,19 @@ def get_updated_expression(self): class _MutableQuadraticCoefficient: - def __init__(self): + def __init__(self, expr, v1id, v2id, var_map): self.expr = None - self.var1 = None - self.var2 = None + self.var_map = var_map + self.v1id = v1id + self.v2id = v2id + + @property + def var1(self): + return self.var_map[self.v1id] + + @property + def var2(self): + return self.var_map[self.v2id] class GurobiDirectQuadratic(GurobiDirectBase): @@ -405,9 +451,10 @@ def _get_expr_from_pyomo_repn(self, repn): if len(repn.linear_vars) > 0: missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] self._add_variables(missing_vars) + coef_list = [value(i) for i in repn.linear_coefs] vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] new_expr = gurobipy.LinExpr( - repn.linear_coefs, + coef_list, vlist, ) else: @@ -424,9 +471,9 @@ def _get_expr_from_pyomo_repn(self, repn): for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] - new_expr += coef * gurobi_x * gurobi_y + new_expr += value(coef) * gurobi_x * gurobi_y - return new_expr, repn.constant + return new_expr def _add_constraints(self, cons: List[ConstraintData]): gurobi_expr_list = [] @@ -437,20 +484,20 @@ def _add_constraints(self, cons: List[ConstraintData]): self._quadratic_cons.add(con) else: self._linear_cons.add(con) - gurobi_expr, repn_constant = self._get_expr_from_pyomo_repn(repn) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) if lb is None and ub is None: raise ValueError( "Constraint does not have a lower " f"or an upper bound: {con} \n" ) elif lb is None: - gurobi_expr_list.append(gurobi_expr <= float(ub - repn_constant)) + gurobi_expr_list.append(gurobi_expr <= float(ub - repn.constant)) elif ub is None: - gurobi_expr_list.append(float(lb - repn_constant) <= gurobi_expr) + gurobi_expr_list.append(float(lb - repn.constant) <= gurobi_expr) elif lb == ub: - gurobi_expr_list.append(gurobi_expr == float(lb - repn_constant)) + gurobi_expr_list.append(gurobi_expr == float(lb - repn.constant)) else: - gurobi_expr_list.append(gurobi_expr == [float(lb-repn_constant), float(ub-repn_constant)]) + gurobi_expr_list.append(gurobi_expr == [float(lb-repn.constant), float(ub-repn.constant)]) gurobi_cons = self._solver_model.addConstrs((gurobi_expr_list[i] for i in range(len(gurobi_expr_list)))).values() self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) @@ -495,11 +542,802 @@ def _set_objective(self, obj): raise ValueError(f'Objective sense is not recognized: {obj.sense}') repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=True) - gurobi_expr, repn_constant = self._get_expr_from_pyomo_repn(repn) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + repn_constant = repn.constant self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) - self._needs_updated = True -class GurobiPersistentQuadratic(GurobiDirectQuadratic): +class _GurobiObserver(Observer): + def __init__(self, opt: GurobiPersistentQuadratic) -> None: + self.opt = opt + + def add_variables(self, variables: List[VarData]): + self.opt._add_variables(variables) + + def add_parameters(self, params: List[ParamData]): + pass + + def add_constraints(self, cons: List[ConstraintData]): + self.opt._add_constraints(cons) + + def add_sos_constraints(self, cons: List[SOSConstraintData]): + self.opt._add_sos_constraints(cons) + + def set_objective(self, obj: ObjectiveData | None): + self.opt._set_objective(obj) + + def remove_constraints(self, cons: List[ConstraintData]): + self.opt._remove_constraints(cons) + + def remove_sos_constraints(self, cons: List[SOSConstraintData]): + self.opt._remove_sos_constraints(cons) + + def remove_variables(self, variables: List[VarData]): + self.opt._remove_variables(variables) + + def remove_parameters(self, params: List[ParamData]): + pass + + def update_variables(self, variables: List[VarData]): + self.opt._update_variables(variables) + + def update_parameters(self, params: List[ParamData]): + self.opt._update_parameters(params) + + +class GurobiPersistent(GurobiDirectQuadratic): _minimum_version = (7, 0, 0) + + def __init__(self, **kwds): + super().__init__(**kwds) + self._pyomo_model = None + self._objective = None + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True + self._callback_func = None + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._last_results_object: Optional[Results] = None + self._observer = _GurobiObserver(self) + self._change_detector = ModelChangeDetector(observers=[self._observer]) + self._constraint_ndx = 0 + + @property + def auto_updates(self): + return self._change_detector.config + + def _clear(self): + super()._clear() + self._pyomo_model = None + self._objective = None + self._mutable_helpers = {} + self._mutable_bounds = {} + self._mutable_quadratic_helpers = {} + self._mutable_objective = None + self._needs_updated = True + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._last_results_object = None + self._constraint_ndx = 0 + + def _create_solver_model(self, pyomo_model): + if pyomo_model is self._pyomo_model: + self.update() + else: + self.set_instance(pyomo_model) + + solution_loader = GurobiPersistentSolutionLoader( + solver_model=self._solver_model, + var_id_map=self._vars, + var_map=self._pyomo_var_to_solver_var_map, + con_map=self._pyomo_con_to_solver_con_map, + linear_cons=self._linear_cons, + quadratic_cons=self._quadratic_cons, + ) + has_obj = self._objective is not None + return self._solver_model, solution_loader, has_obj + + def release_license(self): + self._clear() + self.__class__.release_license() + + def solve(self, model, **kwds) -> Results: + res = super().solve(model, **kwds) + self._needs_updated = False + return res + + def _process_domain_and_bounds(self, var): + res = super()._process_domain_and_bounds(var) + if not is_constant(var._lb): + mutable_lb = _MutableLowerBound(id(var), var.lower, self._pyomo_var_to_solver_var_map) + self._mutable_bounds[id(var), 'lb'] = (var, mutable_lb) + if not is_constant(var._ub): + mutable_ub = _MutableUpperBound(id(var), var.upper, self._pyomo_var_to_solver_var_map) + self._mutable_bounds[id(var), 'ub'] = (var, mutable_ub) + return res + + def _add_variables(self, variables: List[VarData]): + self._invalidate_last_results() + super()._add_variables(variables) + self._vars_added_since_update.update(variables) + self._needs_updated = True + + def set_instance(self, pyomo_model): + if self.config.timer is None: + timer = HierarchicalTimer() + else: + timer = self.config.timer + self._clear() + self._pyomo_model = pyomo_model + self._solver_model = gurobipy.Model(env=self.env()) + timer.start('set_instance') + self._change_detector.set_instance(pyomo_model) + timer.stop('set_instance') + + def update(self): + if self.config.timer is None: + timer = HierarchicalTimer() + else: + timer = self.config.timer + if self._pyomo_model is None: + raise RuntimeError('must call set_instance or solve before update') + timer.start('update') + if self._needs_updated: + self._update_gurobi_model() + self._change_detector.update(timer=timer) + timer.stop('update') + + def _add_constraints(self, cons: List[ConstraintData]): + self._invalidate_last_results() + gurobi_expr_list = [] + for ndx, con in enumerate(cons): + lb, body, ub = con.to_bounded_expression(evaluate_bounds=False) + repn = generate_standard_repn(body, quadratic=True, compute_values=False) + if len(repn.quadratic_vars) > 0: + self._quadratic_cons.add(con) + else: + self._linear_cons.add(con) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + mutable_constant = None + if lb is None and ub is None: + raise ValueError( + "Constraint does not have a lower " + f"or an upper bound: {con} \n" + ) + elif lb is None: + rhs_expr = ub - repn.constant + gurobi_expr_list.append(gurobi_expr <= float(value(rhs_expr))) + if not is_constant(rhs_expr): + mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + elif ub is None: + rhs_expr = lb - repn.constant + gurobi_expr_list.append(float(value(rhs_expr)) <= gurobi_expr) + if not is_constant(rhs_expr): + mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + elif con.equality: + rhs_expr = lb - repn.constant + gurobi_expr_list.append(gurobi_expr == float(value(rhs_expr))) + if not is_constant(rhs_expr): + mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + else: + assert len(repn.quadratic_vars) == 0, "Quadratic range constraints are not supported" + lhs_expr = lb - repn.constant + rhs_expr = ub - repn.constant + gurobi_expr_list.append(gurobi_expr == [float(value(lhs_expr)), float(value(rhs_expr))]) + if not is_constant(lhs_expr) or not is_constant(rhs_expr): + conname = f'c{self._constraint_ndx}[{ndx}]' + mutable_constant = _MutableRangeConstant(lhs_expr, rhs_expr, con, self._pyomo_con_to_solver_con_map, 'Rg' + conname, self._solver_model) + + mlc_list = [] + for c, v in zip(repn.linear_coefs, repn.linear_vars): + if not is_constant(c): + mlc = _MutableLinearCoefficient(c, con, self._pyomo_con_to_solver_con_map, id(v), self._pyomo_var_to_solver_var_map, self._solver_model) + mlc_list.append(mlc) + + if len(repn.quadratic_vars) == 0: + if len(mlc_list) > 0: + self._mutable_helpers[con] = mlc_list + if mutable_constant is not None: + if con not in self._mutable_helpers: + self._mutable_helpers[con] = [] + self._mutable_helpers[con].append(mutable_constant) + else: + if mutable_constant is None: + mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mqc_list = [] + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + if not is_constant(coef): + mqc = _MutableQuadraticCoefficient(coef, id(x), id(y), self._pyomo_var_to_solver_var_map) + mqc_list.append(mqc) + mqc = _MutableQuadraticConstraint( + self._solver_model, + con, + self._pyomo_con_to_solver_con_map, + mutable_constant, + mlc_list, + mqc_list, + ) + self._mutable_quadratic_helpers[con] = mqc + + gurobi_cons = list(self._solver_model.addConstrs( + (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))), + name=f'c{self._constraint_ndx}' + ).values()) + self._constraint_ndx += 1 + self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) + self._constraints_added_since_update.update(cons) + self._needs_updated = True + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + self._invalidate_last_results() + super()._add_sos_constraints(cons) + self._constraints_added_since_update.update(cons) + self._needs_updated = True + + def _set_objective(self, obj): + self._invalidate_last_results() + if obj is None: + sense = gurobipy.GRB.MINIMIZE + gurobi_expr = 0 + repn_constant = 0 + self._mutable_objective = None + else: + if obj.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + elif obj.sense == maximize: + sense = gurobipy.GRB.MAXIMIZE + else: + raise ValueError(f'Objective sense is not recognized: {obj.sense}') + + repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=False) + repn_constant = value(repn.constant) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + + mutable_constant = _MutableConstant(repn.constant, None, None) + + mlc_list = [] + for c, v in zip(repn.linear_coefs, repn.linear_vars): + if not is_constant(c): + mlc = _MutableLinearCoefficient(c, None, None, id(v), self._pyomo_var_to_solver_var_map, self._solver_model) + mlc_list.append(mlc) + + mqc_list = [] + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + if not is_constant(coef): + mqc = _MutableQuadraticCoefficient(coef, id(x), id(y), self._pyomo_var_to_solver_var_map) + mqc_list.append(mqc) + + self._mutable_objective = _MutableObjective(self._solver_model, mutable_constant, mlc_list, mqc_list) + + # hack + # see PR #2454 + if self._objective is not None: + self._solver_model.setObjective(0) + self._solver_model.update() + + self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) + self._objective = obj + self._needs_updated = True + + def _update_gurobi_model(self): + self._solver_model.update() + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._needs_updated = False + + def _remove_constraints(self, cons: List[ConstraintData]): + self._invalidate_last_results() + for con in cons: + if con in self._constraints_added_since_update: + self._update_gurobi_model() + solver_con = self._pyomo_con_to_solver_con_map[con] + self._solver_model.remove(solver_con) + del self._pyomo_con_to_solver_con_map[con] + self._mutable_helpers.pop(con, None) + self._mutable_quadratic_helpers.pop(con, None) + self._needs_updated = True + + def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + self._invalidate_last_results() + for con in cons: + if con in self._constraints_added_since_update: + self._update_gurobi_model() + solver_sos_con = self._pyomo_sos_to_solver_sos_map[con] + self._solver_model.remove(solver_sos_con) + del self._pyomo_sos_to_solver_sos_map[con] + self._needs_updated = True + + def _remove_variables(self, variables: List[VarData]): + self._invalidate_last_results() + for var in variables: + v_id = id(var) + if var in self._vars_added_since_update: + self._update_gurobi_model() + solver_var = self._pyomo_var_to_solver_var_map[v_id] + self._solver_model.remove(solver_var) + del self._pyomo_var_to_solver_var_map[v_id] + self._mutable_bounds.pop(v_id, None) + self._needs_updated = True + + def _update_variables(self, variables: List[VarData]): + self._invalidate_last_results() + for var in variables: + var_id = id(var) + if var_id not in self._pyomo_var_to_solver_var_map: + raise ValueError( + f'The Var provided to update_var needs to be added first: {var}' + ) + self._mutable_bounds.pop((var_id, 'lb'), None) + self._mutable_bounds.pop((var_id, 'ub'), None) + gurobipy_var = self._pyomo_var_to_solver_var_map[var_id] + lb, ub, vtype = self._process_domain_and_bounds(var) + gurobipy_var.setAttr('lb', lb) + gurobipy_var.setAttr('ub', ub) + gurobipy_var.setAttr('vtype', vtype) + self._needs_updated = True + + def _update_parameters(self, params: List[ParamData]): + self._invalidate_last_results() + for con, helpers in self._mutable_helpers.items(): + for helper in helpers: + helper.update() + for k, (v, helper) in self._mutable_bounds.items(): + helper.update() + + for con, helper in self._mutable_quadratic_helpers.items(): + if con in self._constraints_added_since_update: + self._update_gurobi_model() + gurobi_con = helper.gurobi_con + new_gurobi_expr = helper.get_updated_expression() + new_rhs = helper.get_updated_rhs() + new_sense = gurobi_con.qcsense + self._solver_model.remove(gurobi_con) + new_con = self._solver_model.addQConstr( + new_gurobi_expr, new_sense, new_rhs, + ) + self._pyomo_con_to_solver_con_map[con] = new_con + helper.pyomo_con = con + self._constraints_added_since_update.add(con) + + if self._mutable_objective is not None: + new_gurobi_expr = self._mutable_objective.get_updated_expression() + if new_gurobi_expr is not None: + if self._objective.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + else: + sense = gurobipy.GRB.MAXIMIZE + # TODO: need a test for when part of the object is linear + # and part of the objective is quadratic, but both + # parts have mutable coefficients + self._solver_model.setObjective(new_gurobi_expr, sense=sense) + + def _invalidate_last_results(self): + if self._last_results_object is not None: + self._last_results_object.solution_loader.invalidate() + + def get_model_attr(self, attr): + """ + Get the value of an attribute on the Gurobi model. + + Parameters + ---------- + attr: str + The attribute to get. See Gurobi documentation for descriptions of the attributes. + """ + if self._needs_updated: + self._update_gurobi_model() + return self._solver_model.getAttr(attr) + + def write(self, filename): + """ + Write the model to a file (e.g., and lp file). + + Parameters + ---------- + filename: str + Name of the file to which the model should be written. + """ + self._solver_model.write(filename) + self._constraints_added_since_update = OrderedSet() + self._vars_added_since_update = ComponentSet() + self._needs_updated = False + + def set_linear_constraint_attr(self, con, attr, val): + """ + Set the value of an attribute on a gurobi linear constraint. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The pyomo constraint for which the corresponding gurobi constraint attribute + should be modified. + attr: str + The attribute to be modified. Options are: + CBasis + DStart + Lazy + val: any + See gurobi documentation for acceptable values. + """ + if attr in {'Sense', 'RHS', 'ConstrName'}: + raise ValueError( + f'Linear constraint attr {attr} cannot be set with' + ' the set_linear_constraint_attr method. Please use' + ' the remove_constraint and add_constraint methods.' + ) + self._pyomo_con_to_solver_con_map[con].setAttr(attr, val) + self._needs_updated = True + + def set_var_attr(self, var, attr, val): + """ + Set the value of an attribute on a gurobi variable. + + Parameters + ---------- + var: pyomo.core.base.var.VarData + The pyomo var for which the corresponding gurobi var attribute + should be modified. + attr: str + The attribute to be modified. Options are: + Start + VarHintVal + VarHintPri + BranchPriority + VBasis + PStart + val: any + See gurobi documentation for acceptable values. + """ + if attr in {'LB', 'UB', 'VType', 'VarName'}: + raise ValueError( + f'Var attr {attr} cannot be set with' + ' the set_var_attr method. Please use' + ' the update_var method.' + ) + if attr == 'Obj': + raise ValueError( + 'Var attr Obj cannot be set with' + ' the set_var_attr method. Please use' + ' the set_objective method.' + ) + self._pyomo_var_to_solver_var_map[id(var)].setAttr(attr, val) + self._needs_updated = True + + def get_var_attr(self, var, attr): + """ + Get the value of an attribute on a gurobi var. + + Parameters + ---------- + var: pyomo.core.base.var.VarData + The pyomo var for which the corresponding gurobi var attribute + should be retrieved. + attr: str + The attribute to get. See gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_var_to_solver_var_map[id(var)].getAttr(attr) + + def get_linear_constraint_attr(self, con, attr): + """ + Get the value of an attribute on a gurobi linear constraint. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The pyomo constraint for which the corresponding gurobi constraint attribute + should be retrieved. + attr: str + The attribute to get. See the Gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_con_to_solver_con_map[con].getAttr(attr) + + def get_sos_attr(self, con, attr): + """ + Get the value of an attribute on a gurobi sos constraint. + + Parameters + ---------- + con: pyomo.core.base.sos.SOSConstraintData + The pyomo SOS constraint for which the corresponding gurobi SOS constraint attribute + should be retrieved. + attr: str + The attribute to get. See the Gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_sos_to_solver_sos_map[con].getAttr(attr) + + def get_quadratic_constraint_attr(self, con, attr): + """ + Get the value of an attribute on a gurobi quadratic constraint. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The pyomo constraint for which the corresponding gurobi constraint attribute + should be retrieved. + attr: str + The attribute to get. See the Gurobi documentation + """ + if self._needs_updated: + self._update_gurobi_model() + return self._pyomo_con_to_solver_con_map[con].getAttr(attr) + + def set_gurobi_param(self, param, val): + """ + Set a gurobi parameter. + + Parameters + ---------- + param: str + The gurobi parameter to set. Options include any gurobi parameter. + Please see the Gurobi documentation for options. + val: any + The value to set the parameter to. See Gurobi documentation for possible values. + """ + self._solver_model.setParam(param, val) + + def get_gurobi_param_info(self, param): + """ + Get information about a gurobi parameter. + + Parameters + ---------- + param: str + The gurobi parameter to get info for. See Gurobi documentation for possible options. + + Returns + ------- + six-tuple containing the parameter name, type, value, minimum value, maximum value, and default value. + """ + return self._solver_model.getParamInfo(param) + + def _intermediate_callback(self): + def f(gurobi_model, where): + self._callback_func(self._pyomo_model, self, where) + + return f + + def set_callback(self, func=None): + """ + Specify a callback for gurobi to use. + + Parameters + ---------- + func: function + The function to call. The function should have three arguments. The first will be the pyomo model being + solved. The second will be the GurobiPersistent instance. The third will be an enum member of + gurobipy.GRB.Callback. This will indicate where in the branch and bound algorithm gurobi is at. For + example, suppose we want to solve + + .. math:: + + min 2*x + y + + s.t. + + y >= (x-2)**2 + + 0 <= x <= 4 + + y >= 0 + + y integer + + as an MILP using extended cutting planes in callbacks. + + >>> from gurobipy import GRB # doctest:+SKIP + >>> import pyomo.environ as pyo + >>> from pyomo.core.expr.taylor_series import taylor_series_expansion + >>> from pyomo.contrib import appsi + >>> + >>> m = pyo.ConcreteModel() + >>> m.x = pyo.Var(bounds=(0, 4)) + >>> m.y = pyo.Var(within=pyo.Integers, bounds=(0, None)) + >>> m.obj = pyo.Objective(expr=2*m.x + m.y) + >>> m.cons = pyo.ConstraintList() # for the cutting planes + >>> + >>> def _add_cut(xval): + ... # a function to generate the cut + ... m.x.value = xval + ... return m.cons.add(m.y >= taylor_series_expansion((m.x - 2)**2)) + ... + >>> _c = _add_cut(0) # start with 2 cuts at the bounds of x + >>> _c = _add_cut(4) # this is an arbitrary choice + >>> + >>> opt = appsi.solvers.Gurobi() + >>> opt.config.stream_solver = True + >>> opt.set_instance(m) # doctest:+SKIP + >>> opt.gurobi_options['PreCrush'] = 1 + >>> opt.gurobi_options['LazyConstraints'] = 1 + >>> + >>> def my_callback(cb_m, cb_opt, cb_where): + ... if cb_where == GRB.Callback.MIPSOL: + ... cb_opt.cbGetSolution(variables=[m.x, m.y]) + ... if m.y.value < (m.x.value - 2)**2 - 1e-6: + ... cb_opt.cbLazy(_add_cut(m.x.value)) + ... + >>> opt.set_callback(my_callback) + >>> res = opt.solve(m) # doctest:+SKIP + + """ + if func is not None: + self._callback_func = func + self._callback = self._intermediate_callback() + else: + self._callback = None + self._callback_func = None + + def cbCut(self, con): + """ + Add a cut within a callback. + + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The cut to add + """ + if not con.active: + raise ValueError('cbCut expected an active constraint.') + + if is_fixed(con.body): + raise ValueError('cbCut expected a non-trivial constraint') + + repn = generate_standard_repn(con.body, quadratic=True, compute_values=True) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + + if con.has_lb(): + if con.has_ub(): + raise ValueError('Range constraints are not supported in cbCut.') + if not is_fixed(con.lower): + raise ValueError(f'Lower bound of constraint {con} is not constant.') + if con.has_ub(): + if not is_fixed(con.upper): + raise ValueError(f'Upper bound of constraint {con} is not constant.') + + if con.equality: + self._solver_model.cbCut( + lhs=gurobi_expr, + sense=gurobipy.GRB.EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_lb() and (value(con.lower) > -float('inf')): + self._solver_model.cbCut( + lhs=gurobi_expr, + sense=gurobipy.GRB.GREATER_EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_ub() and (value(con.upper) < float('inf')): + self._solver_model.cbCut( + lhs=gurobi_expr, + sense=gurobipy.GRB.LESS_EQUAL, + rhs=value(con.upper - repn.constant), + ) + else: + raise ValueError( + f'Constraint does not have a lower or an upper bound {con} \n' + ) + + def cbGet(self, what): + return self._solver_model.cbGet(what) + + def cbGetNodeRel(self, variables): + """ + Parameters + ---------- + variables: Var or iterable of Var + """ + if not isinstance(variables, Iterable): + variables = [variables] + gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] + var_values = self._solver_model.cbGetNodeRel(gurobi_vars) + for i, v in enumerate(variables): + v.set_value(var_values[i], skip_validation=True) + + def cbGetSolution(self, variables): + """ + Parameters + ---------- + variables: iterable of vars + """ + if not isinstance(variables, Iterable): + variables = [variables] + gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] + var_values = self._solver_model.cbGetSolution(gurobi_vars) + for i, v in enumerate(variables): + v.set_value(var_values[i], skip_validation=True) + + def cbLazy(self, con): + """ + Parameters + ---------- + con: pyomo.core.base.constraint.ConstraintData + The lazy constraint to add + """ + if not con.active: + raise ValueError('cbLazy expected an active constraint.') + + if is_fixed(con.body): + raise ValueError('cbLazy expected a non-trivial constraint') + + repn = generate_standard_repn(con.body, quadratic=True, compute_values=True) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) + + if con.has_lb(): + if con.has_ub(): + raise ValueError('Range constraints are not supported in cbLazy.') + if not is_fixed(con.lower): + raise ValueError(f'Lower bound of constraint {con} is not constant.') + if con.has_ub(): + if not is_fixed(con.upper): + raise ValueError(f'Upper bound of constraint {con} is not constant.') + + if con.equality: + self._solver_model.cbLazy( + lhs=gurobi_expr, + sense=gurobipy.GRB.EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_lb() and (value(con.lower) > -float('inf')): + self._solver_model.cbLazy( + lhs=gurobi_expr, + sense=gurobipy.GRB.GREATER_EQUAL, + rhs=value(con.lower - repn.constant), + ) + elif con.has_ub() and (value(con.upper) < float('inf')): + self._solver_model.cbLazy( + lhs=gurobi_expr, + sense=gurobipy.GRB.LESS_EQUAL, + rhs=value(con.upper - repn.constant), + ) + else: + raise ValueError( + f'Constraint does not have a lower or an upper bound {con} \n' + ) + + def cbSetSolution(self, variables, solution): + if not isinstance(variables, Iterable): + variables = [variables] + gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] + self._solver_model.cbSetSolution(gurobi_vars, solution) + + def cbUseSolution(self): + return self._solver_model.cbUseSolution() + + def reset(self): + self._solver_model.reset() + + def add_variables(self, variables): + self._change_detector.add_variables(variables) + + def add_constraints(self, cons): + self._change_detector.add_constraints(cons) + + def add_sos_constraints(self, cons): + self._change_detector.add_sos_constraints(cons) + + def set_objective(self, obj): + self._change_detector.set_objective(obj) + + def remove_constrains(self, cons): + self._change_detector.remove_constraints(cons) + + def remove_sos_constraints(self, cons): + self._change_detector.remove_sos_constraints(cons) + + def remove_variables(self, variables): + self._change_detector.remove_variables(variables) + + def update_variables(self, variables): + self._change_detector.update_variables(variables) + + def update_parameters(self, params): + self._change_detector.update_parameters(params) \ No newline at end of file diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 748e0127151..a0d87835e13 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -30,9 +30,8 @@ from pyomo.contrib.solver.common.base import SolverBase from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.solvers.ipopt import Ipopt -# from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent from pyomo.contrib.solver.solvers.gurobi.gurobi_direct import GurobiDirect -from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -48,14 +47,14 @@ raise unittest.SkipTest('Parameterized is not available.') all_solvers = [ - # ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ('highs', Highs), ] mip_solvers = [ - # ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), @@ -64,13 +63,13 @@ ('ipopt', Ipopt), ] qcp_solvers = [ - # ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ] qp_solvers = qcp_solvers + [("highs", Highs)] miqcqp_solvers = [ - # ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ] nl_solvers = [('ipopt', Ipopt)] From 862c387a8e6478d9b9c3f176b0059046d02f1198 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:18:04 -0600 Subject: [PATCH 042/104] bugs --- pyomo/contrib/solver/plugins.py | 4 ++-- pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index f29c4f61c4e..fed739232ad 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -35,7 +35,7 @@ def load(): name='gurobi_direct_quadratic', legacy_name='gurobi_direct_quadratic_v2', doc='Direct interface to Gurobi', - )(GurobiDirect) + )(GurobiDirectQuadratic) SolverFactory.register( - name='highs', legacy_name='highs', doc='Persistent interface to HiGHS' + name='highs', legacy_name='highs_v2', doc='Persistent interface to HiGHS' )(Highs) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 844502ca476..b8a8f46d1f6 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -279,7 +279,7 @@ def get_updated_expression(self): incremental_coef_value = ( current_coef_value - self.last_linear_coef_values[ndx] ) - gurobi_expr += incremental_coef_value * coef.var + gurobi_expr += incremental_coef_value * coef.gurobi_var self.last_linear_coef_values[ndx] = current_coef_value for ndx, coef in enumerate(self.quadratic_coefs): current_coef_value = value(coef.expr) @@ -324,7 +324,7 @@ def get_updated_expression(self): class _MutableQuadraticCoefficient: def __init__(self, expr, v1id, v2id, var_map): - self.expr = None + self.expr = expr self.var_map = var_map self.v1id = v1id self.v2id = v2id @@ -860,6 +860,7 @@ def _remove_variables(self, variables: List[VarData]): solver_var = self._pyomo_var_to_solver_var_map[v_id] self._solver_model.remove(solver_var) del self._pyomo_var_to_solver_var_map[v_id] + del self._vars[v_id] self._mutable_bounds.pop(v_id, None) self._needs_updated = True From 8f7a61ed3bf8eafc8eee12544755f61097a2b7be Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:35:33 -0600 Subject: [PATCH 043/104] refactoring gurobi interfaces --- pyomo/contrib/observer/model_observer.py | 7 +- .../solvers/gurobi/gurobi_persistent.py | 6 +- pyomo/contrib/solver/solvers/gurobi_direct.py | 470 ------ .../solver/solvers/gurobi_persistent.py | 1409 ----------------- .../tests/solvers/test_gurobi_persistent.py | 49 +- 5 files changed, 25 insertions(+), 1916 deletions(-) delete mode 100644 pyomo/contrib/solver/solvers/gurobi_direct.py delete mode 100644 pyomo/contrib/solver/solvers/gurobi_persistent.py diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 8f7238c2ee9..bd905e1c61d 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -361,12 +361,15 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): self._named_expressions[con] = [] self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) + for con in cons: + variables = self._vars_referenced_by_con[con] + params = self._params_referenced_by_con[con] for v in variables: self._referenced_variables[id(v)][1][con] = None for p in params: self._referenced_params[id(p)][1][con] = None - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) for obs in self._observers: obs.add_sos_constraints(cons) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index b8a8f46d1f6..e91381f41a3 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -39,10 +39,6 @@ SolutionStatus, ) from pyomo.contrib.solver.common.config import PersistentBranchAndBoundConfig -from pyomo.contrib.solver.solvers.gurobi_direct import ( - GurobiConfigMixin, - GurobiSolverMixin, -) from pyomo.contrib.solver.common.util import ( NoFeasibleSolutionError, NoOptimalSolutionError, @@ -1328,7 +1324,7 @@ def add_sos_constraints(self, cons): def set_objective(self, obj): self._change_detector.set_objective(obj) - def remove_constrains(self, cons): + def remove_constraints(self, cons): self._change_detector.remove_constraints(cons) def remove_sos_constraints(self, cons): diff --git a/pyomo/contrib/solver/solvers/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi_direct.py deleted file mode 100644 index 45ea9dcc873..00000000000 --- a/pyomo/contrib/solver/solvers/gurobi_direct.py +++ /dev/null @@ -1,470 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ - -import datetime -import io -import math -import operator -import os - -from pyomo.common.collections import ComponentMap, ComponentSet -from pyomo.common.config import ConfigValue -from pyomo.common.dependencies import attempt_import -from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import MouseTrap, ApplicationError -from pyomo.common.shutdown import python_is_shutting_down -from pyomo.common.tee import capture_output, TeeStream -from pyomo.common.timing import HierarchicalTimer -from pyomo.core.staleflag import StaleFlagManager -from pyomo.repn.plugins.standard_form import LinearStandardFormCompiler - -from pyomo.contrib.solver.common.base import SolverBase, Availability -from pyomo.contrib.solver.common.config import BranchAndBoundConfig -from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, - NoDualsError, - NoReducedCostsError, - NoSolutionError, - IncompatibleModelError, -) -from pyomo.contrib.solver.common.results import ( - Results, - SolutionStatus, - TerminationCondition, -) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase - - -gurobipy, gurobipy_available = attempt_import('gurobipy') - - -class GurobiConfigMixin: - """ - Mixin class for Gurobi-specific configurations - """ - - def __init__(self): - self.use_mipstart: bool = self.declare( - 'use_mipstart', - ConfigValue( - default=False, - domain=bool, - description="If True, the current values of the integer variables " - "will be passed to Gurobi.", - ), - ) - - -class GurobiConfig(BranchAndBoundConfig, GurobiConfigMixin): - def __init__( - self, - description=None, - doc=None, - implicit=False, - implicit_domain=None, - visibility=0, - ): - BranchAndBoundConfig.__init__( - self, - description=description, - doc=doc, - implicit=implicit, - implicit_domain=implicit_domain, - visibility=visibility, - ) - GurobiConfigMixin.__init__(self) - - -class GurobiDirectSolutionLoader(SolutionLoaderBase): - def __init__(self, grb_model, grb_cons, grb_vars, pyo_cons, pyo_vars, pyo_obj): - self._grb_model = grb_model - self._grb_cons = grb_cons - self._grb_vars = grb_vars - self._pyo_cons = pyo_cons - self._pyo_vars = pyo_vars - self._pyo_obj = pyo_obj - GurobiDirect._register_env_client() - - def __del__(self): - if python_is_shutting_down(): - return - # Free the associated model - if self._grb_model is not None: - self._grb_cons = None - self._grb_vars = None - self._pyo_cons = None - self._pyo_vars = None - self._pyo_obj = None - # explicitly release the model - self._grb_model.dispose() - self._grb_model = None - # Release the gurobi license if this is the last reference to - # the environment (either through a results object or solver - # interface) - GurobiDirect._release_env_client() - - def load_vars(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 - if self._grb_model.SolCount == 0: - raise NoSolutionError() - - iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) - for p_var, g_var in iterator: - p_var.set_value(g_var, skip_validation=True) - StaleFlagManager.mark_all_as_stale(delayed=True) - - def get_primals(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 - if self._grb_model.SolCount == 0: - raise NoSolutionError() - - iterator = zip(self._pyo_vars, self._grb_vars.x.tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) - return ComponentMap(iterator) - - def get_duals(self, cons_to_load=None): - if self._grb_model.Status != gurobipy.GRB.OPTIMAL: - raise NoDualsError() - - def dedup(_iter): - last = None - for con_info_dual in _iter: - if not con_info_dual[1] and con_info_dual[0][0] is last: - continue - last = con_info_dual[0][0] - yield con_info_dual - - iterator = dedup(zip(self._pyo_cons, self._grb_cons.getAttr('Pi').tolist())) - if cons_to_load: - cons_to_load = set(cons_to_load) - iterator = filter( - lambda con_info_dual: con_info_dual[0][0] in cons_to_load, iterator - ) - return {con_info[0]: dual for con_info, dual in iterator} - - def get_reduced_costs(self, vars_to_load=None): - if self._grb_model.Status != gurobipy.GRB.OPTIMAL: - raise NoReducedCostsError() - - iterator = zip(self._pyo_vars, self._grb_vars.getAttr('Rc').tolist()) - if vars_to_load: - vars_to_load = ComponentSet(vars_to_load) - iterator = filter(lambda var_rc: var_rc[0] in vars_to_load, iterator) - return ComponentMap(iterator) - - -class GurobiSolverMixin: - """ - gurobi_direct and gurobi_persistent check availability and set versions - in the same way. This moves the logic to a central location to reduce - duplicate code. - """ - - _num_gurobipy_env_clients = 0 - _gurobipy_env = None - _available = None - _gurobipy_available = gurobipy_available - - def available(self): - if self._available is None: - # this triggers the deferred import, and for the persistent - # interface, may update the _available flag - # - # Note that we set the _available flag on the *most derived - # class* and not on the instance, or on the base class. That - # allows different derived interfaces to have different - # availability (e.g., persistent has a minimum version - # requirement that the direct interface doesn't) - if not self._gurobipy_available: - if self._available is None: - self.__class__._available = Availability.NotFound - else: - self.__class__._available = self._check_license() - return self._available - - @staticmethod - def release_license(): - if GurobiSolverMixin._gurobipy_env is None: - return - if GurobiSolverMixin._num_gurobipy_env_clients: - logger.warning( - "Call to GurobiSolverMixin.release_license() with %s remaining " - "environment clients." % (GurobiSolverMixin._num_gurobipy_env_clients,) - ) - GurobiSolverMixin._gurobipy_env.close() - GurobiSolverMixin._gurobipy_env = None - - @staticmethod - def env(): - if GurobiSolverMixin._gurobipy_env is None: - with capture_output(capture_fd=True): - GurobiSolverMixin._gurobipy_env = gurobipy.Env() - return GurobiSolverMixin._gurobipy_env - - @staticmethod - def _register_env_client(): - GurobiSolverMixin._num_gurobipy_env_clients += 1 - - @staticmethod - def _release_env_client(): - GurobiSolverMixin._num_gurobipy_env_clients -= 1 - if GurobiSolverMixin._num_gurobipy_env_clients <= 0: - # Note that _num_gurobipy_env_clients should never be <0, - # but if it is, release_license will issue a warning (that - # we want to know about) - GurobiSolverMixin.release_license() - - def _check_license(self): - try: - model = gurobipy.Model(env=self.env()) - except gurobipy.GurobiError: - return Availability.BadLicense - - model.setParam('OutputFlag', 0) - try: - model.addVars(range(2001)) - model.optimize() - return Availability.FullLicense - except gurobipy.GurobiError: - return Availability.LimitedLicense - finally: - model.dispose() - - def version(self): - version = ( - gurobipy.GRB.VERSION_MAJOR, - gurobipy.GRB.VERSION_MINOR, - gurobipy.GRB.VERSION_TECHNICAL, - ) - return version - - -class GurobiDirect(GurobiSolverMixin, SolverBase): - """ - Interface to Gurobi using gurobipy - """ - - CONFIG = GurobiConfig() - - _tc_map = None - - def __init__(self, **kwds): - super().__init__(**kwds) - self._register_env_client() - - def __del__(self): - if not python_is_shutting_down(): - self._release_env_client() - - def solve(self, model, **kwds) -> Results: - start_timestamp = datetime.datetime.now(datetime.timezone.utc) - config = self.config(value=kwds, preserve_implicit=True) - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) - if config.timer is None: - config.timer = HierarchicalTimer() - timer = config.timer - - StaleFlagManager.mark_all_as_stale() - - timer.start('compile_model') - repn = LinearStandardFormCompiler().write( - model, mixed_form=True, set_sense=None - ) - timer.stop('compile_model') - - if len(repn.objectives) > 1: - raise IncompatibleModelError( - f"The {self.__class__.__name__} solver only supports models " - f"with zero or one objectives (received {len(repn.objectives)})." - ) - - timer.start('prepare_matrices') - inf = float('inf') - ninf = -inf - bounds = list(map(operator.attrgetter('bounds'), repn.columns)) - lb = [ninf if _b is None else _b for _b in map(operator.itemgetter(0), bounds)] - ub = [inf if _b is None else _b for _b in map(operator.itemgetter(1), bounds)] - CON = gurobipy.GRB.CONTINUOUS - BIN = gurobipy.GRB.BINARY - INT = gurobipy.GRB.INTEGER - vtype = [ - ( - CON - if v.is_continuous() - else BIN if v.is_binary() else INT if v.is_integer() else '?' - ) - for v in repn.columns - ] - sense_type = list('=<>') # Note: ordering matches 0, 1, -1 - sense = [sense_type[r[1]] for r in repn.rows] - timer.stop('prepare_matrices') - - ostreams = [io.StringIO()] + config.tee - res = Results() - - orig_cwd = os.getcwd() - try: - if config.working_dir: - os.chdir(config.working_dir) - with capture_output(TeeStream(*ostreams), capture_fd=False): - gurobi_model = gurobipy.Model(env=self.env()) - - timer.start('transfer_model') - x = gurobi_model.addMVar( - len(repn.columns), - lb=lb, - ub=ub, - obj=repn.c.todense()[0] if repn.c.shape[0] else 0, - vtype=vtype, - ) - A = gurobi_model.addMConstr(repn.A, x, sense, repn.rhs) - if repn.c.shape[0]: - gurobi_model.setAttr('ObjCon', repn.c_offset[0]) - gurobi_model.setAttr('ModelSense', int(repn.objectives[0].sense)) - # Note: calling gurobi_model.update() here is not - # necessary (it will happen as part of optimize()): - # gurobi_model.update() - timer.stop('transfer_model') - - options = config.solver_options - - gurobi_model.setParam('LogToConsole', 1) - - if config.threads is not None: - gurobi_model.setParam('Threads', config.threads) - if config.time_limit is not None: - gurobi_model.setParam('TimeLimit', config.time_limit) - if config.rel_gap is not None: - gurobi_model.setParam('MIPGap', config.rel_gap) - if config.abs_gap is not None: - gurobi_model.setParam('MIPGapAbs', config.abs_gap) - - if config.use_mipstart: - raise MouseTrap("MIPSTART not yet supported") - - for key, option in options.items(): - gurobi_model.setParam(key, option) - - timer.start('optimize') - gurobi_model.optimize() - timer.stop('optimize') - finally: - os.chdir(orig_cwd) - - res = self._postsolve( - timer, - config, - GurobiDirectSolutionLoader( - gurobi_model, A, x, repn.rows, repn.columns, repn.objectives - ), - ) - - res.solver_config = config - res.solver_name = 'Gurobi' - res.solver_version = self.version() - res.solver_log = ostreams[0].getvalue() - - end_timestamp = datetime.datetime.now(datetime.timezone.utc) - res.timing_info.start_timestamp = start_timestamp - res.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() - res.timing_info.timer = timer - return res - - def _postsolve(self, timer: HierarchicalTimer, config, loader): - grb_model = loader._grb_model - status = grb_model.Status - - results = Results() - results.solution_loader = loader - results.timing_info.gurobi_time = grb_model.Runtime - - if grb_model.SolCount > 0: - if status == gurobipy.GRB.OPTIMAL: - results.solution_status = SolutionStatus.optimal - else: - results.solution_status = SolutionStatus.feasible - else: - results.solution_status = SolutionStatus.noSolution - - results.termination_condition = self._get_tc_map().get( - status, TerminationCondition.unknown - ) - - if ( - results.termination_condition - != TerminationCondition.convergenceCriteriaSatisfied - and config.raise_exception_on_nonoptimal_result - ): - raise NoOptimalSolutionError() - - if loader._pyo_obj: - try: - if math.isfinite(grb_model.ObjVal): - results.incumbent_objective = grb_model.ObjVal - else: - results.incumbent_objective = None - except (gurobipy.GurobiError, AttributeError): - results.incumbent_objective = None - try: - results.objective_bound = grb_model.ObjBound - except (gurobipy.GurobiError, AttributeError): - if grb_model.ModelSense == ObjectiveSense.minimize: - results.objective_bound = -math.inf - else: - results.objective_bound = math.inf - else: - results.incumbent_objective = None - results.objective_bound = None - - results.iteration_count = grb_model.getAttr('IterCount') - - timer.start('load solution') - if config.load_solutions: - if grb_model.SolCount > 0: - results.solution_loader.load_vars() - else: - raise NoFeasibleSolutionError() - timer.stop('load solution') - - return results - - def _get_tc_map(self): - if GurobiDirect._tc_map is None: - grb = gurobipy.GRB - tc = TerminationCondition - GurobiDirect._tc_map = { - grb.LOADED: tc.unknown, # problem is loaded, but no solution - grb.OPTIMAL: tc.convergenceCriteriaSatisfied, - grb.INFEASIBLE: tc.provenInfeasible, - grb.INF_OR_UNBD: tc.infeasibleOrUnbounded, - grb.UNBOUNDED: tc.unbounded, - grb.CUTOFF: tc.objectiveLimit, - grb.ITERATION_LIMIT: tc.iterationLimit, - grb.NODE_LIMIT: tc.iterationLimit, - grb.TIME_LIMIT: tc.maxTimeLimit, - grb.SOLUTION_LIMIT: tc.unknown, - grb.INTERRUPTED: tc.interrupted, - grb.NUMERIC: tc.unknown, - grb.SUBOPTIMAL: tc.unknown, - grb.USER_OBJ_LIMIT: tc.objectiveLimit, - } - return GurobiDirect._tc_map diff --git a/pyomo/contrib/solver/solvers/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi_persistent.py deleted file mode 100644 index ea3693c1c70..00000000000 --- a/pyomo/contrib/solver/solvers/gurobi_persistent.py +++ /dev/null @@ -1,1409 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ - -import io -import logging -import math -from typing import List, Optional -from collections.abc import Iterable - -from pyomo.common.collections import ComponentSet, ComponentMap, OrderedSet -from pyomo.common.dependencies import attempt_import -from pyomo.common.errors import ApplicationError -from pyomo.common.tee import capture_output, TeeStream -from pyomo.common.timing import HierarchicalTimer -from pyomo.common.shutdown import python_is_shutting_down -from pyomo.core.kernel.objective import minimize, maximize -from pyomo.core.base import SymbolMap, NumericLabeler, TextLabeler -from pyomo.core.base.var import VarData -from pyomo.core.base.constraint import ConstraintData -from pyomo.core.base.sos import SOSConstraintData -from pyomo.core.base.param import ParamData -from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types -from pyomo.repn import generate_standard_repn -from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression -from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability -from pyomo.contrib.solver.common.results import ( - Results, - TerminationCondition, - SolutionStatus, -) -from pyomo.contrib.solver.common.config import PersistentBranchAndBoundConfig -from pyomo.contrib.solver.solvers.gurobi_direct import ( - GurobiConfigMixin, - GurobiSolverMixin, -) -from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, - NoDualsError, - NoReducedCostsError, - NoSolutionError, - IncompatibleModelError, -) -from pyomo.contrib.solver.common.persistent import ( - PersistentSolverUtils, - PersistentSolverMixin, -) -from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader -from pyomo.core.staleflag import StaleFlagManager - - -logger = logging.getLogger(__name__) - - -def _import_gurobipy(): - try: - import gurobipy - except ImportError: - GurobiPersistent._available = Availability.NotFound - raise - if gurobipy.GRB.VERSION_MAJOR < 7: - GurobiPersistent._available = Availability.BadVersion - raise ImportError('The Persistent Gurobi interface requires gurobipy>=7.0.0') - return gurobipy - - -gurobipy, gurobipy_available = attempt_import('gurobipy', importer=_import_gurobipy) - - -class GurobiConfig(PersistentBranchAndBoundConfig, GurobiConfigMixin): - def __init__( - self, - description=None, - doc=None, - implicit=False, - implicit_domain=None, - visibility=0, - ): - PersistentBranchAndBoundConfig.__init__( - self, - description=description, - doc=doc, - implicit=implicit, - implicit_domain=implicit_domain, - visibility=visibility, - ) - GurobiConfigMixin.__init__(self) - - -class GurobiSolutionLoader(PersistentSolutionLoader): - def load_vars(self, vars_to_load=None, solution_number=0): - self._assert_solution_still_valid() - self._solver._load_vars( - vars_to_load=vars_to_load, solution_number=solution_number - ) - - def get_primals(self, vars_to_load=None, solution_number=0): - self._assert_solution_still_valid() - return self._solver._get_primals( - vars_to_load=vars_to_load, solution_number=solution_number - ) - - -class _MutableLowerBound: - def __init__(self, expr): - self.var = None - self.expr = expr - - def update(self): - self.var.setAttr('lb', value(self.expr)) - - -class _MutableUpperBound: - def __init__(self, expr): - self.var = None - self.expr = expr - - def update(self): - self.var.setAttr('ub', value(self.expr)) - - -class _MutableLinearCoefficient: - def __init__(self): - self.expr = None - self.var = None - self.con = None - self.gurobi_model = None - - def update(self): - self.gurobi_model.chgCoeff(self.con, self.var, value(self.expr)) - - -class _MutableRangeConstant: - def __init__(self): - self.lhs_expr = None - self.rhs_expr = None - self.con = None - self.slack_name = None - self.gurobi_model = None - - def update(self): - rhs_val = value(self.rhs_expr) - lhs_val = value(self.lhs_expr) - self.con.rhs = rhs_val - slack = self.gurobi_model.getVarByName(self.slack_name) - slack.ub = rhs_val - lhs_val - - -class _MutableConstant: - def __init__(self): - self.expr = None - self.con = None - - def update(self): - self.con.rhs = value(self.expr) - - -class _MutableQuadraticConstraint: - def __init__( - self, gurobi_model, gurobi_con, constant, linear_coefs, quadratic_coefs - ): - self.con = gurobi_con - self.gurobi_model = gurobi_model - self.constant = constant - self.last_constant_value = value(self.constant.expr) - self.linear_coefs = linear_coefs - self.last_linear_coef_values = [value(i.expr) for i in self.linear_coefs] - self.quadratic_coefs = quadratic_coefs - self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] - - def get_updated_expression(self): - gurobi_expr = self.gurobi_model.getQCRow(self.con) - for ndx, coef in enumerate(self.linear_coefs): - current_coef_value = value(coef.expr) - incremental_coef_value = ( - current_coef_value - self.last_linear_coef_values[ndx] - ) - gurobi_expr += incremental_coef_value * coef.var - self.last_linear_coef_values[ndx] = current_coef_value - for ndx, coef in enumerate(self.quadratic_coefs): - current_coef_value = value(coef.expr) - incremental_coef_value = ( - current_coef_value - self.last_quadratic_coef_values[ndx] - ) - gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 - self.last_quadratic_coef_values[ndx] = current_coef_value - return gurobi_expr - - def get_updated_rhs(self): - return value(self.constant.expr) - - -class _MutableObjective: - def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): - self.gurobi_model = gurobi_model - self.constant = constant - self.linear_coefs = linear_coefs - self.quadratic_coefs = quadratic_coefs - self.last_quadratic_coef_values = [value(i.expr) for i in self.quadratic_coefs] - - def get_updated_expression(self): - for ndx, coef in enumerate(self.linear_coefs): - coef.var.obj = value(coef.expr) - self.gurobi_model.ObjCon = value(self.constant.expr) - - gurobi_expr = None - for ndx, coef in enumerate(self.quadratic_coefs): - if value(coef.expr) != self.last_quadratic_coef_values[ndx]: - if gurobi_expr is None: - self.gurobi_model.update() - gurobi_expr = self.gurobi_model.getObjective() - current_coef_value = value(coef.expr) - incremental_coef_value = ( - current_coef_value - self.last_quadratic_coef_values[ndx] - ) - gurobi_expr += incremental_coef_value * coef.var1 * coef.var2 - self.last_quadratic_coef_values[ndx] = current_coef_value - return gurobi_expr - - -class _MutableQuadraticCoefficient: - def __init__(self): - self.expr = None - self.var1 = None - self.var2 = None - - -class GurobiPersistent( - GurobiSolverMixin, - PersistentSolverMixin, - PersistentSolverUtils, - PersistentSolverBase, -): - """ - Interface to Gurobi persistent - """ - - CONFIG = GurobiConfig() - _gurobipy_available = gurobipy_available - - def __init__(self, **kwds): - treat_fixed_vars_as_params = kwds.pop('treat_fixed_vars_as_params', True) - PersistentSolverBase.__init__(self, **kwds) - PersistentSolverUtils.__init__( - self, treat_fixed_vars_as_params=treat_fixed_vars_as_params - ) - self._register_env_client() - self._solver_model = None - self._symbol_map = SymbolMap() - self._labeler = None - self._pyomo_var_to_solver_var_map = {} - self._pyomo_con_to_solver_con_map = {} - self._solver_con_to_pyomo_con_map = {} - self._pyomo_sos_to_solver_sos_map = {} - self._range_constraints = OrderedSet() - self._mutable_helpers = {} - self._mutable_bounds = {} - self._mutable_quadratic_helpers = {} - self._mutable_objective = None - self._needs_updated = True - self._callback = None - self._callback_func = None - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._last_results_object: Optional[Results] = None - - def release_license(self): - self._reinit() - self.__class__.release_license() - - def __del__(self): - if not python_is_shutting_down(): - self._release_env_client() - - @property - def symbol_map(self): - return self._symbol_map - - def _solve(self): - config = self._active_config - timer = config.timer - ostreams = [io.StringIO()] + config.tee - - with capture_output(TeeStream(*ostreams), capture_fd=False): - options = config.solver_options - - self._solver_model.setParam('LogToConsole', 1) - - if config.threads is not None: - self._solver_model.setParam('Threads', config.threads) - if config.time_limit is not None: - self._solver_model.setParam('TimeLimit', config.time_limit) - if config.rel_gap is not None: - self._solver_model.setParam('MIPGap', config.rel_gap) - if config.abs_gap is not None: - self._solver_model.setParam('MIPGapAbs', config.abs_gap) - - if config.use_mipstart: - for ( - pyomo_var_id, - gurobi_var, - ) in self._pyomo_var_to_solver_var_map.items(): - pyomo_var = self._vars[pyomo_var_id][0] - if pyomo_var.is_integer() and pyomo_var.value is not None: - self.set_var_attr(pyomo_var, 'Start', pyomo_var.value) - - for key, option in options.items(): - self._solver_model.setParam(key, option) - - timer.start('optimize') - self._solver_model.optimize(self._callback) - timer.stop('optimize') - - self._needs_updated = False - res = self._postsolve(timer) - res.solver_config = config - res.solver_name = 'Gurobi' - res.solver_version = self.version() - res.solver_log = ostreams[0].getvalue() - return res - - def _process_domain_and_bounds( - self, var, var_id, mutable_lbs, mutable_ubs, ndx, gurobipy_var - ): - _v, _lb, _ub, _fixed, _domain_interval, _value = self._vars[id(var)] - lb, ub, step = _domain_interval - if lb is None: - lb = -gurobipy.GRB.INFINITY - if ub is None: - ub = gurobipy.GRB.INFINITY - if step == 0: - vtype = gurobipy.GRB.CONTINUOUS - elif step == 1: - if lb == 0 and ub == 1: - vtype = gurobipy.GRB.BINARY - else: - vtype = gurobipy.GRB.INTEGER - else: - raise ValueError( - f'Unrecognized domain step: {step} (should be either 0 or 1)' - ) - if _fixed: - lb = _value - ub = _value - else: - if _lb is not None: - if not is_constant(_lb): - mutable_bound = _MutableLowerBound(NPV_MaxExpression((_lb, lb))) - if gurobipy_var is None: - mutable_lbs[ndx] = mutable_bound - else: - mutable_bound.var = gurobipy_var - self._mutable_bounds[var_id, 'lb'] = (var, mutable_bound) - lb = max(value(_lb), lb) - if _ub is not None: - if not is_constant(_ub): - mutable_bound = _MutableUpperBound(NPV_MinExpression((_ub, ub))) - if gurobipy_var is None: - mutable_ubs[ndx] = mutable_bound - else: - mutable_bound.var = gurobipy_var - self._mutable_bounds[var_id, 'ub'] = (var, mutable_bound) - ub = min(value(_ub), ub) - - return lb, ub, vtype - - def _add_variables(self, variables: List[VarData]): - var_names = [] - vtypes = [] - lbs = [] - ubs = [] - mutable_lbs = {} - mutable_ubs = {} - for ndx, var in enumerate(variables): - varname = self._symbol_map.getSymbol(var, self._labeler) - lb, ub, vtype = self._process_domain_and_bounds( - var, id(var), mutable_lbs, mutable_ubs, ndx, None - ) - var_names.append(varname) - vtypes.append(vtype) - lbs.append(lb) - ubs.append(ub) - - gurobi_vars = self._solver_model.addVars( - len(variables), lb=lbs, ub=ubs, vtype=vtypes, name=var_names - ) - - for ndx, pyomo_var in enumerate(variables): - gurobi_var = gurobi_vars[ndx] - self._pyomo_var_to_solver_var_map[id(pyomo_var)] = gurobi_var - for ndx, mutable_bound in mutable_lbs.items(): - mutable_bound.var = gurobi_vars[ndx] - for ndx, mutable_bound in mutable_ubs.items(): - mutable_bound.var = gurobi_vars[ndx] - self._vars_added_since_update.update(variables) - self._needs_updated = True - - def _add_parameters(self, params: List[ParamData]): - pass - - def _reinit(self): - saved_config = self.config - saved_tmp_config = self._active_config - self.__init__(treat_fixed_vars_as_params=self._treat_fixed_vars_as_params) - # Note that __init__ registers a new env client, so we need to - # release it here: - self._release_env_client() - self.config = saved_config - self._active_config = saved_tmp_config - - def set_instance(self, model): - if self._last_results_object is not None: - self._last_results_object.solution_loader.invalidate() - if not self.available(): - c = self.__class__ - raise ApplicationError( - f'Solver {c.__module__}.{c.__qualname__} is not available ' - f'({self.available()}).' - ) - self._reinit() - self._model = model - - if self.config.symbolic_solver_labels: - self._labeler = TextLabeler() - else: - self._labeler = NumericLabeler('x') - - self._solver_model = gurobipy.Model(name=model.name or '', env=self.env()) - - self.add_block(model) - if self._objective is None: - self.set_objective(None) - - def _get_expr_from_pyomo_expr(self, expr): - mutable_linear_coefficients = [] - mutable_quadratic_coefficients = [] - repn = generate_standard_repn(expr, quadratic=True, compute_values=False) - - degree = repn.polynomial_degree() - if (degree is None) or (degree > 2): - raise IncompatibleModelError( - f'GurobiAuto does not support expressions of degree {degree}.' - ) - - if len(repn.linear_vars) > 0: - linear_coef_vals = [] - for ndx, coef in enumerate(repn.linear_coefs): - if not is_constant(coef): - mutable_linear_coefficient = _MutableLinearCoefficient() - mutable_linear_coefficient.expr = coef - mutable_linear_coefficient.var = self._pyomo_var_to_solver_var_map[ - id(repn.linear_vars[ndx]) - ] - mutable_linear_coefficients.append(mutable_linear_coefficient) - linear_coef_vals.append(value(coef)) - new_expr = gurobipy.LinExpr( - linear_coef_vals, - [self._pyomo_var_to_solver_var_map[id(i)] for i in repn.linear_vars], - ) - else: - new_expr = 0.0 - - for ndx, v in enumerate(repn.quadratic_vars): - x, y = v - gurobi_x = self._pyomo_var_to_solver_var_map[id(x)] - gurobi_y = self._pyomo_var_to_solver_var_map[id(y)] - coef = repn.quadratic_coefs[ndx] - if not is_constant(coef): - mutable_quadratic_coefficient = _MutableQuadraticCoefficient() - mutable_quadratic_coefficient.expr = coef - mutable_quadratic_coefficient.var1 = gurobi_x - mutable_quadratic_coefficient.var2 = gurobi_y - mutable_quadratic_coefficients.append(mutable_quadratic_coefficient) - coef_val = value(coef) - new_expr += coef_val * gurobi_x * gurobi_y - - return ( - new_expr, - repn.constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) - - def _add_constraints(self, cons: List[ConstraintData]): - for con in cons: - conname = self._symbol_map.getSymbol(con, self._labeler) - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(con.body) - - if ( - gurobi_expr.__class__ in {gurobipy.LinExpr, gurobipy.Var} - or gurobi_expr.__class__ in native_numeric_types - ): - if con.equality: - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addLConstr( - gurobi_expr, gurobipy.GRB.EQUAL, rhs_val, name=conname - ) - if not is_constant(rhs_expr): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_constant.con = gurobipy_con - self._mutable_helpers[con] = [mutable_constant] - elif con.has_lb() and con.has_ub(): - lhs_expr = con.lower - repn_constant - rhs_expr = con.upper - repn_constant - lhs_val = value(lhs_expr) - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addRange( - gurobi_expr, lhs_val, rhs_val, name=conname - ) - self._range_constraints.add(con) - if not is_constant(lhs_expr) or not is_constant(rhs_expr): - mutable_range_constant = _MutableRangeConstant() - mutable_range_constant.lhs_expr = lhs_expr - mutable_range_constant.rhs_expr = rhs_expr - mutable_range_constant.con = gurobipy_con - mutable_range_constant.slack_name = 'Rg' + conname - mutable_range_constant.gurobi_model = self._solver_model - self._mutable_helpers[con] = [mutable_range_constant] - elif con.has_lb(): - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addLConstr( - gurobi_expr, gurobipy.GRB.GREATER_EQUAL, rhs_val, name=conname - ) - if not is_constant(rhs_expr): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_constant.con = gurobipy_con - self._mutable_helpers[con] = [mutable_constant] - elif con.has_ub(): - rhs_expr = con.upper - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addLConstr( - gurobi_expr, gurobipy.GRB.LESS_EQUAL, rhs_val, name=conname - ) - if not is_constant(rhs_expr): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_constant.con = gurobipy_con - self._mutable_helpers[con] = [mutable_constant] - else: - raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" - ) - for tmp in mutable_linear_coefficients: - tmp.con = gurobipy_con - tmp.gurobi_model = self._solver_model - if len(mutable_linear_coefficients) > 0: - if con not in self._mutable_helpers: - self._mutable_helpers[con] = mutable_linear_coefficients - else: - self._mutable_helpers[con].extend(mutable_linear_coefficients) - elif gurobi_expr.__class__ is gurobipy.QuadExpr: - if con.equality: - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addQConstr( - gurobi_expr, gurobipy.GRB.EQUAL, rhs_val, name=conname - ) - elif con.has_lb() and con.has_ub(): - raise NotImplementedError( - 'Quadratic range constraints are not supported' - ) - elif con.has_lb(): - rhs_expr = con.lower - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addQConstr( - gurobi_expr, gurobipy.GRB.GREATER_EQUAL, rhs_val, name=conname - ) - elif con.has_ub(): - rhs_expr = con.upper - repn_constant - rhs_val = value(rhs_expr) - gurobipy_con = self._solver_model.addQConstr( - gurobi_expr, gurobipy.GRB.LESS_EQUAL, rhs_val, name=conname - ) - else: - raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" - ) - if ( - len(mutable_linear_coefficients) > 0 - or len(mutable_quadratic_coefficients) > 0 - or not is_constant(repn_constant) - ): - mutable_constant = _MutableConstant() - mutable_constant.expr = rhs_expr - mutable_quadratic_constraint = _MutableQuadraticConstraint( - self._solver_model, - gurobipy_con, - mutable_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) - self._mutable_quadratic_helpers[con] = mutable_quadratic_constraint - else: - raise ValueError( - f'Unrecognized Gurobi expression type: {str(gurobi_expr.__class__)}' - ) - - self._pyomo_con_to_solver_con_map[con] = gurobipy_con - self._solver_con_to_pyomo_con_map[id(gurobipy_con)] = con - self._constraints_added_since_update.update(cons) - self._needs_updated = True - - def _add_sos_constraints(self, cons: List[SOSConstraintData]): - for con in cons: - conname = self._symbol_map.getSymbol(con, self._labeler) - level = con.level - if level == 1: - sos_type = gurobipy.GRB.SOS_TYPE1 - elif level == 2: - sos_type = gurobipy.GRB.SOS_TYPE2 - else: - raise ValueError( - f"Solver does not support SOS level {level} constraints" - ) - - gurobi_vars = [] - weights = [] - - for v, w in con.get_items(): - v_id = id(v) - gurobi_vars.append(self._pyomo_var_to_solver_var_map[v_id]) - weights.append(w) - - gurobipy_con = self._solver_model.addSOS(sos_type, gurobi_vars, weights) - self._pyomo_sos_to_solver_sos_map[con] = gurobipy_con - self._constraints_added_since_update.update(cons) - self._needs_updated = True - - def _remove_constraints(self, cons: List[ConstraintData]): - for con in cons: - if con in self._constraints_added_since_update: - self._update_gurobi_model() - solver_con = self._pyomo_con_to_solver_con_map[con] - self._solver_model.remove(solver_con) - self._symbol_map.removeSymbol(con) - del self._pyomo_con_to_solver_con_map[con] - del self._solver_con_to_pyomo_con_map[id(solver_con)] - self._range_constraints.discard(con) - self._mutable_helpers.pop(con, None) - self._mutable_quadratic_helpers.pop(con, None) - self._needs_updated = True - - def _remove_sos_constraints(self, cons: List[SOSConstraintData]): - for con in cons: - if con in self._constraints_added_since_update: - self._update_gurobi_model() - solver_sos_con = self._pyomo_sos_to_solver_sos_map[con] - self._solver_model.remove(solver_sos_con) - self._symbol_map.removeSymbol(con) - del self._pyomo_sos_to_solver_sos_map[con] - self._needs_updated = True - - def _remove_variables(self, variables: List[VarData]): - for var in variables: - v_id = id(var) - if var in self._vars_added_since_update: - self._update_gurobi_model() - solver_var = self._pyomo_var_to_solver_var_map[v_id] - self._solver_model.remove(solver_var) - self._symbol_map.removeSymbol(var) - del self._pyomo_var_to_solver_var_map[v_id] - self._mutable_bounds.pop(v_id, None) - self._needs_updated = True - - def _remove_parameters(self, params: List[ParamData]): - pass - - def _update_variables(self, variables: List[VarData]): - for var in variables: - var_id = id(var) - if var_id not in self._pyomo_var_to_solver_var_map: - raise ValueError( - f'The Var provided to update_var needs to be added first: {var}' - ) - self._mutable_bounds.pop((var_id, 'lb'), None) - self._mutable_bounds.pop((var_id, 'ub'), None) - gurobipy_var = self._pyomo_var_to_solver_var_map[var_id] - lb, ub, vtype = self._process_domain_and_bounds( - var, var_id, None, None, None, gurobipy_var - ) - gurobipy_var.setAttr('lb', lb) - gurobipy_var.setAttr('ub', ub) - gurobipy_var.setAttr('vtype', vtype) - self._needs_updated = True - - def update_parameters(self): - for con, helpers in self._mutable_helpers.items(): - for helper in helpers: - helper.update() - for k, (v, helper) in self._mutable_bounds.items(): - helper.update() - - for con, helper in self._mutable_quadratic_helpers.items(): - if con in self._constraints_added_since_update: - self._update_gurobi_model() - gurobi_con = helper.con - new_gurobi_expr = helper.get_updated_expression() - new_rhs = helper.get_updated_rhs() - new_sense = gurobi_con.qcsense - pyomo_con = self._solver_con_to_pyomo_con_map[id(gurobi_con)] - name = self._symbol_map.getSymbol(pyomo_con, self._labeler) - self._solver_model.remove(gurobi_con) - new_con = self._solver_model.addQConstr( - new_gurobi_expr, new_sense, new_rhs, name=name - ) - self._pyomo_con_to_solver_con_map[id(pyomo_con)] = new_con - del self._solver_con_to_pyomo_con_map[id(gurobi_con)] - self._solver_con_to_pyomo_con_map[id(new_con)] = pyomo_con - helper.con = new_con - self._constraints_added_since_update.add(con) - - helper = self._mutable_objective - pyomo_obj = self._objective - new_gurobi_expr = helper.get_updated_expression() - if new_gurobi_expr is not None: - if pyomo_obj.sense == minimize: - sense = gurobipy.GRB.MINIMIZE - else: - sense = gurobipy.GRB.MAXIMIZE - self._solver_model.setObjective(new_gurobi_expr, sense=sense) - - def _set_objective(self, obj): - if obj is None: - sense = gurobipy.GRB.MINIMIZE - gurobi_expr = 0 - repn_constant = 0 - mutable_linear_coefficients = [] - mutable_quadratic_coefficients = [] - else: - if obj.sense == minimize: - sense = gurobipy.GRB.MINIMIZE - elif obj.sense == maximize: - sense = gurobipy.GRB.MAXIMIZE - else: - raise ValueError(f'Objective sense is not recognized: {obj.sense}') - - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(obj.expr) - - mutable_constant = _MutableConstant() - mutable_constant.expr = repn_constant - mutable_objective = _MutableObjective( - self._solver_model, - mutable_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) - self._mutable_objective = mutable_objective - - # These two lines are needed as a workaround - # see PR #2454 - self._solver_model.setObjective(0) - self._solver_model.update() - - self._solver_model.setObjective(gurobi_expr + value(repn_constant), sense=sense) - self._needs_updated = True - - def _postsolve(self, timer: HierarchicalTimer): - config = self._active_config - - gprob = self._solver_model - grb = gurobipy.GRB - status = gprob.Status - - results = Results() - results.solution_loader = GurobiSolutionLoader(self) - results.timing_info.gurobi_time = gprob.Runtime - - if gprob.SolCount > 0: - if status == grb.OPTIMAL: - results.solution_status = SolutionStatus.optimal - else: - results.solution_status = SolutionStatus.feasible - else: - results.solution_status = SolutionStatus.noSolution - - if status == grb.LOADED: # problem is loaded, but no solution - results.termination_condition = TerminationCondition.unknown - elif status == grb.OPTIMAL: # optimal - results.termination_condition = ( - TerminationCondition.convergenceCriteriaSatisfied - ) - elif status == grb.INFEASIBLE: - results.termination_condition = TerminationCondition.provenInfeasible - elif status == grb.INF_OR_UNBD: - results.termination_condition = TerminationCondition.infeasibleOrUnbounded - elif status == grb.UNBOUNDED: - results.termination_condition = TerminationCondition.unbounded - elif status == grb.CUTOFF: - results.termination_condition = TerminationCondition.objectiveLimit - elif status == grb.ITERATION_LIMIT: - results.termination_condition = TerminationCondition.iterationLimit - elif status == grb.NODE_LIMIT: - results.termination_condition = TerminationCondition.iterationLimit - elif status == grb.TIME_LIMIT: - results.termination_condition = TerminationCondition.maxTimeLimit - elif status == grb.SOLUTION_LIMIT: - results.termination_condition = TerminationCondition.unknown - elif status == grb.INTERRUPTED: - results.termination_condition = TerminationCondition.interrupted - elif status == grb.NUMERIC: - results.termination_condition = TerminationCondition.unknown - elif status == grb.SUBOPTIMAL: - results.termination_condition = TerminationCondition.unknown - elif status == grb.USER_OBJ_LIMIT: - results.termination_condition = TerminationCondition.objectiveLimit - else: - results.termination_condition = TerminationCondition.unknown - - if ( - results.termination_condition - != TerminationCondition.convergenceCriteriaSatisfied - and config.raise_exception_on_nonoptimal_result - ): - raise NoOptimalSolutionError() - - results.incumbent_objective = None - results.objective_bound = None - if self._objective is not None: - try: - results.incumbent_objective = gprob.ObjVal - except (gurobipy.GurobiError, AttributeError): - results.incumbent_objective = None - try: - results.objective_bound = gprob.ObjBound - except (gurobipy.GurobiError, AttributeError): - if self._objective.sense == minimize: - results.objective_bound = -math.inf - else: - results.objective_bound = math.inf - - if results.incumbent_objective is not None and not math.isfinite( - results.incumbent_objective - ): - results.incumbent_objective = None - - results.iteration_count = gprob.getAttr('IterCount') - - timer.start('load solution') - if config.load_solutions: - if gprob.SolCount > 0: - self._load_vars() - else: - raise NoFeasibleSolutionError() - timer.stop('load solution') - - return results - - def _load_suboptimal_mip_solution(self, vars_to_load, solution_number): - if ( - self.get_model_attr('NumIntVars') == 0 - and self.get_model_attr('NumBinVars') == 0 - ): - raise ValueError( - 'Cannot obtain suboptimal solutions for a continuous model' - ) - var_map = self._pyomo_var_to_solver_var_map - ref_vars = self._referenced_variables - original_solution_number = self.get_gurobi_param_info('SolutionNumber')[2] - self.set_gurobi_param('SolutionNumber', solution_number) - gurobi_vars_to_load = [var_map[pyomo_var] for pyomo_var in vars_to_load] - vals = self._solver_model.getAttr("Xn", gurobi_vars_to_load) - res = ComponentMap() - for var_id, val in zip(vars_to_load, vals): - using_cons, using_sos, using_obj = ref_vars[var_id] - if using_cons or using_sos or (using_obj is not None): - res[self._vars[var_id][0]] = val - self.set_gurobi_param('SolutionNumber', original_solution_number) - return res - - def _load_vars(self, vars_to_load=None, solution_number=0): - for v, val in self._get_primals( - vars_to_load=vars_to_load, solution_number=solution_number - ).items(): - v.set_value(val, skip_validation=True) - StaleFlagManager.mark_all_as_stale(delayed=True) - - def _get_primals(self, vars_to_load=None, solution_number=0): - if self._needs_updated: - self._update_gurobi_model() # this is needed to ensure that solutions cannot be loaded after the model has been changed - - if self._solver_model.SolCount == 0: - raise NoSolutionError() - - var_map = self._pyomo_var_to_solver_var_map - ref_vars = self._referenced_variables - if vars_to_load is None: - vars_to_load = self._pyomo_var_to_solver_var_map.keys() - else: - vars_to_load = [id(v) for v in vars_to_load] - - if solution_number != 0: - return self._load_suboptimal_mip_solution( - vars_to_load=vars_to_load, solution_number=solution_number - ) - - gurobi_vars_to_load = [var_map[pyomo_var_id] for pyomo_var_id in vars_to_load] - vals = self._solver_model.getAttr("X", gurobi_vars_to_load) - - res = ComponentMap() - for var_id, val in zip(vars_to_load, vals): - using_cons, using_sos, using_obj = ref_vars[var_id] - if using_cons or using_sos or (using_obj is not None): - res[self._vars[var_id][0]] = val - return res - - def _get_reduced_costs(self, vars_to_load=None): - if self._needs_updated: - self._update_gurobi_model() - - if self._solver_model.Status != gurobipy.GRB.OPTIMAL: - raise NoReducedCostsError() - - var_map = self._pyomo_var_to_solver_var_map - ref_vars = self._referenced_variables - res = ComponentMap() - if vars_to_load is None: - vars_to_load = self._pyomo_var_to_solver_var_map.keys() - else: - vars_to_load = [id(v) for v in vars_to_load] - - gurobi_vars_to_load = [var_map[pyomo_var_id] for pyomo_var_id in vars_to_load] - vals = self._solver_model.getAttr("Rc", gurobi_vars_to_load) - - for var_id, val in zip(vars_to_load, vals): - using_cons, using_sos, using_obj = ref_vars[var_id] - if using_cons or using_sos or (using_obj is not None): - res[self._vars[var_id][0]] = val - - return res - - def _get_duals(self, cons_to_load=None): - if self._needs_updated: - self._update_gurobi_model() - - if self._solver_model.Status != gurobipy.GRB.OPTIMAL: - raise NoDualsError() - - con_map = self._pyomo_con_to_solver_con_map - reverse_con_map = self._solver_con_to_pyomo_con_map - dual = {} - - if cons_to_load is None: - linear_cons_to_load = self._solver_model.getConstrs() - quadratic_cons_to_load = self._solver_model.getQConstrs() - else: - gurobi_cons_to_load = OrderedSet( - [con_map[pyomo_con] for pyomo_con in cons_to_load] - ) - linear_cons_to_load = list( - gurobi_cons_to_load.intersection( - OrderedSet(self._solver_model.getConstrs()) - ) - ) - quadratic_cons_to_load = list( - gurobi_cons_to_load.intersection( - OrderedSet(self._solver_model.getQConstrs()) - ) - ) - linear_vals = self._solver_model.getAttr("Pi", linear_cons_to_load) - quadratic_vals = self._solver_model.getAttr("QCPi", quadratic_cons_to_load) - - for gurobi_con, val in zip(linear_cons_to_load, linear_vals): - pyomo_con = reverse_con_map[id(gurobi_con)] - dual[pyomo_con] = val - for gurobi_con, val in zip(quadratic_cons_to_load, quadratic_vals): - pyomo_con = reverse_con_map[id(gurobi_con)] - dual[pyomo_con] = val - - return dual - - def update(self, timer: HierarchicalTimer = None): - if self._needs_updated: - self._update_gurobi_model() - super().update(timer=timer) - self._update_gurobi_model() - - def _update_gurobi_model(self): - self._solver_model.update() - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._needs_updated = False - - def get_model_attr(self, attr): - """ - Get the value of an attribute on the Gurobi model. - - Parameters - ---------- - attr: str - The attribute to get. See Gurobi documentation for descriptions of the attributes. - """ - if self._needs_updated: - self._update_gurobi_model() - return self._solver_model.getAttr(attr) - - def write(self, filename): - """ - Write the model to a file (e.g., and lp file). - - Parameters - ---------- - filename: str - Name of the file to which the model should be written. - """ - self._solver_model.write(filename) - self._constraints_added_since_update = OrderedSet() - self._vars_added_since_update = ComponentSet() - self._needs_updated = False - - def set_linear_constraint_attr(self, con, attr, val): - """ - Set the value of an attribute on a gurobi linear constraint. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The pyomo constraint for which the corresponding gurobi constraint attribute - should be modified. - attr: str - The attribute to be modified. Options are: - CBasis - DStart - Lazy - val: any - See gurobi documentation for acceptable values. - """ - if attr in {'Sense', 'RHS', 'ConstrName'}: - raise ValueError( - f'Linear constraint attr {attr} cannot be set with' - ' the set_linear_constraint_attr method. Please use' - ' the remove_constraint and add_constraint methods.' - ) - self._pyomo_con_to_solver_con_map[con].setAttr(attr, val) - self._needs_updated = True - - def set_var_attr(self, var, attr, val): - """ - Set the value of an attribute on a gurobi variable. - - Parameters - ---------- - var: pyomo.core.base.var.VarData - The pyomo var for which the corresponding gurobi var attribute - should be modified. - attr: str - The attribute to be modified. Options are: - Start - VarHintVal - VarHintPri - BranchPriority - VBasis - PStart - val: any - See gurobi documentation for acceptable values. - """ - if attr in {'LB', 'UB', 'VType', 'VarName'}: - raise ValueError( - f'Var attr {attr} cannot be set with' - ' the set_var_attr method. Please use' - ' the update_var method.' - ) - if attr == 'Obj': - raise ValueError( - 'Var attr Obj cannot be set with' - ' the set_var_attr method. Please use' - ' the set_objective method.' - ) - self._pyomo_var_to_solver_var_map[id(var)].setAttr(attr, val) - self._needs_updated = True - - def get_var_attr(self, var, attr): - """ - Get the value of an attribute on a gurobi var. - - Parameters - ---------- - var: pyomo.core.base.var.VarData - The pyomo var for which the corresponding gurobi var attribute - should be retrieved. - attr: str - The attribute to get. See gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_var_to_solver_var_map[id(var)].getAttr(attr) - - def get_linear_constraint_attr(self, con, attr): - """ - Get the value of an attribute on a gurobi linear constraint. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The pyomo constraint for which the corresponding gurobi constraint attribute - should be retrieved. - attr: str - The attribute to get. See the Gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_con_to_solver_con_map[con].getAttr(attr) - - def get_sos_attr(self, con, attr): - """ - Get the value of an attribute on a gurobi sos constraint. - - Parameters - ---------- - con: pyomo.core.base.sos.SOSConstraintData - The pyomo SOS constraint for which the corresponding gurobi SOS constraint attribute - should be retrieved. - attr: str - The attribute to get. See the Gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_sos_to_solver_sos_map[con].getAttr(attr) - - def get_quadratic_constraint_attr(self, con, attr): - """ - Get the value of an attribute on a gurobi quadratic constraint. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The pyomo constraint for which the corresponding gurobi constraint attribute - should be retrieved. - attr: str - The attribute to get. See the Gurobi documentation - """ - if self._needs_updated: - self._update_gurobi_model() - return self._pyomo_con_to_solver_con_map[con].getAttr(attr) - - def set_gurobi_param(self, param, val): - """ - Set a gurobi parameter. - - Parameters - ---------- - param: str - The gurobi parameter to set. Options include any gurobi parameter. - Please see the Gurobi documentation for options. - val: any - The value to set the parameter to. See Gurobi documentation for possible values. - """ - self._solver_model.setParam(param, val) - - def get_gurobi_param_info(self, param): - """ - Get information about a gurobi parameter. - - Parameters - ---------- - param: str - The gurobi parameter to get info for. See Gurobi documentation for possible options. - - Returns - ------- - six-tuple containing the parameter name, type, value, minimum value, maximum value, and default value. - """ - return self._solver_model.getParamInfo(param) - - def _intermediate_callback(self): - def f(gurobi_model, where): - self._callback_func(self._model, self, where) - - return f - - def set_callback(self, func=None): - """ - Specify a callback for gurobi to use. - - Parameters - ---------- - func: function - The function to call. The function should have three arguments. The first will be the pyomo model being - solved. The second will be the GurobiPersistent instance. The third will be an enum member of - gurobipy.GRB.Callback. This will indicate where in the branch and bound algorithm gurobi is at. For - example, suppose we want to solve - - .. math:: - - min 2*x + y - - s.t. - - y >= (x-2)**2 - - 0 <= x <= 4 - - y >= 0 - - y integer - - as an MILP using extended cutting planes in callbacks. - - >>> from gurobipy import GRB # doctest:+SKIP - >>> import pyomo.environ as pyo - >>> from pyomo.core.expr.taylor_series import taylor_series_expansion - >>> from pyomo.contrib import appsi - >>> - >>> m = pyo.ConcreteModel() - >>> m.x = pyo.Var(bounds=(0, 4)) - >>> m.y = pyo.Var(within=pyo.Integers, bounds=(0, None)) - >>> m.obj = pyo.Objective(expr=2*m.x + m.y) - >>> m.cons = pyo.ConstraintList() # for the cutting planes - >>> - >>> def _add_cut(xval): - ... # a function to generate the cut - ... m.x.value = xval - ... return m.cons.add(m.y >= taylor_series_expansion((m.x - 2)**2)) - ... - >>> _c = _add_cut(0) # start with 2 cuts at the bounds of x - >>> _c = _add_cut(4) # this is an arbitrary choice - >>> - >>> opt = appsi.solvers.Gurobi() - >>> opt.config.stream_solver = True - >>> opt.set_instance(m) # doctest:+SKIP - >>> opt.gurobi_options['PreCrush'] = 1 - >>> opt.gurobi_options['LazyConstraints'] = 1 - >>> - >>> def my_callback(cb_m, cb_opt, cb_where): - ... if cb_where == GRB.Callback.MIPSOL: - ... cb_opt.cbGetSolution(variables=[m.x, m.y]) - ... if m.y.value < (m.x.value - 2)**2 - 1e-6: - ... cb_opt.cbLazy(_add_cut(m.x.value)) - ... - >>> opt.set_callback(my_callback) - >>> res = opt.solve(m) # doctest:+SKIP - - """ - if func is not None: - self._callback_func = func - self._callback = self._intermediate_callback() - else: - self._callback = None - self._callback_func = None - - def cbCut(self, con): - """ - Add a cut within a callback. - - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The cut to add - """ - if not con.active: - raise ValueError('cbCut expected an active constraint.') - - if is_fixed(con.body): - raise ValueError('cbCut expected a non-trivial constraint') - - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(con.body) - - if con.has_lb(): - if con.has_ub(): - raise ValueError('Range constraints are not supported in cbCut.') - if not is_fixed(con.lower): - raise ValueError(f'Lower bound of constraint {con} is not constant.') - if con.has_ub(): - if not is_fixed(con.upper): - raise ValueError(f'Upper bound of constraint {con} is not constant.') - - if con.equality: - self._solver_model.cbCut( - lhs=gurobi_expr, - sense=gurobipy.GRB.EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_lb() and (value(con.lower) > -float('inf')): - self._solver_model.cbCut( - lhs=gurobi_expr, - sense=gurobipy.GRB.GREATER_EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_ub() and (value(con.upper) < float('inf')): - self._solver_model.cbCut( - lhs=gurobi_expr, - sense=gurobipy.GRB.LESS_EQUAL, - rhs=value(con.upper - repn_constant), - ) - else: - raise ValueError( - f'Constraint does not have a lower or an upper bound {con} \n' - ) - - def cbGet(self, what): - return self._solver_model.cbGet(what) - - def cbGetNodeRel(self, variables): - """ - Parameters - ---------- - variables: Var or iterable of Var - """ - if not isinstance(variables, Iterable): - variables = [variables] - gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] - var_values = self._solver_model.cbGetNodeRel(gurobi_vars) - for i, v in enumerate(variables): - v.set_value(var_values[i], skip_validation=True) - - def cbGetSolution(self, variables): - """ - Parameters - ---------- - variables: iterable of vars - """ - if not isinstance(variables, Iterable): - variables = [variables] - gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] - var_values = self._solver_model.cbGetSolution(gurobi_vars) - for i, v in enumerate(variables): - v.set_value(var_values[i], skip_validation=True) - - def cbLazy(self, con): - """ - Parameters - ---------- - con: pyomo.core.base.constraint.ConstraintData - The lazy constraint to add - """ - if not con.active: - raise ValueError('cbLazy expected an active constraint.') - - if is_fixed(con.body): - raise ValueError('cbLazy expected a non-trivial constraint') - - ( - gurobi_expr, - repn_constant, - mutable_linear_coefficients, - mutable_quadratic_coefficients, - ) = self._get_expr_from_pyomo_expr(con.body) - - if con.has_lb(): - if con.has_ub(): - raise ValueError('Range constraints are not supported in cbLazy.') - if not is_fixed(con.lower): - raise ValueError(f'Lower bound of constraint {con} is not constant.') - if con.has_ub(): - if not is_fixed(con.upper): - raise ValueError(f'Upper bound of constraint {con} is not constant.') - - if con.equality: - self._solver_model.cbLazy( - lhs=gurobi_expr, - sense=gurobipy.GRB.EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_lb() and (value(con.lower) > -float('inf')): - self._solver_model.cbLazy( - lhs=gurobi_expr, - sense=gurobipy.GRB.GREATER_EQUAL, - rhs=value(con.lower - repn_constant), - ) - elif con.has_ub() and (value(con.upper) < float('inf')): - self._solver_model.cbLazy( - lhs=gurobi_expr, - sense=gurobipy.GRB.LESS_EQUAL, - rhs=value(con.upper - repn_constant), - ) - else: - raise ValueError( - f'Constraint does not have a lower or an upper bound {con} \n' - ) - - def cbSetSolution(self, variables, solution): - if not isinstance(variables, Iterable): - variables = [variables] - gurobi_vars = [self._pyomo_var_to_solver_var_map[id(i)] for i in variables] - self._solver_model.cbSetSolution(gurobi_vars, solution) - - def cbUseSolution(self): - return self._solver_model.cbUseSolution() - - def reset(self): - self._solver_model.reset() diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py index 8703ae9edff..96cd1498956 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py @@ -11,7 +11,7 @@ import pyomo.common.unittest as unittest import pyomo.environ as pyo -from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiPersistent from pyomo.contrib.solver.common.results import SolutionStatus from pyomo.core.expr.taylor_series import taylor_series_expansion @@ -471,11 +471,11 @@ def test_solution_number(self): res = opt.solve(m) num_solutions = opt.get_model_attr('SolCount') self.assertEqual(num_solutions, 3) - res.solution_loader.load_vars(solution_number=0) + res.solution_loader.load_vars(solution_id=0) self.assertAlmostEqual(pyo.value(m.obj.expr), 6.431184939357673) - res.solution_loader.load_vars(solution_number=1) + res.solution_loader.load_vars(solution_id=1) self.assertAlmostEqual(pyo.value(m.obj.expr), 6.584793218502477) - res.solution_loader.load_vars(solution_number=2) + res.solution_loader.load_vars(solution_id=2) self.assertAlmostEqual(pyo.value(m.obj.expr), 6.592304628123309) def test_zero_time_limit(self): @@ -496,16 +496,14 @@ def test_zero_time_limit(self): self.assertIsNone(res.incumbent_objective) -class TestManualModel(unittest.TestCase): +class TestManualMode(unittest.TestCase): def setUp(self): opt = GurobiPersistent() - opt.config.auto_updates.check_for_new_or_removed_params = False - opt.config.auto_updates.check_for_new_or_removed_vars = False - opt.config.auto_updates.check_for_new_or_removed_constraints = False - opt.config.auto_updates.update_parameters = False - opt.config.auto_updates.update_vars = False - opt.config.auto_updates.update_constraints = False - opt.config.auto_updates.update_named_expressions = False + opt.auto_updates.check_for_new_or_removed_constraints = False + opt.auto_updates.update_parameters = False + opt.auto_updates.update_vars = False + opt.auto_updates.update_constraints = False + opt.auto_updates.update_named_expressions = False self.opt = opt def test_basics(self): @@ -603,16 +601,13 @@ def test_update1(self): opt = self.opt opt.set_instance(m) - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumQConstrs'), 1) opt.remove_constraints([m.c1]) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 0) + self.assertEqual(opt.get_model_attr('NumQConstrs'), 0) opt.add_constraints([m.c1]) - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 0) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumQConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumQConstrs'), 1) def test_update2(self): m = pyo.ConcreteModel() @@ -625,16 +620,13 @@ def test_update2(self): opt = self.opt opt.config.symbolic_solver_labels = True opt.set_instance(m) - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumConstrs'), 1) opt.remove_constraints([m.c2]) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 0) + self.assertEqual(opt.get_model_attr('NumConstrs'), 0) opt.add_constraints([m.c2]) - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 0) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumConstrs'), 1) + self.assertEqual(opt.get_model_attr('NumConstrs'), 1) def test_update3(self): m = pyo.ConcreteModel() @@ -684,16 +676,13 @@ def test_update5(self): opt = self.opt opt.set_instance(m) - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 1) + self.assertEqual(opt.get_model_attr('NumSOS'), 1) opt.remove_sos_constraints([m.c1]) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 0) + self.assertEqual(opt.get_model_attr('NumSOS'), 0) opt.add_sos_constraints([m.c1]) - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 0) - opt.update() - self.assertEqual(opt._solver_model.getAttr('NumSOS'), 1) + self.assertEqual(opt.get_model_attr('NumSOS'), 1) def test_update6(self): m = pyo.ConcreteModel() From 92fa4f5c72a4d26c5568a40e7ce7726ddd12e991 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:49:29 -0600 Subject: [PATCH 044/104] remove unused imports --- .../solver/solvers/gurobi/gurobi_direct.py | 20 ---------- .../solvers/gurobi/gurobi_direct_base.py | 7 +--- .../solvers/gurobi/gurobi_persistent.py | 38 +++---------------- 3 files changed, 7 insertions(+), 58 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index f4a33e2cc54..16c633c7d7c 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -9,40 +9,20 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ -import datetime -import io -import math import operator -import os from pyomo.common.collections import ComponentMap, ComponentSet -from pyomo.common.config import ConfigValue -from pyomo.common.dependencies import attempt_import -from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import MouseTrap, ApplicationError from pyomo.common.shutdown import python_is_shutting_down -from pyomo.common.tee import capture_output, TeeStream -from pyomo.common.timing import HierarchicalTimer from pyomo.core.staleflag import StaleFlagManager from pyomo.repn.plugins.standard_form import LinearStandardFormCompiler -from pyomo.contrib.solver.common.base import SolverBase, Availability -from pyomo.contrib.solver.common.config import BranchAndBoundConfig from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, NoDualsError, NoReducedCostsError, NoSolutionError, IncompatibleModelError, ) -from pyomo.contrib.solver.common.results import ( - Results, - SolutionStatus, - TerminationCondition, -) from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase -import logging from .gurobi_direct_base import GurobiDirectBase, gurobipy diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index d26dbf54c83..41bdb244743 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -12,14 +12,13 @@ import datetime import io import math -import operator import os -from pyomo.common.collections import ComponentMap, ComponentSet +from pyomo.common.collections import ComponentMap from pyomo.common.config import ConfigValue from pyomo.common.dependencies import attempt_import from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import MouseTrap, ApplicationError +from pyomo.common.errors import ApplicationError from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer @@ -33,14 +32,12 @@ NoDualsError, NoReducedCostsError, NoSolutionError, - IncompatibleModelError, ) from pyomo.contrib.solver.common.results import ( Results, SolutionStatus, TerminationCondition, ) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase import logging diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index e91381f41a3..7b0463d2cf1 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -10,61 +10,33 @@ # ___________________________________________________________________________ from __future__ import annotations -import io import logging -import math -from typing import Dict, List, NoReturn, Optional, Sequence, Mapping +from typing import Dict, List, Optional, Sequence, Mapping from collections.abc import Iterable -from pyomo.common.collections import ComponentSet, ComponentMap, OrderedSet -from pyomo.common.dependencies import attempt_import -from pyomo.common.errors import ApplicationError -from pyomo.common.tee import capture_output, TeeStream +from pyomo.common.collections import ComponentSet, OrderedSet from pyomo.common.timing import HierarchicalTimer -from pyomo.common.shutdown import python_is_shutting_down from pyomo.core.base.objective import ObjectiveData from pyomo.core.kernel.objective import minimize, maximize -from pyomo.core.base import SymbolMap, NumericLabeler, TextLabeler from pyomo.core.base.var import VarData from pyomo.core.base.constraint import ConstraintData, Constraint from pyomo.core.base.sos import SOSConstraintData, SOSConstraint from pyomo.core.base.param import ParamData from pyomo.core.expr.numvalue import value, is_constant, is_fixed, native_numeric_types from pyomo.repn import generate_standard_repn -from pyomo.core.expr.numeric_expr import NPV_MaxExpression, NPV_MinExpression -from pyomo.contrib.solver.common.base import PersistentSolverBase, Availability -from pyomo.contrib.solver.common.results import ( - Results, - TerminationCondition, - SolutionStatus, -) -from pyomo.contrib.solver.common.config import PersistentBranchAndBoundConfig -from pyomo.contrib.solver.common.util import ( - NoFeasibleSolutionError, - NoOptimalSolutionError, - NoDualsError, - NoReducedCostsError, - NoSolutionError, - IncompatibleModelError, -) -from pyomo.contrib.solver.common.persistent import ( - PersistentSolverUtils, - PersistentSolverMixin, -) -from pyomo.contrib.solver.common.solution_loader import PersistentSolutionLoader, SolutionLoaderBase +from pyomo.contrib.solver.common.results import Results +from pyomo.contrib.solver.common.util import IncompatibleModelError +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( - GurobiConfig, GurobiDirectBase, gurobipy, - _load_suboptimal_mip_solution, _load_vars, _get_primals, _get_duals, _get_reduced_costs, ) from pyomo.contrib.solver.common.util import get_objective -from pyomo.repn.quadratic import QuadraticRepn, QuadraticRepnVisitor from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector From 8a9fc46b802dcc181a2b00a131ab089dfc8c59a4 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:50:15 -0600 Subject: [PATCH 045/104] run black --- .../solvers/gurobi/gurobi_direct_base.py | 29 +-- .../solvers/gurobi/gurobi_persistent.py | 213 +++++++++++------- .../solver/tests/solvers/test_solvers.py | 11 +- 3 files changed, 153 insertions(+), 100 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index 41bdb244743..df6bb8b5327 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -87,9 +87,7 @@ def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_ solver_model.getAttr('NumIntVars') == 0 and solver_model.getAttr('NumBinVars') == 0 ): - raise ValueError( - 'Cannot obtain suboptimal solutions for a continuous model' - ) + raise ValueError('Cannot obtain suboptimal solutions for a continuous model') original_solution_number = solver_model.getParamInfo('SolutionNumber')[2] solver_model.setParam('SolutionNumber', solution_number) gurobi_vars_to_load = [var_map[id(v)] for v in vars_to_load] @@ -112,7 +110,7 @@ def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): for v, val in _get_primals( solver_model=solver_model, var_map=var_map, - vars_to_load=vars_to_load, + vars_to_load=vars_to_load, solution_number=solution_number, ).items(): v.set_value(val, skip_validation=True) @@ -177,7 +175,7 @@ def _get_duals(solver_model, con_map, linear_cons_to_load, quadratic_cons_to_loa """ if solver_model.Status != gurobipy.GRB.OPTIMAL: raise NoDualsError() - + linear_gurobi_cons = [con_map[c] for c in linear_cons_to_load] quadratic_gurobi_cons = [con_map[c] for c in quadratic_cons_to_load] linear_vals = solver_model.getAttr("Pi", linear_gurobi_cons) @@ -293,7 +291,7 @@ def _create_solver_model(self, pyomo_model): def _pyomo_gurobi_var_iter(self): # generator of tuples (pyomo_var, gurobi_var) raise NotImplementedError('should be implemented by derived classes') - + def _mipstart(self): for pyomo_var, gurobi_var in self._pyomo_gurobi_var_iter(): if pyomo_var.is_integer() and pyomo_var.value is not None: @@ -304,11 +302,8 @@ def solve(self, model, **kwds) -> Results: orig_config = self.config orig_cwd = os.getcwd() try: - config = self.config( - value=kwds, - preserve_implicit=True, - ) - + config = self.config(value=kwds, preserve_implicit=True) + # hack to work around legacy solver wrapper __setattr__ # otherwise, this would just be self.config = config object.__setattr__(self, 'config', config) @@ -329,7 +324,9 @@ def solve(self, model, **kwds) -> Results: if config.working_dir: os.chdir(config.working_dir) with capture_output(TeeStream(*ostreams), capture_fd=False): - gurobi_model, solution_loader, has_obj = self._create_solver_model(model) + gurobi_model, solution_loader, has_obj = self._create_solver_model( + model + ) options = config.solver_options gurobi_model.setParam('LogToConsole', 1) @@ -354,9 +351,7 @@ def solve(self, model, **kwds) -> Results: timer.stop('optimize') res = self._postsolve( - grb_model=gurobi_model, - solution_loader=solution_loader, - has_obj=has_obj, + grb_model=gurobi_model, solution_loader=solution_loader, has_obj=has_obj ) finally: os.chdir(orig_cwd) @@ -450,9 +445,9 @@ def _postsolve(self, grb_model, solution_loader, has_obj): raise NoFeasibleSolutionError() self.config.timer.stop('load solution') - # self.config gets copied a the beginning of + # self.config gets copied a the beginning of # solve and restored at the end, so modifying - # results.solver_config will not actually + # results.solver_config will not actually # modify self.config results.solver_config = self.config results.solver_name = self.name diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 7b0463d2cf1..6628f001421 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -29,8 +29,8 @@ from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( - GurobiDirectBase, - gurobipy, + GurobiDirectBase, + gurobipy, _load_vars, _get_primals, _get_duals, @@ -45,13 +45,7 @@ class GurobiDirectQuadraticSolutionLoader(SolutionLoaderBase): def __init__( - self, - solver_model, - var_id_map, - var_map, - con_map, - linear_cons, - quadratic_cons, + self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons ) -> None: super().__init__() self._solver_model = solver_model @@ -62,9 +56,7 @@ def __init__( self._quadratic_cons = quadratic_cons def load_vars( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=0, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> None: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -76,9 +68,7 @@ def load_vars( ) def get_primals( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=0, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> Mapping[VarData, float]: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -88,10 +78,9 @@ def get_primals( vars_to_load=vars_to_load, solution_number=solution_id, ) - + def get_reduced_costs( - self, - vars_to_load: Optional[Sequence[VarData]] = None, + self, vars_to_load: Optional[Sequence[VarData]] = None ) -> Mapping[VarData, float]: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -100,10 +89,9 @@ def get_reduced_costs( var_map=self._var_map, vars_to_load=vars_to_load, ) - + def get_duals( - self, - cons_to_load: Optional[Sequence[ConstraintData]] = None, + self, cons_to_load: Optional[Sequence[ConstraintData]] = None ) -> Dict[ConstraintData, float]: if cons_to_load is None: cons_to_load = list(self._con_map.keys()) @@ -124,8 +112,12 @@ def get_duals( class GurobiPersistentSolutionLoader(GurobiDirectQuadraticSolutionLoader): - def __init__(self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons) -> None: - super().__init__(solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons) + def __init__( + self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + ) -> None: + super().__init__( + solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + ) self._valid = True def invalidate(self): @@ -135,19 +127,27 @@ def _assert_solution_still_valid(self): if not self._valid: raise RuntimeError('The results in the solver are no longer valid.') - def load_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=0) -> None: + def load_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 + ) -> None: self._assert_solution_still_valid() return super().load_vars(vars_to_load, solution_id) - - def get_primals(self, vars_to_load: Sequence[VarData] | None = None, solution_id=0) -> Mapping[VarData, float]: + + def get_primals( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 + ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_primals(vars_to_load, solution_id) - def get_duals(self, cons_to_load: Sequence[ConstraintData] | None = None) -> Dict[ConstraintData, float]: + def get_duals( + self, cons_to_load: Sequence[ConstraintData] | None = None + ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return super().get_duals(cons_to_load) - - def get_reduced_costs(self, vars_to_load: Sequence[VarData] | None = None) -> Mapping[VarData, float]: + + def get_reduced_costs( + self, vars_to_load: Sequence[VarData] | None = None + ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_reduced_costs(vars_to_load) @@ -194,7 +194,9 @@ def update(self): class _MutableRangeConstant: - def __init__(self, lhs_expr, rhs_expr, pyomo_con, con_map, slack_name, gurobi_model): + def __init__( + self, lhs_expr, rhs_expr, pyomo_con, con_map, slack_name, gurobi_model + ): self.lhs_expr = lhs_expr self.rhs_expr = rhs_expr self.pyomo_con = pyomo_con @@ -268,7 +270,9 @@ def __init__(self, gurobi_model, constant, linear_coefs, quadratic_coefs): self.constant: _MutableConstant = constant self.linear_coefs: List[_MutableLinearCoefficient] = linear_coefs self.quadratic_coefs: List[_MutableQuadraticCoefficient] = quadratic_coefs - self.last_quadratic_coef_values: List[float] = [value(i.expr) for i in self.quadratic_coefs] + self.last_quadratic_coef_values: List[float] = [ + value(i.expr) for i in self.quadratic_coefs + ] def get_updated_expression(self): for ndx, coef in enumerate(self.linear_coefs): @@ -300,7 +304,7 @@ def __init__(self, expr, v1id, v2id, var_map): @property def var1(self): return self.var_map[self.v1id] - + @property def var2(self): return self.var_map[self.v2id] @@ -325,13 +329,21 @@ def _create_solver_model(self, pyomo_model): self._clear() self._solver_model = gurobipy.Model(env=self.env()) timer.start('collect constraints') - cons = list(pyomo_model.component_data_objects(Constraint, descend_into=True, active=True)) + cons = list( + pyomo_model.component_data_objects( + Constraint, descend_into=True, active=True + ) + ) timer.stop('collect constraints') timer.start('translate constraints') self._add_constraints(cons) timer.stop('translate constraints') timer.start('sos') - sos = list(pyomo_model.component_data_objects(SOSConstraint, descend_into=True, active=True)) + sos = list( + pyomo_model.component_data_objects( + SOSConstraint, descend_into=True, active=True + ) + ) self._add_sos_constraints(sos) timer.stop('sos') timer.start('get objective') @@ -351,7 +363,7 @@ def _create_solver_model(self, pyomo_model): ) timer.stop('create gurobipy model') return self._solver_model, solution_loader, has_obj - + def _clear(self): self._solver_model = None self._vars = {} @@ -379,9 +391,7 @@ def _process_domain_and_bounds(self, var): else: vtype = gurobipy.GRB.INTEGER else: - raise ValueError( - f'Unrecognized domain: {var.domain}' - ) + raise ValueError(f'Unrecognized domain: {var.domain}') if var.fixed: lb = var.value ub = lb @@ -415,16 +425,13 @@ def _get_expr_from_pyomo_repn(self, repn): raise IncompatibleModelError( f'GurobiDirectQuadratic only supports linear and quadratic expressions: {expr}.' ) - + if len(repn.linear_vars) > 0: missing_vars = [v for v in repn.linear_vars if id(v) not in self._vars] self._add_variables(missing_vars) coef_list = [value(i) for i in repn.linear_coefs] vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] - new_expr = gurobipy.LinExpr( - coef_list, - vlist, - ) + new_expr = gurobipy.LinExpr(coef_list, vlist) else: new_expr = 0.0 @@ -455,8 +462,7 @@ def _add_constraints(self, cons: List[ConstraintData]): gurobi_expr = self._get_expr_from_pyomo_repn(repn) if lb is None and ub is None: raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" + "Constraint does not have a lower " f"or an upper bound: {con} \n" ) elif lb is None: gurobi_expr_list.append(gurobi_expr <= float(ub - repn.constant)) @@ -465,9 +471,14 @@ def _add_constraints(self, cons: List[ConstraintData]): elif lb == ub: gurobi_expr_list.append(gurobi_expr == float(lb - repn.constant)) else: - gurobi_expr_list.append(gurobi_expr == [float(lb-repn.constant), float(ub-repn.constant)]) + gurobi_expr_list.append( + gurobi_expr + == [float(lb - repn.constant), float(ub - repn.constant)] + ) - gurobi_cons = self._solver_model.addConstrs((gurobi_expr_list[i] for i in range(len(gurobi_expr_list)))).values() + gurobi_cons = self._solver_model.addConstrs( + (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))) + ).values() self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) def _add_sos_constraints(self, cons: List[SOSConstraintData]): @@ -485,7 +496,9 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): gurobi_vars = [] weights = [] - missing_vars = {id(v): v for v, w in con.get_items() if id(v) not in self._vars} + missing_vars = { + id(v): v for v, w in con.get_items() if id(v) not in self._vars + } self._add_variables(list(missing_vars.values())) for v, w in con.get_items(): @@ -608,7 +621,7 @@ def _create_solver_model(self, pyomo_model): ) has_obj = self._objective is not None return self._solver_model, solution_loader, has_obj - + def release_license(self): self._clear() self.__class__.release_license() @@ -617,17 +630,21 @@ def solve(self, model, **kwds) -> Results: res = super().solve(model, **kwds) self._needs_updated = False return res - + def _process_domain_and_bounds(self, var): res = super()._process_domain_and_bounds(var) if not is_constant(var._lb): - mutable_lb = _MutableLowerBound(id(var), var.lower, self._pyomo_var_to_solver_var_map) + mutable_lb = _MutableLowerBound( + id(var), var.lower, self._pyomo_var_to_solver_var_map + ) self._mutable_bounds[id(var), 'lb'] = (var, mutable_lb) if not is_constant(var._ub): - mutable_ub = _MutableUpperBound(id(var), var.upper, self._pyomo_var_to_solver_var_map) + mutable_ub = _MutableUpperBound( + id(var), var.upper, self._pyomo_var_to_solver_var_map + ) self._mutable_bounds[id(var), 'ub'] = (var, mutable_ub) return res - + def _add_variables(self, variables: List[VarData]): self._invalidate_last_results() super()._add_variables(variables) @@ -673,37 +690,60 @@ def _add_constraints(self, cons: List[ConstraintData]): mutable_constant = None if lb is None and ub is None: raise ValueError( - "Constraint does not have a lower " - f"or an upper bound: {con} \n" + "Constraint does not have a lower " f"or an upper bound: {con} \n" ) elif lb is None: rhs_expr = ub - repn.constant gurobi_expr_list.append(gurobi_expr <= float(value(rhs_expr))) if not is_constant(rhs_expr): - mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) elif ub is None: rhs_expr = lb - repn.constant gurobi_expr_list.append(float(value(rhs_expr)) <= gurobi_expr) if not is_constant(rhs_expr): - mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) elif con.equality: rhs_expr = lb - repn.constant gurobi_expr_list.append(gurobi_expr == float(value(rhs_expr))) if not is_constant(rhs_expr): - mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) else: - assert len(repn.quadratic_vars) == 0, "Quadratic range constraints are not supported" + assert ( + len(repn.quadratic_vars) == 0 + ), "Quadratic range constraints are not supported" lhs_expr = lb - repn.constant rhs_expr = ub - repn.constant - gurobi_expr_list.append(gurobi_expr == [float(value(lhs_expr)), float(value(rhs_expr))]) + gurobi_expr_list.append( + gurobi_expr == [float(value(lhs_expr)), float(value(rhs_expr))] + ) if not is_constant(lhs_expr) or not is_constant(rhs_expr): conname = f'c{self._constraint_ndx}[{ndx}]' - mutable_constant = _MutableRangeConstant(lhs_expr, rhs_expr, con, self._pyomo_con_to_solver_con_map, 'Rg' + conname, self._solver_model) + mutable_constant = _MutableRangeConstant( + lhs_expr, + rhs_expr, + con, + self._pyomo_con_to_solver_con_map, + 'Rg' + conname, + self._solver_model, + ) mlc_list = [] for c, v in zip(repn.linear_coefs, repn.linear_vars): if not is_constant(c): - mlc = _MutableLinearCoefficient(c, con, self._pyomo_con_to_solver_con_map, id(v), self._pyomo_var_to_solver_var_map, self._solver_model) + mlc = _MutableLinearCoefficient( + c, + con, + self._pyomo_con_to_solver_con_map, + id(v), + self._pyomo_var_to_solver_var_map, + self._solver_model, + ) mlc_list.append(mlc) if len(repn.quadratic_vars) == 0: @@ -715,15 +755,19 @@ def _add_constraints(self, cons: List[ConstraintData]): self._mutable_helpers[con].append(mutable_constant) else: if mutable_constant is None: - mutable_constant = _MutableConstant(rhs_expr, con, self._pyomo_con_to_solver_con_map) + mutable_constant = _MutableConstant( + rhs_expr, con, self._pyomo_con_to_solver_con_map + ) mqc_list = [] for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): if not is_constant(coef): - mqc = _MutableQuadraticCoefficient(coef, id(x), id(y), self._pyomo_var_to_solver_var_map) + mqc = _MutableQuadraticCoefficient( + coef, id(x), id(y), self._pyomo_var_to_solver_var_map + ) mqc_list.append(mqc) mqc = _MutableQuadraticConstraint( self._solver_model, - con, + con, self._pyomo_con_to_solver_con_map, mutable_constant, mlc_list, @@ -731,10 +775,12 @@ def _add_constraints(self, cons: List[ConstraintData]): ) self._mutable_quadratic_helpers[con] = mqc - gurobi_cons = list(self._solver_model.addConstrs( - (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))), - name=f'c{self._constraint_ndx}' - ).values()) + gurobi_cons = list( + self._solver_model.addConstrs( + (gurobi_expr_list[i] for i in range(len(gurobi_expr_list))), + name=f'c{self._constraint_ndx}', + ).values() + ) self._constraint_ndx += 1 self._pyomo_con_to_solver_con_map.update(zip(cons, gurobi_cons)) self._constraints_added_since_update.update(cons) @@ -761,7 +807,9 @@ def _set_objective(self, obj): else: raise ValueError(f'Objective sense is not recognized: {obj.sense}') - repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=False) + repn = generate_standard_repn( + obj.expr, quadratic=True, compute_values=False + ) repn_constant = value(repn.constant) gurobi_expr = self._get_expr_from_pyomo_repn(repn) @@ -770,16 +818,27 @@ def _set_objective(self, obj): mlc_list = [] for c, v in zip(repn.linear_coefs, repn.linear_vars): if not is_constant(c): - mlc = _MutableLinearCoefficient(c, None, None, id(v), self._pyomo_var_to_solver_var_map, self._solver_model) + mlc = _MutableLinearCoefficient( + c, + None, + None, + id(v), + self._pyomo_var_to_solver_var_map, + self._solver_model, + ) mlc_list.append(mlc) mqc_list = [] for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): if not is_constant(coef): - mqc = _MutableQuadraticCoefficient(coef, id(x), id(y), self._pyomo_var_to_solver_var_map) + mqc = _MutableQuadraticCoefficient( + coef, id(x), id(y), self._pyomo_var_to_solver_var_map + ) mqc_list.append(mqc) - self._mutable_objective = _MutableObjective(self._solver_model, mutable_constant, mlc_list, mqc_list) + self._mutable_objective = _MutableObjective( + self._solver_model, mutable_constant, mlc_list, mqc_list + ) # hack # see PR #2454 @@ -865,9 +924,7 @@ def _update_parameters(self, params: List[ParamData]): new_rhs = helper.get_updated_rhs() new_sense = gurobi_con.qcsense self._solver_model.remove(gurobi_con) - new_con = self._solver_model.addQConstr( - new_gurobi_expr, new_sense, new_rhs, - ) + new_con = self._solver_model.addQConstr(new_gurobi_expr, new_sense, new_rhs) self._pyomo_con_to_solver_con_map[con] = new_con helper.pyomo_con = con self._constraints_added_since_update.add(con) @@ -880,7 +937,7 @@ def _update_parameters(self, params: List[ParamData]): else: sense = gurobipy.GRB.MAXIMIZE # TODO: need a test for when part of the object is linear - # and part of the objective is quadratic, but both + # and part of the objective is quadratic, but both # parts have mutable coefficients self._solver_model.setObjective(new_gurobi_expr, sense=sense) @@ -1309,4 +1366,4 @@ def update_variables(self, variables): self._change_detector.update_variables(variables) def update_parameters(self, params): - self._change_detector.update_parameters(params) \ No newline at end of file + self._change_detector.update_parameters(params) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index a0d87835e13..96e2e7b2c38 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -31,7 +31,10 @@ from pyomo.contrib.solver.common.factory import SolverFactory from pyomo.contrib.solver.solvers.ipopt import Ipopt from pyomo.contrib.solver.solvers.gurobi.gurobi_direct import GurobiDirect -from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import ( + GurobiDirectQuadratic, + GurobiPersistent, +) from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -59,11 +62,9 @@ ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), ] -nlp_solvers = [ - ('ipopt', Ipopt), -] +nlp_solvers = [('ipopt', Ipopt)] qcp_solvers = [ - ('gurobi_persistent', GurobiPersistent), + ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ] From 7249b1941c6b3aad5cd3f7a3bd34f7ab28ef3566 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 15:57:56 -0600 Subject: [PATCH 046/104] update solution loader --- .../solvers/gurobi/gurobi_persistent.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 6628f001421..05acfef2b4f 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -15,6 +15,7 @@ from collections.abc import Iterable from pyomo.common.collections import ComponentSet, OrderedSet +from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.timing import HierarchicalTimer from pyomo.core.base.objective import ObjectiveData from pyomo.core.kernel.objective import minimize, maximize @@ -54,6 +55,25 @@ def __init__( self._con_map = con_map self._linear_cons = linear_cons self._quadratic_cons = quadratic_cons + GurobiDirectBase._register_env_client() + + def __del__(self): + if python_is_shutting_down(): + return + # Free the associated model + if self._solver_model is not None: + self._vars = None + self._var_map = None + self._con_map = None + self._linear_cons = None + self._quadratic_cons = None + # explicitly release the model + self._solver_model.dispose() + self._solver_model = None + # Release the gurobi license if this is the last reference to + # the environment (either through a results object or solver + # interface) + GurobiDirectBase._release_env_client() def load_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 From ac42345de6f87fe5010af92bbdf2cf7d772d95d4 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 17:41:27 -0600 Subject: [PATCH 047/104] updating solution loader --- pyomo/contrib/solver/common/base.py | 7 +-- .../contrib/solver/common/solution_loader.py | 47 +++++++++++++++++-- .../solver/solvers/gurobi/gurobi_direct.py | 32 +++++++++---- pyomo/contrib/solver/solvers/highs.py | 4 +- pyomo/contrib/solver/solvers/ipopt.py | 36 ++++++-------- pyomo/contrib/solver/solvers/sol_reader.py | 30 +++++++++--- 6 files changed, 107 insertions(+), 49 deletions(-) diff --git a/pyomo/contrib/solver/common/base.py b/pyomo/contrib/solver/common/base.py index 280b80629a3..f935f3d4988 100644 --- a/pyomo/contrib/solver/common/base.py +++ b/pyomo/contrib/solver/common/base.py @@ -567,12 +567,7 @@ def _solution_handler( legacy_results._smap_id = id(symbol_map) delete_legacy_soln = True if load_solutions: - if hasattr(model, 'dual') and model.dual.import_enabled(): - for con, val in results.solution_loader.get_duals().items(): - model.dual[con] = val - if hasattr(model, 'rc') and model.rc.import_enabled(): - for var, val in results.solution_loader.get_reduced_costs().items(): - model.rc[var] = val + results.solution_loader.load_import_suffixes() elif results.incumbent_objective is not None: delete_legacy_soln = False for var, val in results.solution_loader.get_primals().items(): diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index 065c00185f6..e399d6bea55 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -9,11 +9,32 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +from __future__ import annotations + from typing import Sequence, Dict, Optional, Mapping, List, Any from pyomo.core.base.constraint import ConstraintData from pyomo.core.base.var import VarData from pyomo.core.staleflag import StaleFlagManager +from pyomo.core.base.suffix import Suffix + + +def load_import_suffixes(pyomo_model, solution_loader: SolutionLoaderBase, solution_id=None): + dual_suffix = None + rc_suffix = None + for suffix in pyomo_model.component_objects(Suffix, descend_into=True, active=True): + if not suffix.import_enabled(): + continue + if suffix.local_name == 'dual': + dual_suffix = suffix + elif suffix.local_name == 'rc': + rc_suffix = suffix + if dual_suffix is not None: + for k, v in solution_loader.get_duals(solution_id=solution_id).items(): + dual_suffix[k] = v + if rc_suffix is not None: + for k, v in solution_loader.get_reduced_costs(solution_id=solution_id).items(): + rc_suffix[k] = v class SolutionLoaderBase: @@ -178,29 +199,45 @@ class PersistentSolutionLoader(SolutionLoaderBase): Loader for persistent solvers """ - def __init__(self, solver): + def __init__(self, solver, pyomo_model): self._solver = solver self._valid = True + self._pyomo_model = pyomo_model def _assert_solution_still_valid(self): if not self._valid: raise RuntimeError('The results in the solver are no longer valid.') - def get_primals(self, vars_to_load=None): + def get_solution_ids(self) -> List[Any]: + self._assert_solution_still_valid() + return super().get_solution_ids() + + def get_number_of_solutions(self) -> int: + self._assert_solution_still_valid() + return super().get_number_of_solutions() + + def get_vars(self, vars_to_load=None, solution_id=None): self._assert_solution_still_valid() - return self._solver._get_primals(vars_to_load=vars_to_load) + return self._solver._get_primals(vars_to_load=vars_to_load, solution_id=solution_id) def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None + self, + cons_to_load: Optional[Sequence[ConstraintData]] = None, + solution_id=None, ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return self._solver._get_duals(cons_to_load=cons_to_load) def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return self._solver._get_reduced_costs(vars_to_load=vars_to_load) + def load_import_suffixes(self, solution_id=None): + load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) + def invalidate(self): self._valid = False diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index 16c633c7d7c..cca23315b1a 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -10,6 +10,7 @@ # ___________________________________________________________________________ import operator +from typing import List from pyomo.common.collections import ComponentMap, ComponentSet from pyomo.common.shutdown import python_is_shutting_down @@ -22,17 +23,18 @@ NoSolutionError, IncompatibleModelError, ) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes from .gurobi_direct_base import GurobiDirectBase, gurobipy class GurobiDirectSolutionLoader(SolutionLoaderBase): - def __init__(self, grb_model, grb_cons, grb_vars, pyo_cons, pyo_vars): + def __init__(self, grb_model, grb_cons, grb_vars, pyo_cons, pyo_vars, pyomo_model): self._grb_model = grb_model self._grb_cons = grb_cons self._grb_vars = grb_vars self._pyo_cons = pyo_cons self._pyo_vars = pyo_vars + self._pyomo_model = pyomo_model GurobiDirectBase._register_env_client() def __del__(self): @@ -44,6 +46,7 @@ def __del__(self): self._grb_vars = None self._pyo_cons = None self._pyo_vars = None + self._pyomo_model = None # explicitly release the model self._grb_model.dispose() self._grb_model = None @@ -52,8 +55,16 @@ def __del__(self): # interface) GurobiDirectBase._release_env_client() - def load_vars(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 + def get_number_of_solutions(self) -> int: + if self._grb_model.SolCount == 0: + return 0 + return 1 + + def get_solution_ids(self) -> List[Any]: + return [0] + + def load_vars(self, vars_to_load=None, solution_id=0): + assert solution_id == 0 if self._grb_model.SolCount == 0: raise NoSolutionError() @@ -65,8 +76,8 @@ def load_vars(self, vars_to_load=None, solution_number=0): p_var.set_value(g_var, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) - def get_primals(self, vars_to_load=None, solution_number=0): - assert solution_number == 0 + def get_vars(self, vars_to_load=None, solution_id=0): + assert solution_id == 0 if self._grb_model.SolCount == 0: raise NoSolutionError() @@ -76,7 +87,8 @@ def get_primals(self, vars_to_load=None, solution_number=0): iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) return ComponentMap(iterator) - def get_duals(self, cons_to_load=None): + def get_duals(self, cons_to_load=None, solution_id=0): + assert solution_id == 0 if self._grb_model.Status != gurobipy.GRB.OPTIMAL: raise NoDualsError() @@ -96,7 +108,8 @@ def dedup(_iter): ) return {con_info[0]: dual for con_info, dual in iterator} - def get_reduced_costs(self, vars_to_load=None): + def get_reduced_costs(self, vars_to_load=None, solution_id=0): + assert solution_id == 0 if self._grb_model.Status != gurobipy.GRB.OPTIMAL: raise NoReducedCostsError() @@ -105,6 +118,9 @@ def get_reduced_costs(self, vars_to_load=None): vars_to_load = ComponentSet(vars_to_load) iterator = filter(lambda var_rc: var_rc[0] in vars_to_load, iterator) return ComponentMap(iterator) + + def load_import_suffixes(self, solution_id=None): + load_import_suffixes(pyomo_model=self._pyomo_model, solution_loader=self, solution_id=solution_id) class GurobiDirect(GurobiDirectBase): diff --git a/pyomo/contrib/solver/solvers/highs.py b/pyomo/contrib/solver/solvers/highs.py index 6eb4afa828a..a1d609c5db6 100644 --- a/pyomo/contrib/solver/solvers/highs.py +++ b/pyomo/contrib/solver/solvers/highs.py @@ -675,7 +675,7 @@ def _postsolve(self): status = highs.getModelStatus() results = Results() - results.solution_loader = PersistentSolutionLoader(self) + results.solution_loader = PersistentSolutionLoader(self, self._model) results.timing_info.highs_time = highs.getRunTime() self._sol = highs.getSolution() @@ -751,7 +751,7 @@ def _postsolve(self): if config.load_solutions: if has_feasible_solution: - self._load_vars() + results.solution_loader.load_solution() else: raise NoFeasibleSolutionError() timer.stop('load solution') diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 075fc998ecc..441b8eb5f61 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -109,9 +109,13 @@ def _error_check(self): ) def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None + self, + vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: self._error_check() + if solution_id is not None: + raise ValueError('IpoptSolutionLoader does not support solution_id') if self._nl_info.scaling is None: scale_list = [1] * len(self._nl_info.variables) obj_scale = 1 @@ -430,35 +434,35 @@ def solve(self, model, **kwds) -> Results: if proven_infeasible: results = Results() results.termination_condition = TerminationCondition.provenInfeasible - results.solution_loader = SolSolutionLoader(None, None) + results.solution_loader = SolSolutionLoader(None, None, model) results.iteration_count = 0 results.timing_info.total_seconds = 0 elif len(nl_info.variables) == 0: if len(nl_info.eliminated_vars) == 0: results = Results() results.termination_condition = TerminationCondition.emptyModel - results.solution_loader = SolSolutionLoader(None, None) + results.solution_loader = SolSolutionLoader(None, None, model) else: results = Results() results.termination_condition = ( TerminationCondition.convergenceCriteriaSatisfied ) results.solution_status = SolutionStatus.optimal - results.solution_loader = SolSolutionLoader(None, nl_info=nl_info) + results.solution_loader = SolSolutionLoader(None, nl_info=nl_info, pyomo_model=model) results.iteration_count = 0 results.timing_info.total_seconds = 0 else: if os.path.isfile(basename + '.sol'): with open(basename + '.sol', 'r', encoding='utf-8') as sol_file: timer.start('parse_sol') - results = self._parse_solution(sol_file, nl_info) + results = self._parse_solution(sol_file, nl_info, model) timer.stop('parse_sol') else: results = Results() if process.returncode != 0: results.extra_info.return_code = process.returncode results.termination_condition = TerminationCondition.error - results.solution_loader = SolSolutionLoader(None, None) + results.solution_loader = SolSolutionLoader(None, None, model) else: try: results.iteration_count = parsed_output_data.pop('iters') @@ -490,19 +494,7 @@ def solve(self, model, **kwds) -> Results: if config.load_solutions: if results.solution_status == SolutionStatus.noSolution: raise NoFeasibleSolutionError() - results.solution_loader.load_vars() - if ( - hasattr(model, 'dual') - and isinstance(model.dual, Suffix) - and model.dual.import_enabled() - ): - model.dual.update(results.solution_loader.get_duals()) - if ( - hasattr(model, 'rc') - and isinstance(model.rc, Suffix) - and model.rc.import_enabled() - ): - model.rc.update(results.solution_loader.get_reduced_costs()) + results.solution_loader.load_solution() if ( results.solution_status in {SolutionStatus.feasible, SolutionStatus.optimal} @@ -665,7 +657,7 @@ def _parse_ipopt_output(self, output: Union[str, io.StringIO]) -> Dict[str, Any] return parsed_data def _parse_solution( - self, instream: io.TextIOBase, nl_info: NLWriterInfo + self, instream: io.TextIOBase, nl_info: NLWriterInfo, pyomo_model ) -> Results: results = Results() res, sol_data = parse_sol_file( @@ -673,10 +665,10 @@ def _parse_solution( ) if res.solution_status == SolutionStatus.noSolution: - res.solution_loader = SolSolutionLoader(None, None) + res.solution_loader = SolSolutionLoader(None, None, pyomo_model=pyomo_model) else: res.solution_loader = IpoptSolutionLoader( - sol_data=sol_data, nl_info=nl_info + sol_data=sol_data, nl_info=nl_info, pyomo_model=pyomo_model, ) return res diff --git a/pyomo/contrib/solver/solvers/sol_reader.py b/pyomo/contrib/solver/solvers/sol_reader.py index e580e2a72f9..7570a1ffc53 100644 --- a/pyomo/contrib/solver/solvers/sol_reader.py +++ b/pyomo/contrib/solver/solvers/sol_reader.py @@ -26,7 +26,7 @@ SolutionStatus, TerminationCondition, ) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes class SolFileData: @@ -49,11 +49,25 @@ class SolSolutionLoader(SolutionLoaderBase): Loader for solvers that create .sol files (e.g., ipopt) """ - def __init__(self, sol_data: SolFileData, nl_info: NLWriterInfo) -> None: + def __init__(self, sol_data: SolFileData, nl_info: NLWriterInfo, pyomo_model) -> None: self._sol_data = sol_data self._nl_info = nl_info + self._pyomo_model = pyomo_model - def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None) -> NoReturn: + def get_number_of_solutions(self) -> int: + if self._nl_info is None: + return 0 + return 1 + + def get_solution_ids(self) -> List[Any]: + return [None] + + def load_import_suffixes(self, solution_id=None): + load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) + + def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None) -> NoReturn: + if solution_id is not None: + raise ValueError(f'{self.__class__.__name__} does not support solution_id') if self._nl_info is None: raise RuntimeError( 'Solution loader does not currently have a valid solution. Please ' @@ -78,9 +92,11 @@ def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None) -> NoRetur StaleFlagManager.mark_all_as_stale(delayed=True) - def get_primals( - self, vars_to_load: Optional[Sequence[VarData]] = None + def get_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: + if solution_id is not None: + raise ValueError(f'{self.__class__.__name__} does not support solution_id') if self._nl_info is None: raise RuntimeError( 'Solution loader does not currently have a valid solution. Please ' @@ -115,8 +131,10 @@ def get_primals( return res def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None, ) -> Dict[ConstraintData, float]: + if solution_id is not None: + raise ValueError(f'{self.__class__.__name__} does not support solution_id') if self._nl_info is None: raise RuntimeError( 'Solution loader does not currently have a valid solution. Please ' From 275d848d2c5eb09db7ac6b519794e1a0065b5869 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 12 Aug 2025 17:43:11 -0600 Subject: [PATCH 048/104] run black --- pyomo/contrib/observer/component_collector.py | 5 +++- pyomo/contrib/observer/model_observer.py | 24 ++++++++-------- .../observer/tests/test_change_detector.py | 28 ++++++++++--------- 3 files changed, 32 insertions(+), 25 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index 5cbbdaf31bd..d52ec46086c 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -10,7 +10,10 @@ # ___________________________________________________________________________ from pyomo.core.expr.visitor import StreamBasedExpressionVisitor -from pyomo.core.expr.numeric_expr import ExternalFunctionExpression, NPV_ExternalFunctionExpression +from pyomo.core.expr.numeric_expr import ( + ExternalFunctionExpression, + NPV_ExternalFunctionExpression, +) from pyomo.core.base.var import VarData, ScalarVar from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.expression import ExpressionData, ScalarExpression diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index bd905e1c61d..4ab52100376 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -208,10 +208,7 @@ def update_parameters(self, params: List[ParamData]): class ModelChangeDetector: - def __init__( - self, observers: Sequence[Observer], - **kwds, - ): + def __init__(self, observers: Sequence[Observer], **kwds): """ Parameters ---------- @@ -237,13 +234,15 @@ def __init__( ) # var_id: [dict[constraints, None], dict[sos constraints, None], None or objective] self._referenced_params = ( {} - ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] + ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] self._vars_referenced_by_con = {} self._vars_referenced_by_obj = [] self._params_referenced_by_con = {} self._params_referenced_by_obj = [] self._expr_types = None - self.config: AutoUpdateConfig = AutoUpdateConfig()(value=kwds, preserve_implicit=True) + self.config: AutoUpdateConfig = AutoUpdateConfig()( + value=kwds, preserve_implicit=True + ) def set_instance(self, model): saved_config = self.config @@ -347,7 +346,10 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): if con in self._active_sos: raise ValueError(f'Constraint {con.name} has already been added') sos_items = list(con.get_items()) - self._active_sos[con] = ([i[0] for i in sos_items], [i[1] for i in sos_items]) + self._active_sos[con] = ( + [i[0] for i in sos_items], + [i[1] for i in sos_items], + ) variables = [] params = [] for v, p in sos_items: @@ -616,14 +618,14 @@ def _check_for_var_changes(self): vars_to_update.append(v) cons_to_update = list(cons_to_update.keys()) return vars_to_update, cons_to_update, update_obj - + def _check_for_param_changes(self): params_to_update = [] for pid, (p, val) in self._params.items(): if p.value != val: params_to_update.append(p) return params_to_update - + def _check_for_named_expression_changes(self): cons_to_update = [] for con, ne_list in self._named_expressions.items(): @@ -644,7 +646,7 @@ def _check_for_new_objective(self): new_obj = get_objective(self._model) if new_obj is not self._objective: update_obj = True - return new_obj, update_obj + return new_obj, update_obj def _check_for_objective_changes(self): update_obj = False @@ -717,7 +719,7 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if update_obj: need_to_set_objective = True timer.stop('named expressions') - + timer.start('objective') new_obj = self._objective if config.check_for_new_objective: diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index efda8a181d9..29e0de01eb9 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -6,7 +6,11 @@ import pyomo.environ as pe from pyomo.common import unittest from typing import List -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig +from pyomo.contrib.observer.model_observer import ( + Observer, + ModelChangeDetector, + AutoUpdateConfig, +) from pyomo.common.collections import ComponentMap import logging @@ -31,11 +35,9 @@ def __init__(self): def check(self, expected): unittest.assertStructuredAlmostEqual( - first=expected, - second=self.counts, - places=7, + first=expected, second=self.counts, places=7 ) - + def _process(self, comps, key): for c in comps: if c not in self.counts: @@ -120,7 +122,7 @@ def test_objective(self): detector.set_instance(m) obs.check(expected) - m.obj = pe.Objective(expr=m.x**2 + m.p*m.y**2) + m.obj = pe.Objective(expr=m.x**2 + m.p * m.y**2) detector.update() expected[m.obj] = make_count_dict() expected[m.obj]['set'] += 1 @@ -131,7 +133,7 @@ def test_objective(self): expected[m.p] = make_count_dict() expected[m.p]['add'] += 1 obs.check(expected) - + m.y.setlb(0) detector.update() expected[m.y]['update'] += 1 @@ -161,7 +163,7 @@ def test_objective(self): obs.check(expected) del m.obj - m.obj = pe.Objective(expr=m.p*m.x) + m.obj = pe.Objective(expr=m.p * m.x) detector.update() expected[m.p]['add'] += 1 expected[m.y]['remove'] += 1 @@ -186,7 +188,7 @@ def test_constraints(self): obs.check(expected) m.obj = pe.Objective(expr=m.y) - m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p)**2) + m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p) ** 2) detector.update() expected[m.x] = make_count_dict() expected[m.y] = make_count_dict() @@ -208,9 +210,9 @@ def test_constraints(self): obs.pprint() expected[m.c1]['remove'] += 1 expected[m.c1]['add'] += 1 - # because x and p are only used in the - # one constraint, they get removed when - # the constraint is removed and then + # because x and p are only used in the + # one constraint, they get removed when + # the constraint is removed and then # added again when the constraint is added expected[m.x]['update'] += 1 expected[m.x]['remove'] += 1 @@ -220,4 +222,4 @@ def test_constraints(self): obs.check(expected) def test_vars_and_params_elsewhere(self): - pass \ No newline at end of file + pass From 70ca6e72d50ac39a3c580f116dfd6c75a9b29e92 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 07:20:44 -0600 Subject: [PATCH 049/104] updating solution loader --- .../solver/solvers/gurobi/gurobi_direct.py | 2 +- .../solvers/gurobi/gurobi_direct_base.py | 2 +- .../solvers/gurobi/gurobi_persistent.py | 44 ++++++++++++++----- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index cca23315b1a..82b47ccb24b 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -193,7 +193,7 @@ def _create_solver_model(self, pyomo_model): self._gurobi_vars = x solution_loader = GurobiDirectSolutionLoader( - gurobi_model, A, x, repn.rows, repn.columns + gurobi_model, A, x, repn.rows, repn.columns, pyomo_model ) has_obj = len(repn.objectives) > 0 diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index df6bb8b5327..ae887a52fa5 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -440,7 +440,7 @@ def _postsolve(self, grb_model, solution_loader, has_obj): self.config.timer.start('load solution') if self.config.load_solutions: if grb_model.SolCount > 0: - results.solution_loader.load_vars() + results.solution_loader.load_solution() else: raise NoFeasibleSolutionError() self.config.timer.stop('load solution') diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 05acfef2b4f..8477d855a02 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -27,7 +27,7 @@ from pyomo.repn import generate_standard_repn from pyomo.contrib.solver.common.results import Results from pyomo.contrib.solver.common.util import IncompatibleModelError -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( GurobiDirectBase, @@ -46,7 +46,7 @@ class GurobiDirectQuadraticSolutionLoader(SolutionLoaderBase): def __init__( - self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model ) -> None: super().__init__() self._solver_model = solver_model @@ -55,6 +55,7 @@ def __init__( self._con_map = con_map self._linear_cons = linear_cons self._quadratic_cons = quadratic_cons + self._pyomo_model = pyomo_model GurobiDirectBase._register_env_client() def __del__(self): @@ -75,6 +76,12 @@ def __del__(self): # interface) GurobiDirectBase._release_env_client() + def get_number_of_solutions(self) -> int: + return self._solver_model.SolCount + + def get_solution_ids(self) -> List: + return list(range(self.get_number_of_solutions())) + def load_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> None: @@ -87,7 +94,7 @@ def load_vars( solution_number=solution_id, ) - def get_primals( + def get_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> Mapping[VarData, float]: if vars_to_load is None: @@ -100,7 +107,7 @@ def get_primals( ) def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 ) -> Mapping[VarData, float]: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -111,7 +118,7 @@ def get_reduced_costs( ) def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=0 ) -> Dict[ConstraintData, float]: if cons_to_load is None: cons_to_load = list(self._con_map.keys()) @@ -130,13 +137,16 @@ def get_duals( quadratic_cons_to_load=quadratic_cons_to_load, ) + def load_import_suffixes(self, solution_id=None): + load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) + class GurobiPersistentSolutionLoader(GurobiDirectQuadraticSolutionLoader): def __init__( - self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model ) -> None: super().__init__( - solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons + solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model ) self._valid = True @@ -153,23 +163,35 @@ def load_vars( self._assert_solution_still_valid() return super().load_vars(vars_to_load, solution_id) - def get_primals( + def get_vars( self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_primals(vars_to_load, solution_id) def get_duals( - self, cons_to_load: Sequence[ConstraintData] | None = None + self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=0, ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return super().get_duals(cons_to_load) def get_reduced_costs( - self, vars_to_load: Sequence[VarData] | None = None + self, vars_to_load: Sequence[VarData] | None = None, solution_id=0, ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_reduced_costs(vars_to_load) + + def get_number_of_solutions(self) -> int: + self._assert_solution_still_valid() + return super().get_number_of_solutions() + + def get_solution_ids(self) -> List: + self._assert_solution_still_valid() + return super().get_solution_ids() + + def load_import_suffixes(self, solution_id=None): + self._assert_solution_still_valid() + super().load_import_suffixes(solution_id) class _MutableLowerBound: @@ -380,6 +402,7 @@ def _create_solver_model(self, pyomo_model): con_map=self._pyomo_con_to_solver_con_map, linear_cons=self._linear_cons, quadratic_cons=self._quadratic_cons, + pyomo_model=pyomo_model, ) timer.stop('create gurobipy model') return self._solver_model, solution_loader, has_obj @@ -638,6 +661,7 @@ def _create_solver_model(self, pyomo_model): con_map=self._pyomo_con_to_solver_con_map, linear_cons=self._linear_cons, quadratic_cons=self._quadratic_cons, + pyomo_model=pyomo_model, ) has_obj = self._objective is not None return self._solver_model, solution_loader, has_obj From 1788ff371d52448a16b539c868aa40142c416b73 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 07:23:17 -0600 Subject: [PATCH 050/104] dont free gurobi models twice --- .../solver/solvers/gurobi/gurobi_persistent.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 05acfef2b4f..847ec958bdd 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -58,18 +58,6 @@ def __init__( GurobiDirectBase._register_env_client() def __del__(self): - if python_is_shutting_down(): - return - # Free the associated model - if self._solver_model is not None: - self._vars = None - self._var_map = None - self._con_map = None - self._linear_cons = None - self._quadratic_cons = None - # explicitly release the model - self._solver_model.dispose() - self._solver_model = None # Release the gurobi license if this is the last reference to # the environment (either through a results object or solver # interface) From 2885f42e665a6fc87c854d60984145dffd86547a Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 08:07:54 -0600 Subject: [PATCH 051/104] update solution loader --- pyomo/contrib/solver/common/base.py | 2 +- .../solver/solvers/gurobi/gurobi_direct.py | 18 +++++++------- .../solvers/gurobi/gurobi_direct_base.py | 8 +++---- .../solvers/gurobi/gurobi_persistent.py | 22 ++++++++--------- pyomo/contrib/solver/solvers/highs.py | 16 +++++++++---- pyomo/contrib/solver/solvers/ipopt.py | 2 +- .../solver/tests/solvers/test_ipopt.py | 2 +- .../solver/tests/solvers/test_solvers.py | 12 +++++----- .../contrib/solver/tests/unit/test_results.py | 11 +++++---- .../solver/tests/unit/test_solution.py | 24 ++++++++++--------- 10 files changed, 65 insertions(+), 52 deletions(-) diff --git a/pyomo/contrib/solver/common/base.py b/pyomo/contrib/solver/common/base.py index f935f3d4988..0782e577c43 100644 --- a/pyomo/contrib/solver/common/base.py +++ b/pyomo/contrib/solver/common/base.py @@ -570,7 +570,7 @@ def _solution_handler( results.solution_loader.load_import_suffixes() elif results.incumbent_objective is not None: delete_legacy_soln = False - for var, val in results.solution_loader.get_primals().items(): + for var, val in results.solution_loader.get_vars().items(): legacy_soln.variable[symbol_map.getSymbol(var)] = {'Value': val} if hasattr(model, 'dual') and model.dual.import_enabled(): for con, val in results.solution_loader.get_duals().items(): diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index 82b47ccb24b..fd932f90c15 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -10,7 +10,7 @@ # ___________________________________________________________________________ import operator -from typing import List +from typing import List, Any from pyomo.common.collections import ComponentMap, ComponentSet from pyomo.common.shutdown import python_is_shutting_down @@ -63,8 +63,8 @@ def get_number_of_solutions(self) -> int: def get_solution_ids(self) -> List[Any]: return [0] - def load_vars(self, vars_to_load=None, solution_id=0): - assert solution_id == 0 + def load_vars(self, vars_to_load=None, solution_id=None): + assert solution_id == None if self._grb_model.SolCount == 0: raise NoSolutionError() @@ -76,8 +76,8 @@ def load_vars(self, vars_to_load=None, solution_id=0): p_var.set_value(g_var, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) - def get_vars(self, vars_to_load=None, solution_id=0): - assert solution_id == 0 + def get_vars(self, vars_to_load=None, solution_id=None): + assert solution_id == None if self._grb_model.SolCount == 0: raise NoSolutionError() @@ -87,8 +87,8 @@ def get_vars(self, vars_to_load=None, solution_id=0): iterator = filter(lambda var_val: var_val[0] in vars_to_load, iterator) return ComponentMap(iterator) - def get_duals(self, cons_to_load=None, solution_id=0): - assert solution_id == 0 + def get_duals(self, cons_to_load=None, solution_id=None): + assert solution_id == None if self._grb_model.Status != gurobipy.GRB.OPTIMAL: raise NoDualsError() @@ -108,8 +108,8 @@ def dedup(_iter): ) return {con_info[0]: dual for con_info, dual in iterator} - def get_reduced_costs(self, vars_to_load=None, solution_id=0): - assert solution_id == 0 + def get_reduced_costs(self, vars_to_load=None, solution_id=None): + assert solution_id == None if self._grb_model.Status != gurobipy.GRB.OPTIMAL: raise NoReducedCostsError() diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index ae887a52fa5..e99d24025d5 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -99,7 +99,7 @@ def _load_suboptimal_mip_solution(solver_model, var_map, vars_to_load, solution_ return res -def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): +def _load_vars(solver_model, var_map, vars_to_load, solution_number=None): """ solver_model: gurobipy.Model var_map: Dict[int, gurobipy.Var] @@ -107,7 +107,7 @@ def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): vars_to_load: List[VarData] solution_number: int """ - for v, val in _get_primals( + for v, val in _get_vars( solver_model=solver_model, var_map=var_map, vars_to_load=vars_to_load, @@ -117,7 +117,7 @@ def _load_vars(solver_model, var_map, vars_to_load, solution_number=0): StaleFlagManager.mark_all_as_stale(delayed=True) -def _get_primals(solver_model, var_map, vars_to_load, solution_number=0): +def _get_vars(solver_model, var_map, vars_to_load, solution_number=None): """ solver_model: gurobipy.Model var_map: Dict[int, gurobipy.Var] @@ -128,7 +128,7 @@ def _get_primals(solver_model, var_map, vars_to_load, solution_number=0): if solver_model.SolCount == 0: raise NoSolutionError() - if solution_number != 0: + if solution_number not in {0, None}: return _load_suboptimal_mip_solution( solver_model=solver_model, var_map=var_map, diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 752b8512128..9f19bae307f 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -33,7 +33,7 @@ GurobiDirectBase, gurobipy, _load_vars, - _get_primals, + _get_vars, _get_duals, _get_reduced_costs, ) @@ -71,7 +71,7 @@ def get_solution_ids(self) -> List: return list(range(self.get_number_of_solutions())) def load_vars( - self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> None: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -83,11 +83,11 @@ def load_vars( ) def get_vars( - self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: if vars_to_load is None: vars_to_load = list(self._vars.values()) - return _get_primals( + return _get_vars( solver_model=self._solver_model, var_map=self._var_map, vars_to_load=vars_to_load, @@ -95,7 +95,7 @@ def get_vars( ) def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=0 + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: if vars_to_load is None: vars_to_load = list(self._vars.values()) @@ -106,7 +106,7 @@ def get_reduced_costs( ) def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=0 + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None ) -> Dict[ConstraintData, float]: if cons_to_load is None: cons_to_load = list(self._con_map.keys()) @@ -146,25 +146,25 @@ def _assert_solution_still_valid(self): raise RuntimeError('The results in the solver are no longer valid.') def load_vars( - self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None ) -> None: self._assert_solution_still_valid() return super().load_vars(vars_to_load, solution_id) def get_vars( - self, vars_to_load: Sequence[VarData] | None = None, solution_id=0 + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None ) -> Mapping[VarData, float]: self._assert_solution_still_valid() - return super().get_primals(vars_to_load, solution_id) + return super().get_vars(vars_to_load, solution_id) def get_duals( - self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=0, + self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None, ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return super().get_duals(cons_to_load) def get_reduced_costs( - self, vars_to_load: Sequence[VarData] | None = None, solution_id=0, + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None, ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_reduced_costs(vars_to_load) diff --git a/pyomo/contrib/solver/solvers/highs.py b/pyomo/contrib/solver/solvers/highs.py index a1d609c5db6..0abf02813ab 100644 --- a/pyomo/contrib/solver/solvers/highs.py +++ b/pyomo/contrib/solver/solvers/highs.py @@ -758,12 +758,16 @@ def _postsolve(self): return results - def _load_vars(self, vars_to_load=None): + def _load_vars(self, vars_to_load=None, solution_id=None): + if solution_id is not None: + raise NotImplementedError('highs interface does not currently support multiple solutions') for v, val in self._get_primals(vars_to_load=vars_to_load).items(): v.set_value(val, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) - def _get_primals(self, vars_to_load=None): + def _get_primals(self, vars_to_load=None, solution_id=None): + if solution_id is not None: + raise NotImplementedError('highs interface does not currently support multiple solutions') if self._sol is None or not self._sol.value_valid: raise NoSolutionError() @@ -786,7 +790,9 @@ def _get_primals(self, vars_to_load=None): return res - def _get_reduced_costs(self, vars_to_load=None): + def _get_reduced_costs(self, vars_to_load=None, solution_id=None): + if solution_id is not None: + raise NotImplementedError('highs interface does not currently support multiple solutions') if self._sol is None or not self._sol.dual_valid: raise NoReducedCostsError() res = ComponentMap() @@ -804,7 +810,9 @@ def _get_reduced_costs(self, vars_to_load=None): return res - def _get_duals(self, cons_to_load=None): + def _get_duals(self, cons_to_load=None, solution_id=None): + if solution_id is not None: + raise NotImplementedError('highs interface does not currently support multiple solutions') if self._sol is None or not self._sol.dual_valid: raise NoDualsError() diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 441b8eb5f61..80f9775a657 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -508,7 +508,7 @@ def solve(self, model, **kwds) -> Results: nl_info.objectives[0].expr, substitution_map={ id(v): val - for v, val in results.solution_loader.get_primals().items() + for v, val in results.solution_loader.get_vars().items() }, descend_into_named_expressions=True, remove_named_expressions=True, diff --git a/pyomo/contrib/solver/tests/solvers/test_ipopt.py b/pyomo/contrib/solver/tests/solvers/test_ipopt.py index d788b66982a..f0049922d55 100644 --- a/pyomo/contrib/solver/tests/solvers/test_ipopt.py +++ b/pyomo/contrib/solver/tests/solvers/test_ipopt.py @@ -62,7 +62,7 @@ def test_custom_instantiation(self): class TestIpoptSolutionLoader(unittest.TestCase): def test_get_reduced_costs_error(self): - loader = ipopt.IpoptSolutionLoader(None, None) + loader = ipopt.IpoptSolutionLoader(None, None, None) with self.assertRaises(NoSolutionError): loader.get_reduced_costs() diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 96e2e7b2c38..9c67ed7b1e5 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -1646,12 +1646,12 @@ def test_solution_loader( m.y.value = None res.solution_loader.load_vars([m.y]) self.assertAlmostEqual(m.y.value, 1) - primals = res.solution_loader.get_primals() + primals = res.solution_loader.get_vars() self.assertIn(m.x, primals) self.assertIn(m.y, primals) self.assertAlmostEqual(primals[m.x], 1) self.assertAlmostEqual(primals[m.y], 1) - primals = res.solution_loader.get_primals([m.y]) + primals = res.solution_loader.get_vars([m.y]) self.assertNotIn(m.x, primals) self.assertIn(m.y, primals) self.assertAlmostEqual(primals[m.y], 1) @@ -2000,7 +2000,7 @@ def test_variables_elsewhere2( res = opt.solve(m) self.assertEqual(res.solution_status, SolutionStatus.optimal) self.assertAlmostEqual(res.incumbent_objective, 1) - sol = res.solution_loader.get_primals() + sol = res.solution_loader.get_vars() self.assertIn(m.x, sol) self.assertIn(m.y, sol) self.assertIn(m.z, sol) @@ -2010,7 +2010,7 @@ def test_variables_elsewhere2( res = opt.solve(m) self.assertEqual(res.solution_status, SolutionStatus.optimal) self.assertAlmostEqual(res.incumbent_objective, 0) - sol = res.solution_loader.get_primals() + sol = res.solution_loader.get_vars() self.assertIn(m.x, sol) self.assertIn(m.y, sol) self.assertNotIn(m.z, sol) @@ -2172,7 +2172,7 @@ def test_scaling(self, name: str, opt_class: Type[SolverBase], use_presolve: boo self.assertAlmostEqual(res.incumbent_objective, 1) self.assertAlmostEqual(m.x.value, 1) self.assertAlmostEqual(m.y.value, 1) - primals = res.solution_loader.get_primals() + primals = res.solution_loader.get_vars() self.assertAlmostEqual(primals[m.x], 1) self.assertAlmostEqual(primals[m.y], 1) if check_duals: @@ -2188,7 +2188,7 @@ def test_scaling(self, name: str, opt_class: Type[SolverBase], use_presolve: boo self.assertAlmostEqual(res.incumbent_objective, 2) self.assertAlmostEqual(m.x.value, 2) self.assertAlmostEqual(m.y.value, 2) - primals = res.solution_loader.get_primals() + primals = res.solution_loader.get_vars() self.assertAlmostEqual(primals[m.x], 2) self.assertAlmostEqual(primals[m.y], 2) if check_duals: diff --git a/pyomo/contrib/solver/tests/unit/test_results.py b/pyomo/contrib/solver/tests/unit/test_results.py index a818f4ff4ad..a4def8f9089 100644 --- a/pyomo/contrib/solver/tests/unit/test_results.py +++ b/pyomo/contrib/solver/tests/unit/test_results.py @@ -49,8 +49,9 @@ def __init__( self._duals = duals self._reduced_costs = reduced_costs - def get_primals( - self, vars_to_load: Optional[Sequence[VarData]] = None + def get_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: if self._primals is None: raise RuntimeError( @@ -66,7 +67,8 @@ def get_primals( return primals def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, + solution_id=None, ) -> Dict[ConstraintData, float]: if self._duals is None: raise RuntimeError( @@ -83,7 +85,8 @@ def get_duals( return duals def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None + self, vars_to_load: Optional[Sequence[VarData]] = None, + solution_id=None, ) -> Mapping[VarData, float]: if self._reduced_costs is None: raise RuntimeError( diff --git a/pyomo/contrib/solver/tests/unit/test_solution.py b/pyomo/contrib/solver/tests/unit/test_solution.py index 0453f0e0cb2..a0fc4ac9b2f 100644 --- a/pyomo/contrib/solver/tests/unit/test_solution.py +++ b/pyomo/contrib/solver/tests/unit/test_solution.py @@ -18,7 +18,7 @@ class TestSolutionLoaderBase(unittest.TestCase): def test_member_list(self): - expected_list = ['load_vars', 'get_primals', 'get_duals', 'get_reduced_costs'] + expected_list = ['load_vars', 'get_vars', 'get_duals', 'get_reduced_costs', 'load_import_suffixes', 'get_number_of_solutions', 'get_solution_ids', 'load_solution'] method_list = [ method for method in dir(SolutionLoaderBase) @@ -29,18 +29,16 @@ def test_member_list(self): def test_solution_loader_base(self): self.instance = SolutionLoaderBase() with self.assertRaises(NotImplementedError): - self.instance.get_primals() - with self.assertRaises(NotImplementedError): - self.instance.get_duals() - with self.assertRaises(NotImplementedError): - self.instance.get_reduced_costs() + self.instance.get_vars() + self.assertEqual(self.instance.get_duals(), NotImplemented) + self.assertEqual(self.instance.get_reduced_costs(), NotImplemented) class TestSolSolutionLoader(unittest.TestCase): # I am currently unsure how to test this further because it relies heavily on # SolFileData and NLWriterInfo def test_member_list(self): - expected_list = ['load_vars', 'get_primals', 'get_duals', 'get_reduced_costs'] + expected_list = ['load_vars', 'get_vars', 'get_duals', 'get_reduced_costs', 'load_import_suffixes', 'get_number_of_solutions', 'get_solution_ids', 'load_solution'] method_list = [ method for method in dir(SolutionLoaderBase) @@ -53,10 +51,14 @@ class TestPersistentSolutionLoader(unittest.TestCase): def test_member_list(self): expected_list = [ 'load_vars', - 'get_primals', + 'get_vars', 'get_duals', 'get_reduced_costs', 'invalidate', + 'load_import_suffixes', + 'get_number_of_solutions', + 'get_solution_ids', + 'load_solution' ] method_list = [ method @@ -69,12 +71,12 @@ def test_default_initialization(self): # Realistically, a solver object should be passed into this. # However, it works with a string. It'll just error loudly if you # try to run get_primals, etc. - self.instance = PersistentSolutionLoader('ipopt') + self.instance = PersistentSolutionLoader('ipopt', None) self.assertTrue(self.instance._valid) self.assertEqual(self.instance._solver, 'ipopt') def test_invalid(self): - self.instance = PersistentSolutionLoader('ipopt') + self.instance = PersistentSolutionLoader('ipopt', None) self.instance.invalidate() with self.assertRaises(RuntimeError): - self.instance.get_primals() + self.instance.get_vars() From 23ba4d96173c363953f574498ca1827c62c3ddd7 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 09:35:47 -0600 Subject: [PATCH 052/104] typo --- pyomo/contrib/observer/tests/test_change_detector.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index efda8a181d9..b88877a9f63 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -24,7 +24,7 @@ def __init__(self): super().__init__() self.counts = ComponentMap() """ - counts is be a mapping from component (e.g., variable) to another + counts is a mapping from component (e.g., variable) to another mapping from string ('add', 'remove', 'update', or 'set') to an int that indicates the number of times the corresponding method has been called """ @@ -220,4 +220,4 @@ def test_constraints(self): obs.check(expected) def test_vars_and_params_elsewhere(self): - pass \ No newline at end of file + pass From d7b991825afbbc3eb2010bb5d4823a97f1378bea Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 10:00:38 -0600 Subject: [PATCH 053/104] update observer tests --- pyomo/contrib/observer/model_observer.py | 7 +- .../observer/tests/test_change_detector.py | 158 ++++++++++++++++-- 2 files changed, 149 insertions(+), 16 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 39b832cc266..bd89d4f600d 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -366,12 +366,15 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): self._named_expressions[con] = [] self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) + for con in cons: + variables = self._vars_referenced_by_con[con] + params = self._params_referenced_by_con[con] for v in variables: self._referenced_variables[id(v)][1][con] = None for p in params: self._referenced_params[id(p)][1][con] = None - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) for obs in self._observers: obs.add_sos_constraints(cons) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 92b7bbff390..46527618ab8 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -14,7 +14,7 @@ from pyomo.core.base.param import ParamData from pyomo.core.base.sos import SOSConstraintData from pyomo.core.base.var import VarData -import pyomo.environ as pe +import pyomo.environ as pyo from pyomo.common import unittest from typing import List from pyomo.contrib.observer.model_observer import ( @@ -118,10 +118,10 @@ def update_parameters(self, params: List[ParamData]): class TestChangeDetector(unittest.TestCase): def test_objective(self): - m = pe.ConcreteModel() - m.x = pe.Var() - m.y = pe.Var() - m.p = pe.Param(mutable=True, initialize=1) + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.p = pyo.Param(mutable=True, initialize=1) obs = ObserverChecker() detector = ModelChangeDetector([obs]) @@ -133,7 +133,7 @@ def test_objective(self): detector.set_instance(m) obs.check(expected) - m.obj = pe.Objective(expr=m.x**2 + m.p * m.y**2) + m.obj = pyo.Objective(expr=m.x**2 + m.p * m.y**2) detector.update() expected[m.obj] = make_count_dict() expected[m.obj]['set'] += 1 @@ -174,7 +174,7 @@ def test_objective(self): obs.check(expected) del m.obj - m.obj = pe.Objective(expr=m.p * m.x) + m.obj = pyo.Objective(expr=m.p * m.x) detector.update() expected[m.p]['add'] += 1 expected[m.y]['remove'] += 1 @@ -183,10 +183,10 @@ def test_objective(self): expected[m.obj]['set'] += 1 def test_constraints(self): - m = pe.ConcreteModel() - m.x = pe.Var() - m.y = pe.Var() - m.p = pe.Param(mutable=True, initialize=1) + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.p = pyo.Param(mutable=True, initialize=1) obs = ObserverChecker() detector = ModelChangeDetector([obs]) @@ -198,8 +198,8 @@ def test_constraints(self): detector.set_instance(m) obs.check(expected) - m.obj = pe.Objective(expr=m.y) - m.c1 = pe.Constraint(expr=m.y >= (m.x - m.p) ** 2) + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.Constraint(expr=m.y >= (m.x - m.p) ** 2) detector.update() expected[m.x] = make_count_dict() expected[m.y] = make_count_dict() @@ -232,5 +232,135 @@ def test_constraints(self): expected[m.p]['add'] += 1 obs.check(expected) + def test_sos(self): + m = pyo.ConcreteModel() + m.a = pyo.Set(initialize=[1, 2, 3], ordered=True) + m.x = pyo.Var(m.a, within=pyo.Binary) + m.y = pyo.Var(within=pyo.Binary) + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.SOSConstraint(var=m.x, sos=1) + + obs = ObserverChecker() + detector = ModelChangeDetector([obs]) + detector.set_instance(m) + + expected = ComponentMap() + expected[m.obj] = make_count_dict() + for i in m.a: + expected[m.x[i]] = make_count_dict() + expected[m.y] = make_count_dict() + expected[m.c1] = make_count_dict() + expected[m.obj]['set'] += 1 + for i in m.a: + expected[m.x[i]]['add'] += 1 + expected[m.y]['add'] += 1 + expected[m.c1]['add'] += 1 + obs.check(expected) + + for i in m.a: + expected[m.x[i]]['remove'] += 1 + expected[m.c1]['remove'] += 1 + del m.c1 + detector.update() + obs.check(expected) + def test_vars_and_params_elsewhere(self): - pass + m1 = pyo.ConcreteModel() + m1.x = pyo.Var() + m1.y = pyo.Var() + m1.p = pyo.Param(mutable=True, initialize=1) + + m2 = pyo.ConcreteModel() + + obs = ObserverChecker() + detector = ModelChangeDetector([obs]) + + expected = ComponentMap() + expected[None] = make_count_dict() + expected[None]['set'] += 1 + + detector.set_instance(m2) + obs.check(expected) + + m2.obj = pyo.Objective(expr=m1.y) + m2.c1 = pyo.Constraint(expr=m1.y >= (m1.x - m1.p) ** 2) + detector.update() + expected[m1.x] = make_count_dict() + expected[m1.y] = make_count_dict() + expected[m1.p] = make_count_dict() + expected[m1.x]['add'] += 1 + expected[m1.y]['add'] += 1 + expected[m1.p]['add'] += 1 + expected[m2.c1] = make_count_dict() + expected[m2.c1]['add'] += 1 + expected[m2.obj] = make_count_dict() + expected[m2.obj]['set'] += 1 + obs.check(expected) + + # now fix a variable and make sure the + # constraint gets removed and added + m1.x.fix(1) + obs.pprint() + detector.update() + obs.pprint() + expected[m2.c1]['remove'] += 1 + expected[m2.c1]['add'] += 1 + # because x and p are only used in the + # one constraint, they get removed when + # the constraint is removed and then + # added again when the constraint is added + expected[m1.x]['update'] += 1 + expected[m1.x]['remove'] += 1 + expected[m1.x]['add'] += 1 + expected[m1.p]['remove'] += 1 + expected[m1.p]['add'] += 1 + obs.check(expected) + + def test_named_expression(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.p = pyo.Param(mutable=True, initialize=1) + + obs = ObserverChecker() + detector = ModelChangeDetector([obs]) + + expected = ComponentMap() + expected[None] = make_count_dict() + expected[None]['set'] += 1 + + detector.set_instance(m) + obs.check(expected) + + m.obj = pyo.Objective(expr=m.y) + m.e = pyo.Expression(expr=m.x - m.p) + m.c1 = pyo.Constraint(expr=m.y >= m.e) + detector.update() + expected[m.x] = make_count_dict() + expected[m.y] = make_count_dict() + expected[m.p] = make_count_dict() + expected[m.x]['add'] += 1 + expected[m.y]['add'] += 1 + expected[m.p]['add'] += 1 + expected[m.c1] = make_count_dict() + expected[m.c1]['add'] += 1 + expected[m.obj] = make_count_dict() + expected[m.obj]['set'] += 1 + obs.check(expected) + + # now modify the named expression and make sure the + # constraint gets removed and added + m.e.expr = (m.x - m.p) ** 2 + detector.update() + expected[m.c1]['remove'] += 1 + expected[m.c1]['add'] += 1 + # because x and p are only used in the + # one constraint, they get removed when + # the constraint is removed and then + # added again when the constraint is added + expected[m.x]['remove'] += 1 + expected[m.x]['add'] += 1 + expected[m.p]['remove'] += 1 + expected[m.p]['add'] += 1 + obs.check(expected) + From c313fe53f890bdfc000dfd2d68a76fcef8f9de8b Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 15:48:39 -0600 Subject: [PATCH 054/104] run black --- pyomo/contrib/observer/tests/test_change_detector.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 46527618ab8..f3ce6e28b5c 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -363,4 +363,3 @@ def test_named_expression(self): expected[m.p]['remove'] += 1 expected[m.p]['add'] += 1 obs.check(expected) - From d16bee5fd1f27cc7c3b55ff858ab28b7478a6975 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 18:00:06 -0600 Subject: [PATCH 055/104] adding tests for trivial constraints and fixing bugs --- pyomo/contrib/observer/model_observer.py | 2 +- .../contrib/solver/common/solution_loader.py | 30 +++++++++++ .../solvers/gurobi/gurobi_direct_base.py | 23 +++++++- .../solvers/gurobi/gurobi_persistent.py | 6 ++- .../solver/tests/solvers/test_solvers.py | 53 +++++++++++++++++++ pyomo/repn/plugins/standard_form.py | 3 +- 6 files changed, 113 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 4ab52100376..325e7a8aee6 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -614,7 +614,7 @@ def _check_for_var_changes(self): vars_to_update.append(v) elif _domain_interval != v.domain.get_interval(): vars_to_update.append(v) - elif v.value != _value: + elif v.fixed and v.value != _value: vars_to_update.append(v) cons_to_update = list(cons_to_update.keys()) return vars_to_update, cons_to_update, update_obj diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index e399d6bea55..be0ea7ad00c 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -17,6 +17,7 @@ from pyomo.core.base.var import VarData from pyomo.core.staleflag import StaleFlagManager from pyomo.core.base.suffix import Suffix +from .util import NoSolutionError def load_import_suffixes(pyomo_model, solution_loader: SolutionLoaderBase, solution_id=None): @@ -194,6 +195,35 @@ def load_import_suffixes(self, solution_id=None): return NotImplemented +class NoSolutionSolutionLoader(SolutionLoaderBase): + def __init__(self) -> None: + pass + + def get_solution_ids(self) -> List[Any]: + return [] + + def get_number_of_solutions(self) -> int: + return 0 + + def load_solution(self, solution_id=None): + raise NoSolutionError() + + def load_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> None: + raise NoSolutionError() + + def get_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> Mapping[VarData, float]: + raise NoSolutionError() + + def get_duals(self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None) -> Dict[ConstraintData, float]: + raise NoSolutionError() + + def get_reduced_costs(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> Mapping[VarData, float]: + raise NoSolutionError() + + def load_import_suffixes(self, solution_id=None): + raise NoSolutionError() + + class PersistentSolutionLoader(SolutionLoaderBase): """ Loader for persistent solvers diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index e99d24025d5..ce77c31c6f7 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -18,7 +18,7 @@ from pyomo.common.config import ConfigValue from pyomo.common.dependencies import attempt_import from pyomo.common.enums import ObjectiveSense -from pyomo.common.errors import ApplicationError +from pyomo.common.errors import ApplicationError, InfeasibleConstraintException from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.tee import capture_output, TeeStream from pyomo.common.timing import HierarchicalTimer @@ -33,6 +33,7 @@ NoReducedCostsError, NoSolutionError, ) +from pyomo.contrib.solver.common.solution_loader import NoSolutionSolutionLoader from pyomo.contrib.solver.common.results import ( Results, SolutionStatus, @@ -353,6 +354,8 @@ def solve(self, model, **kwds) -> Results: res = self._postsolve( grb_model=gurobi_model, solution_loader=solution_loader, has_obj=has_obj ) + except InfeasibleConstraintException: + res = self._get_infeasible_results() finally: os.chdir(orig_cwd) @@ -390,6 +393,24 @@ def _get_tc_map(self): } return GurobiDirectBase._tc_map + def _get_infeasible_results(self): + res = Results() + res.solution_loader = NoSolutionSolutionLoader() + res.solution_status = SolutionStatus.noSolution + res.termination_condition = TerminationCondition.provenInfeasible + res.incumbent_objective = None + res.objective_bound = None + res.iteration_count = None + res.timing_info.gurobi_time = None + res.solver_config = self.config + res.solver_name = self.name + res.solver_version = self.version() + if self.config.raise_exception_on_nonoptimal_result: + raise NoOptimalSolutionError() + if self.config.load_solutions: + raise NoFeasibleSolutionError() + return res + def _postsolve(self, grb_model, solution_loader, has_obj): status = grb_model.Status diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 9f19bae307f..27a61e27916 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -17,6 +17,7 @@ from pyomo.common.collections import ComponentSet, OrderedSet from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.timing import HierarchicalTimer +from pyomo.common.errors import InfeasibleConstraintException from pyomo.core.base.objective import ObjectiveData from pyomo.core.kernel.objective import minimize, maximize from pyomo.core.base.var import VarData @@ -464,7 +465,9 @@ def _get_expr_from_pyomo_repn(self, repn): vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] new_expr = gurobipy.LinExpr(coef_list, vlist) else: - new_expr = 0.0 + # this can't just be zero in case the constraint is a + # trivial one + new_expr = gurobipy.LinExpr() if len(repn.quadratic_vars) > 0: missing_vars = {} @@ -714,6 +717,7 @@ def _add_constraints(self, cons: List[ConstraintData]): for ndx, con in enumerate(cons): lb, body, ub = con.to_bounded_expression(evaluate_bounds=False) repn = generate_standard_repn(body, quadratic=True, compute_values=False) + if len(repn.quadratic_vars) > 0: self._quadratic_cons.add(con) else: diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 9c67ed7b1e5..d154253475c 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -35,6 +35,7 @@ GurobiDirectQuadratic, GurobiPersistent, ) +from pyomo.contrib.solver.common.util import NoSolutionError, NoFeasibleSolutionError, NoOptimalSolutionError from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -1053,6 +1054,58 @@ def test_results_infeasible( ): res.solution_loader.get_reduced_costs() + @parameterized.expand(input=_load_tests(all_solvers)) + def test_trivial_constraints( + self, name: str, opt_class: Type[SolverBase], use_presolve: bool + ): + opt: SolverBase = opt_class() + if not opt.available(): + raise unittest.SkipTest(f'Solver {opt.name} not available.') + if any(name.startswith(i) for i in nl_solvers_set): + if use_presolve: + opt.config.writer_config.linear_presolve = True + else: + opt.config.writer_config.linear_presolve = False + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.Constraint(expr=m.y >= m.x) + m.c2 = pyo.Constraint(expr=m.y >= -m.x) + m.c3 = pyo.Constraint(expr=m.x >= 0) + + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 0) + + m.x.fix(1) + opt.config.tee = True + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 1) + self.assertAlmostEqual(m.y.value, 1) + + m.x.fix(-1) + with self.assertRaises(NoOptimalSolutionError): + res = opt.solve(m) + + opt.config.load_solutions = False + opt.config.raise_exception_on_nonoptimal_result = False + res = opt.solve(m) + self.assertNotEqual(res.solution_status, SolutionStatus.optimal) + if isinstance(opt, Ipopt): + acceptable_termination_conditions = { + TerminationCondition.locallyInfeasible, + TerminationCondition.unbounded, + TerminationCondition.provenInfeasible, + } + else: + acceptable_termination_conditions = { + TerminationCondition.provenInfeasible, + TerminationCondition.infeasibleOrUnbounded, + } + self.assertIn(res.termination_condition, acceptable_termination_conditions) + self.assertIsNone(res.incumbent_objective) + @parameterized.expand(input=_load_tests(all_solvers)) def test_duals(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): opt: SolverBase = opt_class() diff --git a/pyomo/repn/plugins/standard_form.py b/pyomo/repn/plugins/standard_form.py index 314a1822e09..59c15910350 100644 --- a/pyomo/repn/plugins/standard_form.py +++ b/pyomo/repn/plugins/standard_form.py @@ -20,6 +20,7 @@ InEnum, document_kwargs_from_configdict, ) +from pyomo.common.errors import InfeasibleConstraintException from pyomo.common.dependencies import scipy, numpy as np from pyomo.common.enums import ObjectiveSense from pyomo.common.gc_manager import PauseGC @@ -462,7 +463,7 @@ def write(self, model): # TODO: add a (configurable) feasibility tolerance if (lb is None or lb <= offset) and (ub is None or ub >= offset): continue - raise InfeasibleError( + raise InfeasibleConstraintException( f"model contains a trivially infeasible constraint, '{con.name}'" ) From a4e2b81410b9edba643ef118f21e4ca9f3cc0b37 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 18:05:43 -0600 Subject: [PATCH 056/104] run black --- .../contrib/solver/common/solution_loader.py | 69 ++++++++----------- .../solver/solvers/gurobi/gurobi_direct.py | 13 ++-- .../solvers/gurobi/gurobi_persistent.py | 43 +++++++++--- pyomo/contrib/solver/solvers/highs.py | 16 +++-- pyomo/contrib/solver/solvers/ipopt.py | 10 +-- pyomo/contrib/solver/solvers/sol_reader.py | 17 +++-- .../contrib/solver/tests/unit/test_results.py | 9 +-- .../solver/tests/unit/test_solution.py | 24 ++++++- 8 files changed, 125 insertions(+), 76 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index e399d6bea55..3ad688d937f 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -19,7 +19,9 @@ from pyomo.core.base.suffix import Suffix -def load_import_suffixes(pyomo_model, solution_loader: SolutionLoaderBase, solution_id=None): +def load_import_suffixes( + pyomo_model, solution_loader: SolutionLoaderBase, solution_id=None +): dual_suffix = None rc_suffix = None for suffix in pyomo_model.component_objects(Suffix, descend_into=True, active=True): @@ -46,10 +48,10 @@ class SolutionLoaderBase: def get_solution_ids(self) -> List[Any]: """ - If there are multiple solutions available, this will return a - list of the solution ids which can then be used with other - methods like `load_soltuion`. If only one solution is - available, this will return [None]. If no solutions + If there are multiple solutions available, this will return a + list of the solution ids which can then be used with other + methods like `load_soltuion`. If only one solution is + available, this will return [None]. If no solutions are available, this will return None Returns @@ -58,7 +60,7 @@ def get_solution_ids(self) -> List[Any]: The identifiers for multiple solutions """ return NotImplemented - + def get_number_of_solutions(self) -> int: """ Returns @@ -75,7 +77,7 @@ def load_solution(self, solution_id=None): Parameters ---------- solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be loaded. If None, the default solution will be used. """ # this should load everything it can @@ -83,36 +85,31 @@ def load_solution(self, solution_id=None): self.load_import_suffixes(solution_id=solution_id) def load_vars( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> None: """ - Load the solution of the primal variables into the value attribute + Load the solution of the primal variables into the value attribute of the variables. Parameters ---------- vars_to_load: list - The minimum set of variables whose solution should be loaded. If - vars_to_load is None, then the solution to all primal variables - will be loaded. Even if vars_to_load is specified, the values of + The minimum set of variables whose solution should be loaded. If + vars_to_load is None, then the solution to all primal variables + will be loaded. Even if vars_to_load is specified, the values of other variables may also be loaded depending on the interface. solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be loaded. If None, the default solution will be used. """ for var, val in self.get_vars( - vars_to_load=vars_to_load, - solution_id=solution_id + vars_to_load=vars_to_load, solution_id=solution_id ).items(): var.set_value(val, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) def get_vars( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: """ Returns a ComponentMap mapping variable to var value. @@ -123,7 +120,7 @@ def get_vars( A list of the variables whose solution value should be retrieved. If vars_to_load is None, then the values for all variables will be retrieved. solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be retrieved. If None, the default solution will be used. Returns @@ -136,9 +133,7 @@ def get_vars( ) def get_duals( - self, - cons_to_load: Optional[Sequence[ConstraintData]] = None, - solution_id=None, + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None ) -> Dict[ConstraintData, float]: """ Returns a dictionary mapping constraint to dual value. @@ -149,7 +144,7 @@ def get_duals( A list of the constraints whose duals should be retrieved. If cons_to_load is None, then the duals for all constraints will be retrieved. solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be retrieved. If None, the default solution will be used. Returns @@ -160,9 +155,7 @@ def get_duals( return NotImplemented def get_reduced_costs( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: """ Returns a ComponentMap mapping variable to reduced cost. @@ -173,7 +166,7 @@ def get_reduced_costs( A list of the variables whose reduced cost should be retrieved. If vars_to_load is None, then the reduced costs for all variables will be loaded. solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be retrieved. If None, the default solution will be used. Returns @@ -182,13 +175,13 @@ def get_reduced_costs( Maps variables to reduced costs """ return NotImplemented - + def load_import_suffixes(self, solution_id=None): """ Parameters ---------- solution_id: Optional[Any] - If there are multiple solutions, this specifies which solution + If there are multiple solutions, this specifies which solution should be loaded. If None, the default solution will be used. """ return NotImplemented @@ -211,27 +204,25 @@ def _assert_solution_still_valid(self): def get_solution_ids(self) -> List[Any]: self._assert_solution_still_valid() return super().get_solution_ids() - + def get_number_of_solutions(self) -> int: self._assert_solution_still_valid() return super().get_number_of_solutions() def get_vars(self, vars_to_load=None, solution_id=None): self._assert_solution_still_valid() - return self._solver._get_primals(vars_to_load=vars_to_load, solution_id=solution_id) + return self._solver._get_primals( + vars_to_load=vars_to_load, solution_id=solution_id + ) def get_duals( - self, - cons_to_load: Optional[Sequence[ConstraintData]] = None, - solution_id=None, + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return self._solver._get_duals(cons_to_load=cons_to_load) def get_reduced_costs( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return self._solver._get_reduced_costs(vars_to_load=vars_to_load) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py index fd932f90c15..42ab82a1d7c 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct.py @@ -23,7 +23,10 @@ NoSolutionError, IncompatibleModelError, ) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes +from pyomo.contrib.solver.common.solution_loader import ( + SolutionLoaderBase, + load_import_suffixes, +) from .gurobi_direct_base import GurobiDirectBase, gurobipy @@ -59,7 +62,7 @@ def get_number_of_solutions(self) -> int: if self._grb_model.SolCount == 0: return 0 return 1 - + def get_solution_ids(self) -> List[Any]: return [0] @@ -118,9 +121,11 @@ def get_reduced_costs(self, vars_to_load=None, solution_id=None): vars_to_load = ComponentSet(vars_to_load) iterator = filter(lambda var_rc: var_rc[0] in vars_to_load, iterator) return ComponentMap(iterator) - + def load_import_suffixes(self, solution_id=None): - load_import_suffixes(pyomo_model=self._pyomo_model, solution_loader=self, solution_id=solution_id) + load_import_suffixes( + pyomo_model=self._pyomo_model, solution_loader=self, solution_id=solution_id + ) class GurobiDirect(GurobiDirectBase): diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 9f19bae307f..e36e1e6db1e 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -27,7 +27,10 @@ from pyomo.repn import generate_standard_repn from pyomo.contrib.solver.common.results import Results from pyomo.contrib.solver.common.util import IncompatibleModelError -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes +from pyomo.contrib.solver.common.solution_loader import ( + SolutionLoaderBase, + load_import_suffixes, +) from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( GurobiDirectBase, @@ -46,7 +49,14 @@ class GurobiDirectQuadraticSolutionLoader(SolutionLoaderBase): def __init__( - self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model + self, + solver_model, + var_id_map, + var_map, + con_map, + linear_cons, + quadratic_cons, + pyomo_model, ) -> None: super().__init__() self._solver_model = solver_model @@ -66,7 +76,7 @@ def __del__(self): def get_number_of_solutions(self) -> int: return self._solver_model.SolCount - + def get_solution_ids(self) -> List: return list(range(self.get_number_of_solutions())) @@ -131,10 +141,23 @@ def load_import_suffixes(self, solution_id=None): class GurobiPersistentSolutionLoader(GurobiDirectQuadraticSolutionLoader): def __init__( - self, solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model + self, + solver_model, + var_id_map, + var_map, + con_map, + linear_cons, + quadratic_cons, + pyomo_model, ) -> None: super().__init__( - solver_model, var_id_map, var_map, con_map, linear_cons, quadratic_cons, pyomo_model + solver_model, + var_id_map, + var_map, + con_map, + linear_cons, + quadratic_cons, + pyomo_model, ) self._valid = True @@ -158,25 +181,25 @@ def get_vars( return super().get_vars(vars_to_load, solution_id) def get_duals( - self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None, + self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None ) -> Dict[ConstraintData, float]: self._assert_solution_still_valid() return super().get_duals(cons_to_load) def get_reduced_costs( - self, vars_to_load: Sequence[VarData] | None = None, solution_id=None, + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None ) -> Mapping[VarData, float]: self._assert_solution_still_valid() return super().get_reduced_costs(vars_to_load) - + def get_number_of_solutions(self) -> int: self._assert_solution_still_valid() return super().get_number_of_solutions() - + def get_solution_ids(self) -> List: self._assert_solution_still_valid() return super().get_solution_ids() - + def load_import_suffixes(self, solution_id=None): self._assert_solution_still_valid() super().load_import_suffixes(solution_id) diff --git a/pyomo/contrib/solver/solvers/highs.py b/pyomo/contrib/solver/solvers/highs.py index 0abf02813ab..87796b91b68 100644 --- a/pyomo/contrib/solver/solvers/highs.py +++ b/pyomo/contrib/solver/solvers/highs.py @@ -760,14 +760,18 @@ def _postsolve(self): def _load_vars(self, vars_to_load=None, solution_id=None): if solution_id is not None: - raise NotImplementedError('highs interface does not currently support multiple solutions') + raise NotImplementedError( + 'highs interface does not currently support multiple solutions' + ) for v, val in self._get_primals(vars_to_load=vars_to_load).items(): v.set_value(val, skip_validation=True) StaleFlagManager.mark_all_as_stale(delayed=True) def _get_primals(self, vars_to_load=None, solution_id=None): if solution_id is not None: - raise NotImplementedError('highs interface does not currently support multiple solutions') + raise NotImplementedError( + 'highs interface does not currently support multiple solutions' + ) if self._sol is None or not self._sol.value_valid: raise NoSolutionError() @@ -792,7 +796,9 @@ def _get_primals(self, vars_to_load=None, solution_id=None): def _get_reduced_costs(self, vars_to_load=None, solution_id=None): if solution_id is not None: - raise NotImplementedError('highs interface does not currently support multiple solutions') + raise NotImplementedError( + 'highs interface does not currently support multiple solutions' + ) if self._sol is None or not self._sol.dual_valid: raise NoReducedCostsError() res = ComponentMap() @@ -812,7 +818,9 @@ def _get_reduced_costs(self, vars_to_load=None, solution_id=None): def _get_duals(self, cons_to_load=None, solution_id=None): if solution_id is not None: - raise NotImplementedError('highs interface does not currently support multiple solutions') + raise NotImplementedError( + 'highs interface does not currently support multiple solutions' + ) if self._sol is None or not self._sol.dual_valid: raise NoDualsError() diff --git a/pyomo/contrib/solver/solvers/ipopt.py b/pyomo/contrib/solver/solvers/ipopt.py index 80f9775a657..bce5e9bd867 100644 --- a/pyomo/contrib/solver/solvers/ipopt.py +++ b/pyomo/contrib/solver/solvers/ipopt.py @@ -109,9 +109,7 @@ def _error_check(self): ) def get_reduced_costs( - self, - vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: self._error_check() if solution_id is not None: @@ -448,7 +446,9 @@ def solve(self, model, **kwds) -> Results: TerminationCondition.convergenceCriteriaSatisfied ) results.solution_status = SolutionStatus.optimal - results.solution_loader = SolSolutionLoader(None, nl_info=nl_info, pyomo_model=model) + results.solution_loader = SolSolutionLoader( + None, nl_info=nl_info, pyomo_model=model + ) results.iteration_count = 0 results.timing_info.total_seconds = 0 else: @@ -668,7 +668,7 @@ def _parse_solution( res.solution_loader = SolSolutionLoader(None, None, pyomo_model=pyomo_model) else: res.solution_loader = IpoptSolutionLoader( - sol_data=sol_data, nl_info=nl_info, pyomo_model=pyomo_model, + sol_data=sol_data, nl_info=nl_info, pyomo_model=pyomo_model ) return res diff --git a/pyomo/contrib/solver/solvers/sol_reader.py b/pyomo/contrib/solver/solvers/sol_reader.py index 7570a1ffc53..f405cc85943 100644 --- a/pyomo/contrib/solver/solvers/sol_reader.py +++ b/pyomo/contrib/solver/solvers/sol_reader.py @@ -26,7 +26,10 @@ SolutionStatus, TerminationCondition, ) -from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase, load_import_suffixes +from pyomo.contrib.solver.common.solution_loader import ( + SolutionLoaderBase, + load_import_suffixes, +) class SolFileData: @@ -49,7 +52,9 @@ class SolSolutionLoader(SolutionLoaderBase): Loader for solvers that create .sol files (e.g., ipopt) """ - def __init__(self, sol_data: SolFileData, nl_info: NLWriterInfo, pyomo_model) -> None: + def __init__( + self, sol_data: SolFileData, nl_info: NLWriterInfo, pyomo_model + ) -> None: self._sol_data = sol_data self._nl_info = nl_info self._pyomo_model = pyomo_model @@ -58,14 +63,16 @@ def get_number_of_solutions(self) -> int: if self._nl_info is None: return 0 return 1 - + def get_solution_ids(self) -> List[Any]: return [None] def load_import_suffixes(self, solution_id=None): load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) - def load_vars(self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None) -> NoReturn: + def load_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None + ) -> NoReturn: if solution_id is not None: raise ValueError(f'{self.__class__.__name__} does not support solution_id') if self._nl_info is None: @@ -131,7 +138,7 @@ def get_vars( return res def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None, + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None ) -> Dict[ConstraintData, float]: if solution_id is not None: raise ValueError(f'{self.__class__.__name__} does not support solution_id') diff --git a/pyomo/contrib/solver/tests/unit/test_results.py b/pyomo/contrib/solver/tests/unit/test_results.py index a4def8f9089..3dad4c523d2 100644 --- a/pyomo/contrib/solver/tests/unit/test_results.py +++ b/pyomo/contrib/solver/tests/unit/test_results.py @@ -50,8 +50,7 @@ def __init__( self._reduced_costs = reduced_costs def get_vars( - self, vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: if self._primals is None: raise RuntimeError( @@ -67,8 +66,7 @@ def get_vars( return primals def get_duals( - self, cons_to_load: Optional[Sequence[ConstraintData]] = None, - solution_id=None, + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None ) -> Dict[ConstraintData, float]: if self._duals is None: raise RuntimeError( @@ -85,8 +83,7 @@ def get_duals( return duals def get_reduced_costs( - self, vars_to_load: Optional[Sequence[VarData]] = None, - solution_id=None, + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: if self._reduced_costs is None: raise RuntimeError( diff --git a/pyomo/contrib/solver/tests/unit/test_solution.py b/pyomo/contrib/solver/tests/unit/test_solution.py index a0fc4ac9b2f..79e5b39aaf6 100644 --- a/pyomo/contrib/solver/tests/unit/test_solution.py +++ b/pyomo/contrib/solver/tests/unit/test_solution.py @@ -18,7 +18,16 @@ class TestSolutionLoaderBase(unittest.TestCase): def test_member_list(self): - expected_list = ['load_vars', 'get_vars', 'get_duals', 'get_reduced_costs', 'load_import_suffixes', 'get_number_of_solutions', 'get_solution_ids', 'load_solution'] + expected_list = [ + 'load_vars', + 'get_vars', + 'get_duals', + 'get_reduced_costs', + 'load_import_suffixes', + 'get_number_of_solutions', + 'get_solution_ids', + 'load_solution', + ] method_list = [ method for method in dir(SolutionLoaderBase) @@ -38,7 +47,16 @@ class TestSolSolutionLoader(unittest.TestCase): # I am currently unsure how to test this further because it relies heavily on # SolFileData and NLWriterInfo def test_member_list(self): - expected_list = ['load_vars', 'get_vars', 'get_duals', 'get_reduced_costs', 'load_import_suffixes', 'get_number_of_solutions', 'get_solution_ids', 'load_solution'] + expected_list = [ + 'load_vars', + 'get_vars', + 'get_duals', + 'get_reduced_costs', + 'load_import_suffixes', + 'get_number_of_solutions', + 'get_solution_ids', + 'load_solution', + ] method_list = [ method for method in dir(SolutionLoaderBase) @@ -58,7 +76,7 @@ def test_member_list(self): 'load_import_suffixes', 'get_number_of_solutions', 'get_solution_ids', - 'load_solution' + 'load_solution', ] method_list = [ method From d25e7215f4f728ff4541060f84f9b6cd92d09229 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 13 Aug 2025 18:06:51 -0600 Subject: [PATCH 057/104] run black --- .../contrib/solver/common/solution_loader.py | 30 ++++++++++++------- .../solvers/gurobi/gurobi_persistent.py | 2 +- .../solver/tests/solvers/test_solvers.py | 10 +++++-- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index b0bddeb56ba..666ea66e1e9 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -194,25 +194,33 @@ def __init__(self) -> None: def get_solution_ids(self) -> List[Any]: return [] - + def get_number_of_solutions(self) -> int: return 0 - + def load_solution(self, solution_id=None): raise NoSolutionError() - - def load_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> None: + + def load_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> None: raise NoSolutionError() - - def get_vars(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> Mapping[VarData, float]: + + def get_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> Mapping[VarData, float]: raise NoSolutionError() - - def get_duals(self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None) -> Dict[ConstraintData, float]: + + def get_duals( + self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None + ) -> Dict[ConstraintData, float]: raise NoSolutionError() - - def get_reduced_costs(self, vars_to_load: Sequence[VarData] | None = None, solution_id=None) -> Mapping[VarData, float]: + + def get_reduced_costs( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> Mapping[VarData, float]: raise NoSolutionError() - + def load_import_suffixes(self, solution_id=None): raise NoSolutionError() diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index c0211604324..b9ea9a6c8e8 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -488,7 +488,7 @@ def _get_expr_from_pyomo_repn(self, repn): vlist = [self._pyomo_var_to_solver_var_map[id(v)] for v in repn.linear_vars] new_expr = gurobipy.LinExpr(coef_list, vlist) else: - # this can't just be zero in case the constraint is a + # this can't just be zero in case the constraint is a # trivial one new_expr = gurobipy.LinExpr() diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index d154253475c..189b0373780 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -35,7 +35,11 @@ GurobiDirectQuadratic, GurobiPersistent, ) -from pyomo.contrib.solver.common.util import NoSolutionError, NoFeasibleSolutionError, NoOptimalSolutionError +from pyomo.contrib.solver.common.util import ( + NoSolutionError, + NoFeasibleSolutionError, + NoOptimalSolutionError, +) from pyomo.contrib.solver.solvers.highs import Highs from pyomo.core.expr.numeric_expr import LinearExpression from pyomo.core.expr.compare import assertExpressionsEqual @@ -1073,11 +1077,11 @@ def test_trivial_constraints( m.c1 = pyo.Constraint(expr=m.y >= m.x) m.c2 = pyo.Constraint(expr=m.y >= -m.x) m.c3 = pyo.Constraint(expr=m.x >= 0) - + res = opt.solve(m) self.assertAlmostEqual(m.x.value, 0) self.assertAlmostEqual(m.y.value, 0) - + m.x.fix(1) opt.config.tee = True res = opt.solve(m) From 9e7cd0dc9f4679a7b19b4706b2ffcf963bb7dc3f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 14 Aug 2025 07:15:39 -0600 Subject: [PATCH 058/104] moving scip to contrib solvers --- .../solvers => contrib/solver/solvers/scip}/scip_direct.py | 0 .../solvers => contrib/solver/solvers/scip}/scip_persistent.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename pyomo/{solvers/plugins/solvers => contrib/solver/solvers/scip}/scip_direct.py (100%) rename pyomo/{solvers/plugins/solvers => contrib/solver/solvers/scip}/scip_persistent.py (100%) diff --git a/pyomo/solvers/plugins/solvers/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py similarity index 100% rename from pyomo/solvers/plugins/solvers/scip_direct.py rename to pyomo/contrib/solver/solvers/scip/scip_direct.py diff --git a/pyomo/solvers/plugins/solvers/scip_persistent.py b/pyomo/contrib/solver/solvers/scip/scip_persistent.py similarity index 100% rename from pyomo/solvers/plugins/solvers/scip_persistent.py rename to pyomo/contrib/solver/solvers/scip/scip_persistent.py From bf204bb2b6d21644fa5dd8f0d83521075541cb9f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 14 Aug 2025 07:15:57 -0600 Subject: [PATCH 059/104] moving scip to contrib solvers --- pyomo/contrib/solver/solvers/scip/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 pyomo/contrib/solver/solvers/scip/__init__.py diff --git a/pyomo/contrib/solver/solvers/scip/__init__.py b/pyomo/contrib/solver/solvers/scip/__init__.py new file mode 100644 index 00000000000..e69de29bb2d From dabf031e34490ed6dcf00e25eaba68abafc7a022 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 14 Aug 2025 23:36:24 -0600 Subject: [PATCH 060/104] porting scip interface --- .../contrib/solver/common/solution_loader.py | 20 +- .../solvers/gurobi/gurobi_direct_base.py | 1 - .../solver/solvers/scip/scip_direct.py | 1228 ++++++++--------- 3 files changed, 604 insertions(+), 645 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index 666ea66e1e9..f8723b6e0f4 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -18,6 +18,10 @@ from pyomo.core.staleflag import StaleFlagManager from pyomo.core.base.suffix import Suffix from .util import NoSolutionError +import logging + + +logger = logging.getLogger(__name__) def load_import_suffixes( @@ -33,11 +37,19 @@ def load_import_suffixes( elif suffix.local_name == 'rc': rc_suffix = suffix if dual_suffix is not None: - for k, v in solution_loader.get_duals(solution_id=solution_id).items(): - dual_suffix[k] = v + duals = solution_loader.get_duals(solution_id=solution_id) + if duals is NotImplemented: + logger.warning(f'Cannot load duals into suffix') + else: + for k, v in duals.items(): + dual_suffix[k] = v if rc_suffix is not None: - for k, v in solution_loader.get_reduced_costs(solution_id=solution_id).items(): - rc_suffix[k] = v + rc = solution_loader.get_reduced_costs(solution_id=solution_id) + if rc is NotImplemented: + logger.warning(f'cannot load duals into suffix') + else: + for k, v in rc.items(): + rc_suffix[k] = v class SolutionLoaderBase: diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py index ce77c31c6f7..8989fc5047a 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_direct_base.py @@ -362,7 +362,6 @@ def solve(self, model, **kwds) -> Results: # hack to work around legacy solver wrapper __setattr__ # otherwise, this would just be self.config = orig_config object.__setattr__(self, 'config', orig_config) - self.config = orig_config res.solver_log = ostreams[0].getvalue() end_timestamp = datetime.datetime.now(datetime.timezone.utc) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index c862d9047c1..b8d4d14a6c1 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -9,392 +9,578 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +import datetime +import io import logging -import sys - -from pyomo.common.collections import ComponentSet, ComponentMap, Bunch -from pyomo.common.tempfiles import TempfileManager -from pyomo.core import Var +from typing import Tuple, List, Optional, Sequence, Mapping, Dict + +from pyomo.common.collections import ComponentMap +from pyomo.common.numeric_types import native_numeric_types +from pyomo.common.errors import InfeasibleConstraintException, ApplicationError +from pyomo.common.timing import HierarchicalTimer +from pyomo.core.base.block import BlockData +from pyomo.core.base.var import VarData, ScalarVar +from pyomo.core.base.param import ParamData, ScalarParam +from pyomo.core.base.constraint import Constraint, ConstraintData +from pyomo.core.base.sos import SOSConstraint, SOSConstraintData +from pyomo.core.kernel.objective import minimize, maximize from pyomo.core.expr.numeric_expr import ( - SumExpression, - ProductExpression, - UnaryFunctionExpression, + NegationExpression, PowExpression, + ProductExpression, + MonomialTermExpression, DivisionExpression, + SumExpression, + LinearExpression, + UnaryFunctionExpression, + NPV_NegationExpression, + NPV_PowExpression, + NPV_ProductExpression, + NPV_DivisionExpression, + NPV_SumExpression, + NPV_UnaryFunctionExpression, ) -from pyomo.core.expr.numvalue import is_fixed -from pyomo.core.expr.numvalue import value +from pyomo.core.base.expression import ExpressionData, ScalarExpression +from pyomo.core.expr.relational_expr import EqualityExpression, InequalityExpression, RangedExpression from pyomo.core.staleflag import StaleFlagManager -from pyomo.repn import generate_standard_repn -from pyomo.solvers.plugins.solvers.direct_solver import DirectSolver -from pyomo.solvers.plugins.solvers.direct_or_persistent_solver import ( - DirectOrPersistentSolver, +from pyomo.core.expr.visitor import StreamBasedExpressionVisitor +from pyomo.common.dependencies import attempt_import +from pyomo.contrib.solver.common.base import SolverBase, Availability +from pyomo.contrib.solver.common.config import BranchAndBoundConfig +from pyomo.contrib.solver.common.util import ( + NoFeasibleSolutionError, + NoOptimalSolutionError, ) -from pyomo.core.kernel.objective import minimize, maximize -from pyomo.opt.results.results_ import SolverResults -from pyomo.opt.results.solution import Solution, SolutionStatus -from pyomo.opt.results.solver import TerminationCondition, SolverStatus -from pyomo.opt.base import SolverFactory +from pyomo.contrib.solver.common.util import get_objective +from pyomo.contrib.solver.common.solution_loader import NoSolutionSolutionLoader +from pyomo.contrib.solver.common.results import ( + Results, + SolutionStatus, + TerminationCondition, +) +from pyomo.contrib.solver.common.solution_loader import ( + SolutionLoaderBase, + load_import_suffixes, +) +from pyomo.common.config import ConfigValue +from pyomo.common.tee import capture_output, TeeStream -logger = logging.getLogger("pyomo.solvers") +logger = logging.getLogger(__name__) -class DegreeError(ValueError): - pass +scip, scip_available = attempt_import('pyscipyopt') -def _is_numeric(x): - try: - float(x) - except ValueError: - return False - return True +class ScipConfig(BranchAndBoundConfig): + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + BranchAndBoundConfig.__init__( + self, + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + self.use_mipstart: bool = self.declare( + 'use_mipstart', + ConfigValue( + default=False, + domain=bool, + description="If True, the current values of the integer variables " + "will be passed to Scip.", + ), + ) -@SolverFactory.register("scip_direct", doc="Direct python interface to SCIP") -class SCIPDirect(DirectSolver): +def _handle_var(node, data, opt): + if id(node) not in opt._pyomo_var_to_solver_var_map: + scip_var = opt._add_var(node) + else: + scip_var = opt._pyomo_var_to_solver_var_map[id(node)] + return scip_var - def __init__(self, **kwds): - kwds["type"] = "scipdirect" - DirectSolver.__init__(self, **kwds) - self._init() - self._solver_model = None - def _init(self): - try: - import pyscipopt +def _handle_param(node, data, opt): + if not node.mutable: + return node.value + if id(node) not in opt._pyomo_param_to_solver_param_map: + scip_param = opt._add_param(node) + else: + scip_param = opt._pyomo_param_to_solver_param_map[id(node)] + return scip_param - self._scip = pyscipopt - self._python_api_exists = True - self._version = tuple( - int(k) for k in str(self._scip.Model().version()).split(".") - ) - self._version_major = self._version[0] - except ImportError: - self._python_api_exists = False - except Exception as e: - print(f"Import of pyscipopt failed - SCIP message={str(e)}\n") - self._python_api_exists = False - - # Note: Undefined capabilities default to None - self._max_constraint_degree = None - self._max_obj_degree = 1 - self._capabilities.linear = True - self._capabilities.quadratic_objective = False - self._capabilities.quadratic_constraint = True - self._capabilities.integer = True - self._capabilities.sos1 = True - self._capabilities.sos2 = True - self._skip_trivial_constraints = True - - # Dictionary used exclusively for SCIP, as we want the constraint expressions - self._pyomo_var_to_solver_var_expr_map = ComponentMap() - self._pyomo_con_to_solver_con_expr_map = dict() - - def _apply_solver(self): - StaleFlagManager.mark_all_as_stale() - - # Suppress solver output if requested - if self._tee: - self._solver_model.hideOutput(quiet=False) - else: - self._solver_model.hideOutput(quiet=True) - # Redirect solver output to a logfile if requested - if self._keepfiles: - # Only save log file when the user wants to keep it. - self._solver_model.setLogfile(self._log_file) - print(f"Solver log file: {self._log_file}") +def _handle_float(node, data, opt): + return float(node) - # Set user specified parameters - for key, option in self.options.items(): - try: - key_type = type(self._solver_model.getParam(key)) - except KeyError: - raise ValueError(f"Key {key} is an invalid parameter for SCIP") - if key_type == str: - self._solver_model.setParam(key, option) - else: - if not _is_numeric(option): - raise ValueError( - f"Value {option} for parameter {key} is not a string and can't be converted to float" - ) - self._solver_model.setParam(key, float(option)) - - self._solver_model.optimize() - - # TODO: Check if this is even needed, or if it is sufficient to close the open file - # if self._keepfiles: - # self._solver_model.setLogfile(None) - - # FIXME: can we get a return code indicating if SCIP had a significant failure? - return Bunch(rc=None, log=None) - - def _get_expr_from_pyomo_repn(self, repn, max_degree=None): - referenced_vars = ComponentSet() - - degree = repn.polynomial_degree() - if (max_degree is not None) and (degree > max_degree): - raise DegreeError( - "While SCIP supports general non-linear constraints, the objective must be linear. " - "Please reformulate the objective by introducing a new variable. " - "For min problems: min z s.t z >= f(x). For max problems: max z s.t z <= f(x). " - "f(x) is the original non-linear objective." - ) +def _handle_negation(node, data, opt): + return -data[0] - new_expr = repn.constant - if len(repn.linear_vars) > 0: - referenced_vars.update(repn.linear_vars) - new_expr += sum( - repn.linear_coefs[i] * self._pyomo_var_to_solver_var_expr_map[var] - for i, var in enumerate(repn.linear_vars) - ) +def _handle_pow(node, data, opt): + return data[0] ** data[1] - for i, v in enumerate(repn.quadratic_vars): - x, y = v - new_expr += ( - repn.quadratic_coefs[i] - * self._pyomo_var_to_solver_var_expr_map[x] - * self._pyomo_var_to_solver_var_expr_map[y] - ) - referenced_vars.add(x) - referenced_vars.add(y) - - if repn.nonlinear_expr is not None: - - def get_nl_expr_recursively(pyomo_expr): - if not hasattr(pyomo_expr, "args"): - if not isinstance(pyomo_expr, Var): - return float(pyomo_expr) - else: - referenced_vars.add(pyomo_expr) - return self._pyomo_var_to_solver_var_expr_map[pyomo_expr] - scip_expr_list = [0 for i in range(pyomo_expr.nargs())] - for i in range(pyomo_expr.nargs()): - scip_expr_list[i] = get_nl_expr_recursively(pyomo_expr.args[i]) - if isinstance(pyomo_expr, PowExpression): - if len(scip_expr_list) != 2: - raise ValueError( - f"PowExpression has {len(scip_expr_list)} many terms instead of two!" - ) - return scip_expr_list[0] ** (scip_expr_list[1]) - elif isinstance(pyomo_expr, ProductExpression): - return self._scip.quickprod(scip_expr_list) - elif isinstance(pyomo_expr, SumExpression): - return self._scip.quicksum(scip_expr_list) - elif isinstance(pyomo_expr, DivisionExpression): - if len(scip_expr_list) != 2: - raise ValueError( - f"DivisionExpression has {len(scip_expr_list)} many terms instead of two!" - ) - return scip_expr_list[0] / scip_expr_list[1] - elif isinstance(pyomo_expr, UnaryFunctionExpression): - if len(scip_expr_list) != 1: - raise ValueError( - f"UnaryExpression has {len(scip_expr_list)} many terms instead of one!" - ) - if pyomo_expr.name == "sin": - return self._scip.sin(scip_expr_list[0]) - elif pyomo_expr.name == "cos": - return self._scip.cos(scip_expr_list[0]) - elif pyomo_expr.name == "exp": - return self._scip.exp(scip_expr_list[0]) - elif pyomo_expr.name == "log": - return self._scip.log(scip_expr_list[0]) - else: - raise NotImplementedError( - f"PySCIPOpt through Pyomo does not support the unary function {pyomo_expr.name}" - ) - else: - raise NotImplementedError( - f"PySCIPOpt through Pyomo does not yet support expression type {type(pyomo_expr)}" - ) - new_expr += get_nl_expr_recursively(repn.nonlinear_expr) +def _handle_product(node, data, opt): + assert len(data) == 2 + return data[0] * data[1] + + +def _handle_division(node, data, opt): + return data[0] / data[1] + + +def _handle_sum(node, data, opt): + return sum(data) + + +def _handle_exp(node, data, opt): + return scip.exp(data[0]) + + +def _handle_log(node, data, opt): + return scip.log(data[0]) + + +def _handle_sin(node, data, opt): + return scip.sin(data[0]) + + +def _handle_cos(node, data, opt): + return scip.cos(data[0]) + + +def _handle_sqrt(node, data, opt): + return scip.sqrt(data[0]) + + +def _handle_abs(node, data, opt): + return abs(data[0]) + - return new_expr, referenced_vars +def _handle_tan(node, data, opt): + return scip.sin(data[0]) / scip.cos(data[0]) - def _get_expr_from_pyomo_expr(self, expr, max_degree=None): - if max_degree is None or max_degree >= 2: - repn = generate_standard_repn(expr, quadratic=True) + +_unary_map = { + 'exp': _handle_exp, + 'log': _handle_log, + 'sin': _handle_sin, + 'cos': _handle_cos, + 'sqrt': _handle_sqrt, + 'abs': _handle_abs, + 'tan': _handle_tan, +} + + +def _handle_unary(node, data, opt): + if node.getname() in _unary_map: + return _unary_map[node.getname()](node, data, opt) + else: + raise NotImplementedError(f'unable to handle unary expression: {str(node)}') + + +def _handle_equality(node, data, opt): + return data[0] == data[1] + + +def _handle_ranged(node, data, opt): + return data[0] <= (data[1] <= data[2]) + + +def _handle_inequality(node, data, opt): + return data[0] <= data[1] + + +def _handle_named_expression(node, data, opt): + return data[0] + + +_operator_map = { + NegationExpression: _handle_negation, + PowExpression: _handle_pow, + ProductExpression: _handle_product, + MonomialTermExpression: _handle_product, + DivisionExpression: _handle_division, + SumExpression: _handle_sum, + LinearExpression: _handle_sum, + UnaryFunctionExpression: _handle_unary, + NPV_NegationExpression: _handle_negation, + NPV_PowExpression: _handle_pow, + NPV_ProductExpression: _handle_product, + NPV_DivisionExpression: _handle_division, + NPV_SumExpression: _handle_sum, + NPV_UnaryFunctionExpression: _handle_unary, + EqualityExpression: _handle_equality, + RangedExpression: _handle_ranged, + InequalityExpression: _handle_inequality, + ScalarExpression: _handle_named_expression, + ExpressionData: _handle_named_expression, + VarData: _handle_var, + ScalarVar: _handle_var, + ParamData: _handle_param, + ScalarParam: _handle_param, + float: _handle_float, + int: _handle_float, +} + + +class _PyomoToScipVisitor(StreamBasedExpressionVisitor): + def __init__(self, solver, **kwds): + super().__init__(**kwds) + self.solver = solver + + def exitNode(self, node, data): + nt = type(node) + if nt in _operator_map: + return _operator_map[nt](node, data, self.solver) + elif nt in native_numeric_types: + _operator_map[nt] = _handle_float + return _handle_float(node, data, self.solver) else: - repn = generate_standard_repn(expr, quadratic=False) + raise NotImplementedError(f'unrecognized expression type: {nt}') + + +logger = logging.getLogger("pyomo.solvers") + + +class ScipDirectSolutionLoader(SolutionLoaderBase): + def __init__( + self, + solver_model, + var_id_map, + var_map, + con_map, + pyomo_model, + opt, + ) -> None: + super().__init__() + self._solver_model = solver_model + self._vars = var_id_map + self._var_map = var_map + self._con_map = con_map + self._pyomo_model = pyomo_model + # make sure the scip model does not get freed until the solution loader is garbage collected + self._opt = opt + + def get_number_of_solutions(self) -> int: + return self._solver_model.getNSols() + + def get_solution_ids(self) -> List: + return list(range(self.get_number_of_solutions())) + + def load_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None + ) -> None: + for v, val in self.get_vars(vars_to_load=vars_to_load, solution_id=solution_id).items(): + v.value = val + + def get_vars( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None + ) -> Mapping[VarData, float]: + if vars_to_load is None: + vars_to_load = list(self._vars.values()) + if solution_id is None: + solution_id = 0 + sol = self._solver_model.getSols()[solution_id] + res = ComponentMap() + for v in vars_to_load: + sv = self._var_map[id(v)] + res[v] = sol[sv] + return res + + def get_reduced_costs( + self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None + ) -> Mapping[VarData, float]: + return NotImplemented + + def get_duals( + self, cons_to_load: Optional[Sequence[ConstraintData]] = None, solution_id=None + ) -> Dict[ConstraintData, float]: + return NotImplemented + + def load_import_suffixes(self, solution_id=None): + load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) + + +class SCIPDirect(SolverBase): - scip_expr, referenced_vars = self._get_expr_from_pyomo_repn(repn, max_degree) + _available = None + _tc_map = None + _minimum_version = (5, 5, 0) # this is probably conservative - return scip_expr, referenced_vars + CONFIG = ScipConfig() + + def __init__(self, **kwds): + super().__init__(**kwds) + self._solver_model = None + self._vars = {} # var id to var + self._params = {} # param id to param + self._pyomo_var_to_solver_var_map = {} # var id to scip var + self._pyomo_con_to_solver_con_map = {} + self._pyomo_param_to_solver_param_map = {} # param id to scip var with equal bounds + self._pyomo_sos_to_solver_sos_map = {} + self._expr_visitor = _PyomoToScipVisitor(self) + self._objective = None # pyomo objective + self._obj_var = None # a scip variable because the objective cannot be nonlinear + self._obj_con = None # a scip constraint (obj_var >= obj_expr) + + def _clear(self): + self._solver_model = None + self._vars = {} + self._params = {} + self._pyomo_var_to_solver_var_map = {} + self._pyomo_con_to_solver_con_map = {} + self._pyomo_param_to_solver_param_map = {} + self._pyomo_sos_to_solver_sos_map = {} + self._objective = None + self._obj_var = None + self._obj_con = None + + def available(self) -> Availability: + if self._available is not None: + return self._available + + if not scip_available: + SCIPDirect._available = Availability.NotFound + elif self.version() < self._minimum_version: + SCIPDirect._available = Availability.BadVersion + else: + SCIPDirect._available = Availability.FullLicense + + return self._available + + def version(self) -> Tuple: + return tuple(int(i) for i in scip.__version__) + + def solve(self, model: BlockData, **kwargs) -> Results: + start_timestamp = datetime.datetime.now(datetime.timezone.utc) + orig_config = self.config + if not self.available(): + raise ApplicationError( + f'{self.name} is not available: {self.available()}' + ) + try: + config = self.config(value=kwds, preserve_implicit=True) + + # hack to work around legacy solver wrapper __setattr__ + # otherwise, this would just be self.config = config + object.__setattr__(self, 'config', config) + + StaleFlagManager.mark_all_as_stale() + + if config.timer is None: + config.timer = HierarchicalTimer() + timer = config.timer + + ostreams = [io.StringIO()] + config.tee + + scip_model, solution_loader, has_obj = self._create_solver_model(model) + + scip_model.hideOutput(quiet=False) + if config.threads is not None: + scip_model.setParam('lp/threads', config.threads) + if config.time_limit is not None: + scip_model.setParam('limits/time', config.time_limit) + if config.rel_gap is not None: + scip_model.setParam('limits/gap', config.rel_gap) + if config.abs_gap is not None: + scip_model.setParam('limits/absgap', config.abs_gap) + + if config.use_mipstart: + self._mipstart() + + for key, option in config.solver_options.items(): + scip_model.setParam(key, option) + + timer.start('optimize') + with capture_output(TeeStream(*ostreams), capture_fd=False): + scip_model.optimize() + timer.stop('optimize') + + results = self._postsolve(scip_model, solution_loader, has_obj) + except InfeasibleConstraintException: + results = self._get_infeasible_results() + finally: + # hack to work around legacy solver wrapper __setattr__ + # otherwise, this would just be self.config = orig_config + object.__setattr__(self, 'config', orig_config) + + results.solver_log = ostreams[0].getvalue() + end_timestamp = datetime.datetime.now(datetime.timezone.utc) + results.timing_info.start_timestamp = start_timestamp + results.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() + results.timing_info.timer = timer + return results + + def _get_tc_map(self): + if SCIPDirect._tc_map is None: + tc = TerminationCondition + SCIPDirect._tc_map = { + "unknown": tc.unknown, + "userinterrupt": tc.interrupted, + "nodelimit": tc.iterationLimit, + "totalnodelimit": tc.iterationLimit, + "stallnodelimit": tc.iterationLimit, + "timelimit": tc.maxTimeLimit, + "memlimit": tc.unknown, + "gaplimit": tc.convergenceCriteriaSatisfied, # TODO: check this + "primallimit": tc.objectiveLimit, + "duallimit": tc.objectiveLimit, + "sollimit": tc.unknown, + "bestsollimit": tc.unknown, + "restartlimit": tc.unknown, + "optimal": tc.convergenceCriteriaSatisfied, + "infeasible": tc.provenInfeasible, + "unbounded": tc.unbounded, + "inforunbd": tc.infeasibleOrUnbounded, + "terminate": tc.unknown, + } + return SCIPDirect._tc_map + + def _get_infeasible_results(self): + res = Results() + res.solution_loader = NoSolutionSolutionLoader() + res.solution_status = SolutionStatus.noSolution + res.termination_condition = TerminationCondition.provenInfeasible + res.incumbent_objective = None + res.objective_bound = None + res.iteration_count = None + res.timing_info.scip_time = None + res.solver_config = self.config + res.solver_name = self.name + res.solver_version = self.version() + if self.config.raise_exception_on_nonoptimal_result: + raise NoOptimalSolutionError() + if self.config.load_solutions: + raise NoFeasibleSolutionError() + return res def _scip_lb_ub_from_var(self, var): if var.is_fixed(): val = var.value return val, val - if var.has_lb(): - lb = value(var.lb) - else: + + lb, ub = var.bounds() + + if lb is None: lb = -self._solver_model.infinity() - if var.has_ub(): - ub = value(var.ub) - else: + if ub is None: ub = self._solver_model.infinity() return lb, ub def _add_var(self, var): - varname = self._symbol_map.getSymbol(var, self._labeler) vtype = self._scip_vtype_from_var(var) lb, ub = self._scip_lb_ub_from_var(var) - scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype, name=varname) - - self._pyomo_var_to_solver_var_expr_map[var] = scip_var - self._pyomo_var_to_solver_var_map[var] = scip_var.name - self._solver_var_to_pyomo_var_map[varname] = var - self._referenced_variables[var] = 0 - - def close(self): + scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype) + + self._vars[id(var)] = var + self._pyomo_var_to_solver_var_map[id(var)] = scip_var + return scip_var + + def _add_param(self, p): + vtype = "C" + lb = ub = p.value + scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype) + self._params[id(p)] = p + self._pyomo_param_to_solver_param_map[id(p)] = scip_var + return scip_var + + def __del__(self): """Frees SCIP resources used by this solver instance.""" - if self._solver_model is not None: self._solver_model.freeProb() self._solver_model = None - def __exit__(self, t, v, traceback): - super().__exit__(t, v, traceback) - self.close() - - def _set_instance(self, model, kwds={}): - DirectOrPersistentSolver._set_instance(self, model, kwds) - self.available() - try: - self._solver_model = self._scip.Model() - except Exception: - e = sys.exc_info()[1] - msg = ( - "Unable to create SCIP model. " - f"Have you installed PySCIPOpt correctly?\n\n\t Error message: {e}" + def _add_constraints(self, cons: List[ConstraintData]): + for con in cons: + self._add_constraint(con) + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + for on in cons: + self._add_sos_constraint(con) + + def _create_solver_model(self, model): + timer = self.config.timer + timer.start('create scip model') + self._clear() + self._solver_model = scip.Model() + timer.start('collect constraints') + cons = list( + model.component_data_objects( + Constraint, descend_into=True, active=True ) - raise Exception(msg) - - self._add_block(model) - - for var, n_ref in self._referenced_variables.items(): - if n_ref != 0: - if var.fixed: - if not self._output_fixed_variable_bounds: - raise ValueError( - f"Encountered a fixed variable {var.name} inside " - "an active objective or constraint " - f"expression on model {self._pyomo_model.name}, which is usually " - "indicative of a preprocessing error. Use " - "the IO-option 'output_fixed_variable_bounds=True' " - "to suppress this error and fix the variable " - "by overwriting its bounds in the SCIP instance." - ) - - def _add_block(self, block): - DirectOrPersistentSolver._add_block(self, block) - - def _add_constraint(self, con): - if not con.active: - return None - - if is_fixed(con.body) and self._skip_trivial_constraints: - return None - - conname = self._symbol_map.getSymbol(con, self._labeler) - - if con._linear_canonical_form: - scip_expr, referenced_vars = self._get_expr_from_pyomo_repn( - con.canonical_form(), self._max_constraint_degree - ) - else: - scip_expr, referenced_vars = self._get_expr_from_pyomo_expr( - con.body, self._max_constraint_degree - ) - - if con.has_lb(): - if not is_fixed(con.lower): - raise ValueError(f"Lower bound of constraint {con} is not constant.") - con_lower = value(con.lower) - if type(con_lower) != float and type(con_lower) != int: - logger.warning( - f"Constraint {conname} has LHS type {type(value(con.lower))}. " - f"Converting to float as SCIP fails otherwise." - ) - con_lower = float(con_lower) - if con.has_ub(): - if not is_fixed(con.upper): - raise ValueError(f"Upper bound of constraint {con} is not constant.") - con_upper = value(con.upper) - - if con.equality: - scip_cons = self._solver_model.addCons(scip_expr == con_lower, name=conname) - elif con.has_lb() and con.has_ub(): - scip_cons = self._solver_model.addCons(con_lower <= scip_expr, name=conname) - rhs = con_upper - if hasattr(con.body, "constant"): - con_constant = value(con.body.constant) - if not isinstance(con_constant, (float, int)): - con_constant = float(con_constant) - rhs -= con_constant - self._solver_model.chgRhs(scip_cons, rhs) - elif con.has_lb(): - scip_cons = self._solver_model.addCons(con_lower <= scip_expr, name=conname) - elif con.has_ub(): - scip_cons = self._solver_model.addCons(scip_expr <= con_upper, name=conname) - else: - raise ValueError( - f"Constraint does not have a lower or an upper bound: {con} \n" + ) + timer.stop('collect constraints') + timer.start('translate constraints') + self._add_constraints(cons) + timer.stop('translate constraints') + timer.start('sos') + sos = list( + model.component_data_objects( + SOSConstraint, descend_into=True, active=True ) + ) + self._add_sos_constraints(sos) + timer.stop('sos') + timer.start('get objective') + obj = get_objective(model) + timer.stop('get objective') + timer.start('translate objective') + self._set_objective(obj) + timer.stop('translate objective') + has_obj = obj is not None + solution_loader = ScipDirectSolutionLoader( + solver_model=self._solver_model, + var_id_map=self._vars, + var_map=self._pyomo_var_to_solver_var_map, + con_map=self._pyomo_con_to_solver_con_map, + pyomo_model=model, + opt=self, + ) + timer.stop('create scip model') + return self._solver_model, solution_loader, has_obj - for var in referenced_vars: - self._referenced_variables[var] += 1 - self._vars_referenced_by_con[con] = referenced_vars - self._pyomo_con_to_solver_con_expr_map[con] = scip_cons - self._pyomo_con_to_solver_con_map[con] = scip_cons.name - self._solver_con_to_pyomo_con_map[conname] = con + def _add_constraint(self, con): + scip_expr = self._expr_visitor.walk_expression(con.expr) + scip_con = self._solver_model.addCons(scip_expr) + self._pyomo_con_to_solver_con_map[con] = scip_con def _add_sos_constraint(self, con): - if not con.active: - return None - - conname = self._symbol_map.getSymbol(con, self._labeler) level = con.level if level not in [1, 2]: - raise ValueError(f"Solver does not support SOS level {level} constraints") + raise ValueError(f"{self.name} does not support SOS level {level} constraints") scip_vars = [] weights = [] - self._vars_referenced_by_con[con] = ComponentSet() - - if hasattr(con, "get_items"): - # aml sos constraint - sos_items = list(con.get_items()) - else: - # kernel sos constraint - sos_items = list(con.items()) - - for v, w in sos_items: - self._vars_referenced_by_con[con].add(v) - scip_vars.append(self._pyomo_var_to_solver_var_expr_map[v]) - self._referenced_variables[v] += 1 + for v, w in con.get_items(): + vid = id(v) + if vid not in self._pyomo_var_to_solver_var_map: + self._add_var(v) + scip_vars.append(self._pyomo_var_to_solver_var_map[vid]) weights.append(w) if level == 1: scip_cons = self._solver_model.addConsSOS1( - scip_vars, weights=weights, name=conname + scip_vars, weights=weights ) else: scip_cons = self._solver_model.addConsSOS2( - scip_vars, weights=weights, name=conname + scip_vars, weights=weights ) - self._pyomo_con_to_solver_con_expr_map[con] = scip_cons - self._pyomo_con_to_solver_con_map[con] = scip_cons.name - self._solver_con_to_pyomo_con_map[conname] = con + self._pyomo_con_to_solver_con_map[con] = scip_cons def _scip_vtype_from_var(self, var): """ @@ -421,342 +607,104 @@ def _scip_vtype_from_var(self, var): return vtype def _set_objective(self, obj): + if self._obj_var is None: + self._obj_var = self._solver_model.addVar( + lb=-self._solver_model.infinity(), + ub=self._solver_model.infinity(), + vtype="C" + ) + if self._objective is not None: - for var in self._vars_referenced_by_obj: - self._referenced_variables[var] -= 1 - self._vars_referenced_by_obj = ComponentSet() - self._objective = None + self._solver_model.delCons(self._obj_con) - if obj.active is False: - raise ValueError("Cannot add inactive objective to solver.") + if obj is None: + scip_expr = 0 + else: + scip_expr = self._expr_visitor.walk_expression(obj.expr) if obj.sense == minimize: sense = "minimize" + self._obj_con = self._solver_model.addCons(self._obj_var >= scip_expr) elif obj.sense == maximize: sense = "maximize" + self._obj_con = self._solver_model.addCons(self._obj_var <= scip_expr) else: raise ValueError(f"Objective sense is not recognized: {obj.sense}") - scip_expr, referenced_vars = self._get_expr_from_pyomo_expr( - obj.expr, self._max_obj_degree - ) - - for var in referenced_vars: - self._referenced_variables[var] += 1 - - self._solver_model.setObjective(scip_expr, sense=sense) + self._solver_model.setObjective(self._obj_var, sense=sense) self._objective = obj - self._vars_referenced_by_obj = referenced_vars - - def _get_solver_solution_status(self, scip, soln): - """ """ - # Get the status of the SCIP Model currently - status = scip.getStatus() - - # Go through each potential case and update appropriately - if scip.getStage() == 1: # SCIP Model is created but not yet optimized - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Model is loaded, but no solution information is available." - ) - self.results.solver.termination_condition = TerminationCondition.error - soln.status = SolutionStatus.unknown - elif status == "optimal": # optimal - self.results.solver.status = SolverStatus.ok - self.results.solver.termination_message = ( - "Model was solved to optimality (subject to tolerances), " - "and an optimal solution is available." - ) - self.results.solver.termination_condition = TerminationCondition.optimal - soln.status = SolutionStatus.optimal - elif status == "infeasible": - self.results.solver.status = SolverStatus.warning - self.results.solver.termination_message = ( - "Model was proven to be infeasible" - ) - self.results.solver.termination_condition = TerminationCondition.infeasible - soln.status = SolutionStatus.infeasible - elif status == "inforunbd": - self.results.solver.status = SolverStatus.warning - self.results.solver.termination_message = ( - "Problem proven to be infeasible or unbounded." - ) - self.results.solver.termination_condition = ( - TerminationCondition.infeasibleOrUnbounded - ) - soln.status = SolutionStatus.unsure - elif status == "unbounded": - self.results.solver.status = SolverStatus.warning - self.results.solver.termination_message = ( - "Model was proven to be unbounded." - ) - self.results.solver.termination_condition = TerminationCondition.unbounded - soln.status = SolutionStatus.unbounded - elif status == "gaplimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the gap dropped below " - "the value specified in the " - "limits/gap parameter." - ) - self.results.solver.termination_condition = TerminationCondition.unknown - soln.status = SolutionStatus.stoppedByLimit - elif status == "stallnodelimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the stalling node limit " - "exceeded the value specified in the " - "limits/stallnodes parameter." - ) - self.results.solver.termination_condition = TerminationCondition.unknown - soln.status = SolutionStatus.stoppedByLimit - elif status == "restartlimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the total number of restarts " - "exceeded the value specified in the " - "limits/restarts parameter." - ) - self.results.solver.termination_condition = TerminationCondition.unknown - soln.status = SolutionStatus.stoppedByLimit - elif status == "nodelimit" or status == "totalnodelimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the number of " - "branch-and-cut nodes explored exceeded the limits specified " - "in the limits/nodes or limits/totalnodes parameter" - ) - self.results.solver.termination_condition = ( - TerminationCondition.maxEvaluations - ) - soln.status = SolutionStatus.stoppedByLimit - elif status == "timelimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the time expended exceeded " - "the value specified in the limits/time parameter." - ) - self.results.solver.termination_condition = ( - TerminationCondition.maxTimeLimit - ) - soln.status = SolutionStatus.stoppedByLimit - elif status == "sollimit" or status == "bestsollimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the number of solutions found " - "reached the value specified in the limits/solutions or" - "limits/bestsol parameter." - ) - self.results.solver.termination_condition = TerminationCondition.unknown - soln.status = SolutionStatus.stoppedByLimit - elif status == "memlimit": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization terminated because the memory used exceeded " - "the value specified in the limits/memory parameter." - ) - self.results.solver.termination_condition = TerminationCondition.unknown - soln.status = SolutionStatus.stoppedByLimit - elif status == "userinterrupt": - self.results.solver.status = SolverStatus.aborted - self.results.solver.termination_message = ( - "Optimization was terminated by the user." - ) - self.results.solver.termination_condition = TerminationCondition.error - soln.status = SolutionStatus.error - else: - self.results.solver.status = SolverStatus.error - self.results.solver.termination_message = ( - f"Unhandled SCIP status ({str(status)})" - ) - self.results.solver.termination_condition = TerminationCondition.error - soln.status = SolutionStatus.error - return soln - - def _postsolve(self): - # Constraint duals and variable - # reduced-costs were removed as in SCIP they contain - # too many caveats. Slacks were removed as later - # planned interfaces do not intend to support. - # Scan through the solver suffix list - # and throw an exception if the user has specified - # any others. - for suffix in self._suffixes: - raise RuntimeError( - f"***The scip_direct solver plugin cannot extract solution suffix={suffix}" - ) - - scip = self._solver_model - status = scip.getStatus() - scip_vars = scip.getVars() - n_bin_vars = sum([scip_var.vtype() == "BINARY" for scip_var in scip_vars]) - n_int_vars = sum([scip_var.vtype() == "INTEGER" for scip_var in scip_vars]) - n_con_vars = sum([scip_var.vtype() == "CONTINUOUS" for scip_var in scip_vars]) - - self.results = SolverResults() - soln = Solution() - - self.results.solver.name = f"SCIP{self._version}" - self.results.solver.wallclock_time = scip.getSolvingTime() - - soln = self._get_solver_solution_status(scip, soln) - self.results.problem.name = scip.getProbName() - - self.results.problem.upper_bound = None - self.results.problem.lower_bound = None - if scip.getNSols() > 0: - scip_has_sol = True - else: - scip_has_sol = False - if not scip_has_sol and (status == "inforunbd" or status == "infeasible"): - pass + def _postsolve( + self, + scip_model, + solution_loader: ScipDirectSolutionLoader, + has_obj + ): + + results = Results() + results.solution_loader = solution_loader + results.timing_info.scip_time = scip_model.getSolvingTime() + results.termination_condition = self._get_tc_map().get(scip_model.getStatus(), TerminationCondition.unknown) + + if solution_loader.get_number_of_solutions() > 0: + if results.termination_condition == TerminationCondition.convergenceCriteriaSatisfied: + results.solution_status = SolutionStatus.optimal + else: + results.solution_status = SolutionStatus.feasible else: - if n_bin_vars + n_int_vars == 0: - self.results.problem.upper_bound = scip.getObjVal() - self.results.problem.lower_bound = scip.getObjVal() - elif scip.getObjectiveSense() == "minimize": # minimizing - if scip_has_sol: - self.results.problem.upper_bound = scip.getObjVal() - else: - self.results.problem.upper_bound = scip.infinity() - self.results.problem.lower_bound = scip.getDualbound() - else: # maximizing - self.results.problem.upper_bound = scip.getDualbound() - if scip_has_sol: - self.results.problem.lower_bound = scip.getObjVal() + results.solution_status = SolutionStatus.noSolution + + if ( + results.termination_condition + != TerminationCondition.convergenceCriteriaSatisfied + and self.config.raise_exception_on_nonoptimal_result + ): + raise NoOptimalSolutionError() + + if has_obj: + try: + if scip_model.getObjVal() < scip_model.infinity(): + results.incumbent_objective = scip_model.getObjVal() else: - self.results.problem.lower_bound = -scip.infinity() - + results.incumbent_objective = None + except: + results.incumbent_objective = None try: - soln.gap = ( - self.results.problem.upper_bound - self.results.problem.lower_bound - ) - except TypeError: - soln.gap = None - - self.results.problem.number_of_constraints = scip.getNConss(transformed=False) - self.results.problem.number_of_variables = scip.getNVars(transformed=False) - self.results.problem.number_of_binary_variables = n_bin_vars - self.results.problem.number_of_integer_variables = n_int_vars - self.results.problem.number_of_continuous_variables = n_con_vars - self.results.problem.number_of_objectives = 1 - self.results.problem.number_of_solutions = scip.getNSols() - - # if a solve was stopped by a limit, we still need to check to - # see if there is a solution available - this may not always - # be the case, both in LP and MIP contexts. - if self._save_results: - """ - This code in this if statement is only needed for backwards compatibility. It is more efficient to set - _save_results to False and use load_vars, load_duals, etc. - """ - - if scip.getNSols() > 0: - soln_variables = soln.variable - - scip_vars = scip.getVars() - scip_var_names = [scip_var.name for scip_var in scip_vars] - var_names = set(self._solver_var_to_pyomo_var_map.keys()) - assert set(scip_var_names) == var_names - var_vals = [scip.getVal(scip_var) for scip_var in scip_vars] - - for scip_var, val, name in zip(scip_vars, var_vals, scip_var_names): - pyomo_var = self._solver_var_to_pyomo_var_map[name] - if self._referenced_variables[pyomo_var] > 0: - soln_variables[name] = {"Value": val} - - elif self._load_solutions: - if scip.getNSols() > 0: - self.load_vars() - - self.results.solution.insert(soln) - - # finally, clean any temporary files registered with the temp file - # manager, created populated *directly* by this plugin. - TempfileManager.pop(remove=not self._keepfiles) - - return DirectOrPersistentSolver._postsolve(self) - - def warm_start_capable(self): - return True - - def _warm_start(self): - partial_sol = False - for pyomo_var in self._pyomo_var_to_solver_var_expr_map: - if pyomo_var.value is None: - partial_sol = True - break - if partial_sol: - scip_sol = self._solver_model.createPartialSol() - else: - scip_sol = self._solver_model.createSol() - for pyomo_var, scip_var in self._pyomo_var_to_solver_var_expr_map.items(): - if pyomo_var.value is not None: - scip_sol[scip_var] = value(pyomo_var) - if partial_sol: - self._solver_model.addSol(scip_sol) + results.objective_bound = scip_model.getDualbound() + if results.objective_bound <= -scip_model.infinity(): + results.objective_bound = -math.inf + if results.objective_bound >= scip_model.infinity(): + results.objective_bound = math.inf + except: + if self._objective.sense == minimize: + results.objective_bound = -math.inf + else: + results.objective_bound = math.inf else: - feasible = self._solver_model.checkSol(scip_sol, printreason=not self._tee) - if feasible: - self._solver_model.addSol(scip_sol) - else: - logger.warning("Warm start solution was not accepted by SCIP") - self._solver_model.freeSol(scip_sol) - - def _load_vars(self, vars_to_load=None): - var_map = self._pyomo_var_to_solver_var_expr_map - ref_vars = self._referenced_variables - if vars_to_load is None: - vars_to_load = var_map.keys() - - scip_vars_to_load = [var_map[pyomo_var] for pyomo_var in vars_to_load] - vals = [self._solver_model.getVal(scip_var) for scip_var in scip_vars_to_load] - - for var, val in zip(vars_to_load, vals): - if ref_vars[var] > 0: - var.set_value(val, skip_validation=True) - - def _load_rc(self, vars_to_load=None): - raise NotImplementedError( - "SCIP via Pyomo does not support reduced cost loading." - ) + results.incumbent_objective = None + results.objective_bound = None - def _load_duals(self, cons_to_load=None): - raise NotImplementedError( - "SCIP via Pyomo does not support dual solution loading" - ) - - def _load_slacks(self, cons_to_load=None): - raise NotImplementedError("SCIP via Pyomo does not support slack loading") - - def load_duals(self, cons_to_load=None): - """ - Load the duals into the 'dual' suffix. The 'dual' suffix must live on the parent model. - - Parameters - ---------- - cons_to_load: list of Constraint - """ - self._load_duals(cons_to_load) - - def load_rc(self, vars_to_load): - """ - Load the reduced costs into the 'rc' suffix. The 'rc' suffix must live on the parent model. - - Parameters - ---------- - vars_to_load: list of Var - """ - self._load_rc(vars_to_load) - - def load_slacks(self, cons_to_load=None): - """ - Load the values of the slack variables into the 'slack' suffix. The 'slack' suffix must live on the parent - model. - - Parameters - ---------- - cons_to_load: list of Constraint - """ - self._load_slacks(cons_to_load) + self.config.timer.start('load solution') + if self.config.load_solutions: + if solution_loader.get_number_of_solutions() > 0: + solution_loader.load_solution() + else: + raise NoFeasibleSolutionError() + self.config.timer.stop('load solution') + + results.iteration_count = scip_model.getNNodes() + results.solver_config = self.config + results.solver_name = self.name + results.solver_version = self.version() + + return results + + def _mipstart(self): + # TODO: it is also possible to specify continuous variables, but + # I think we should have a differnt option for that + sol = self._solver_model.createPartialSol() + for vid, scip_var in self._pyomo_var_to_solver_var_map.items(): + pyomo_var = self._vars[vid] + if pyomo_var.is_integer(): + sol[scip_var] = pyomo_var.value + self._solver_model.addSol(sol) From 96017680cd9329626ae18bb2bdf7993f97940f74 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 14 Aug 2025 23:47:44 -0600 Subject: [PATCH 061/104] porting scip interface --- pyomo/contrib/solver/solvers/scip/scip_direct.py | 8 ++++---- pyomo/solvers/plugins/solvers/__init__.py | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index b8d4d14a6c1..5ea3391eecd 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -69,7 +69,7 @@ logger = logging.getLogger(__name__) -scip, scip_available = attempt_import('pyscipyopt') +scip, scip_available = attempt_import('pyscipopt') class ScipConfig(BranchAndBoundConfig): @@ -360,9 +360,9 @@ def available(self) -> Availability: return self._available def version(self) -> Tuple: - return tuple(int(i) for i in scip.__version__) + return tuple(int(i) for i in scip.__version__.split('.')) - def solve(self, model: BlockData, **kwargs) -> Results: + def solve(self, model: BlockData, **kwds) -> Results: start_timestamp = datetime.datetime.now(datetime.timezone.utc) orig_config = self.config if not self.available(): @@ -470,7 +470,7 @@ def _scip_lb_ub_from_var(self, var): val = var.value return val, val - lb, ub = var.bounds() + lb, ub = var.bounds if lb is None: lb = -self._solver_model.infinity() diff --git a/pyomo/solvers/plugins/solvers/__init__.py b/pyomo/solvers/plugins/solvers/__init__.py index 55baaab9de8..cf10af15186 100644 --- a/pyomo/solvers/plugins/solvers/__init__.py +++ b/pyomo/solvers/plugins/solvers/__init__.py @@ -31,8 +31,6 @@ mosek_persistent, xpress_direct, xpress_persistent, - scip_direct, - scip_persistent, SAS, KNITROAMPL, ) From 9d2f22ab5ba29157c7765c82b0fd0e67d34a8e22 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 14 Aug 2025 23:55:03 -0600 Subject: [PATCH 062/104] porting scip interface --- pyomo/contrib/solver/plugins.py | 6 ++++++ pyomo/contrib/solver/tests/solvers/test_solvers.py | 10 +++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index fed739232ad..a4d6f5f9004 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -15,6 +15,7 @@ from .solvers.gurobi.gurobi_direct import GurobiDirect from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent from .solvers.highs import Highs +from .solvers.scip.scip_direct import SCIPDirect def load(): @@ -39,3 +40,8 @@ def load(): SolverFactory.register( name='highs', legacy_name='highs_v2', doc='Persistent interface to HiGHS' )(Highs) + SolverFactory.register( + name='scip_direct', + legacy_name='scip_direct_v2', + doc='Direct interface pyscipopt', + ) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 189b0373780..f4988ca5c8b 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -35,6 +35,7 @@ GurobiDirectQuadratic, GurobiPersistent, ) +from pyomo.contrib.solver.solvers.scip.scip_direct import SCIPDirect from pyomo.contrib.solver.common.util import ( NoSolutionError, NoFeasibleSolutionError, @@ -60,23 +61,30 @@ ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ('highs', Highs), + ('scip_direct', SCIPDirect), ] mip_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), + ('scip_direct', SCIPDirect), +] +nlp_solvers = [ + ('ipopt', Ipopt), + ('scip_direct', SCIPDirect), ] -nlp_solvers = [('ipopt', Ipopt)] qcp_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), + ('scip_direct', SCIPDirect), ] qp_solvers = qcp_solvers + [("highs", Highs)] miqcqp_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), + ('scip_direct', SCIPDirect), ] nl_solvers = [('ipopt', Ipopt)] nl_solvers_set = {i[0] for i in nl_solvers} From b4837295c57ee4005643a19f783f25dff3530be9 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 15 Aug 2025 00:49:53 -0600 Subject: [PATCH 063/104] bugs and tests --- .../solver/solvers/scip/scip_direct.py | 25 ++- .../solver/tests/solvers/test_solvers.py | 180 ++++++++++-------- 2 files changed, 120 insertions(+), 85 deletions(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 5ea3391eecd..1ff470223bd 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -12,6 +12,7 @@ import datetime import io import logging +import math from typing import Tuple, List, Optional, Sequence, Mapping, Dict from pyomo.common.collections import ComponentMap @@ -40,6 +41,7 @@ NPV_SumExpression, NPV_UnaryFunctionExpression, ) +from pyomo.gdp.disjunct import AutoLinkedBinaryVar from pyomo.core.base.expression import ExpressionData, ScalarExpression from pyomo.core.expr.relational_expr import EqualityExpression, InequalityExpression, RangedExpression from pyomo.core.staleflag import StaleFlagManager @@ -50,6 +52,7 @@ from pyomo.contrib.solver.common.util import ( NoFeasibleSolutionError, NoOptimalSolutionError, + NoSolutionError, ) from pyomo.contrib.solver.common.util import get_objective from pyomo.contrib.solver.common.solution_loader import NoSolutionSolutionLoader @@ -109,6 +112,8 @@ def _handle_var(node, data, opt): def _handle_param(node, data, opt): + if not opt.is_persistent(): + return node.value if not node.mutable: return node.value if id(node) not in opt._pyomo_param_to_solver_param_map: @@ -231,6 +236,7 @@ def _handle_named_expression(node, data, opt): ScalarParam: _handle_param, float: _handle_float, int: _handle_float, + AutoLinkedBinaryVar: _handle_var, } @@ -287,6 +293,8 @@ def load_vars( def get_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> Mapping[VarData, float]: + if self.get_number_of_solutions() == 0: + raise NoSolutionError() if vars_to_load is None: vars_to_load = list(self._vars.values()) if solution_id is None: @@ -403,7 +411,7 @@ def solve(self, model: BlockData, **kwds) -> Results: scip_model.setParam(key, option) timer.start('optimize') - with capture_output(TeeStream(*ostreams), capture_fd=False): + with capture_output(TeeStream(*ostreams), capture_fd=True): scip_model.optimize() timer.stop('optimize') @@ -619,17 +627,20 @@ def _set_objective(self, obj): if obj is None: scip_expr = 0 + sense = "minimize" else: scip_expr = self._expr_visitor.walk_expression(obj.expr) + if obj.sense == minimize: + sense = "minimize" + elif obj.sense == maximize: + sense = "maximize" + else: + raise ValueError(f"Objective sense is not recognized: {obj.sense}") - if obj.sense == minimize: - sense = "minimize" + if sense == "minimize": self._obj_con = self._solver_model.addCons(self._obj_var >= scip_expr) - elif obj.sense == maximize: - sense = "maximize" - self._obj_con = self._solver_model.addCons(self._obj_var <= scip_expr) else: - raise ValueError(f"Objective sense is not recognized: {obj.sense}") + self._obj_con = self._solver_model.addCons(self._obj_var <= scip_expr) self._solver_model.setObjective(self._obj_var, sense=sense) self._objective = obj diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index f4988ca5c8b..1b6f122219c 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -88,6 +88,13 @@ ] nl_solvers = [('ipopt', Ipopt)] nl_solvers_set = {i[0] for i in nl_solvers} +dual_solvers = [ + ('gurobi_persistent', GurobiPersistent), + ('gurobi_direct', GurobiDirect), + ('gurobi_direct_quadratic', GurobiDirectQuadratic), + ('ipopt', Ipopt), + ('highs', Highs), +] def _load_tests(solver_list): @@ -114,7 +121,7 @@ def test_all_solvers_list(): class TestDualSignConvention(unittest.TestCase): - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_equality(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): opt: SolverBase = opt_class() if not opt.available(): @@ -166,7 +173,7 @@ def test_equality(self, name: str, opt_class: Type[SolverBase], use_presolve: bo self.assertAlmostEqual(duals[m.c1], 0) self.assertAlmostEqual(duals[m.c2], -1) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_inequality( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -228,7 +235,7 @@ def test_inequality( self.assertAlmostEqual(duals[m.c1], 0.5) self.assertAlmostEqual(duals[m.c2], 0.5) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_bounds(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): opt: SolverBase = opt_class() if not opt.available(): @@ -283,7 +290,7 @@ def test_bounds(self, name: str, opt_class: Type[SolverBase], use_presolve: bool rc = res.solution_loader.get_reduced_costs() self.assertAlmostEqual(rc[m.x], -1) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_range(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): opt: SolverBase = opt_class() if not opt.available(): @@ -335,7 +342,7 @@ def test_range(self, name: str, opt_class: Type[SolverBase], use_presolve: bool) self.assertAlmostEqual(duals[m.c1], -0.5) self.assertAlmostEqual(duals[m.c2], -0.5) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_equality_max( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -389,7 +396,7 @@ def test_equality_max( self.assertAlmostEqual(duals[m.c1], 0) self.assertAlmostEqual(duals[m.c2], 1) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_inequality_max( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -451,7 +458,7 @@ def test_inequality_max( self.assertAlmostEqual(duals[m.c1], -0.5) self.assertAlmostEqual(duals[m.c2], -0.5) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_bounds_max( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -508,7 +515,7 @@ def test_bounds_max( rc = res.solution_loader.get_reduced_costs() self.assertAlmostEqual(rc[m.x], 1) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_range_max( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -664,16 +671,18 @@ def test_range_constraint( res = opt.solve(m) self.assertEqual(res.solution_status, SolutionStatus.optimal) self.assertAlmostEqual(m.x.value, -1) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c], 1) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c], 1) m.obj.sense = pyo.maximize res = opt.solve(m) self.assertEqual(res.solution_status, SolutionStatus.optimal) self.assertAlmostEqual(m.x.value, 1) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c], 1) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c], 1) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_reduced_costs( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -702,7 +711,7 @@ def test_reduced_costs( self.assertAlmostEqual(rc[m.x], -3) self.assertAlmostEqual(rc[m.y], -4) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_reduced_costs2( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): @@ -768,9 +777,10 @@ def test_param_changes( else: bound = res.objective_bound self.assertTrue(bound <= m.y.value) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c1], (1 + a1 / (a2 - a1))) - self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c1], (1 + a1 / (a2 - a1))) + self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) @parameterized.expand(input=_load_tests(all_solvers)) def test_immutable_param( @@ -815,9 +825,10 @@ def test_immutable_param( else: bound = res.objective_bound self.assertTrue(bound <= m.y.value) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c1], (1 + a1 / (a2 - a1))) - self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c1], (1 + a1 / (a2 - a1))) + self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) @parameterized.expand(input=_load_tests(all_solvers)) def test_equality(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): @@ -831,6 +842,8 @@ def test_equality(self, name: str, opt_class: Type[SolverBase], use_presolve: bo check_duals = False else: opt.config.writer_config.linear_presolve = False + if (name, opt_class) not in dual_solvers: + check_duals = False m = pyo.ConcreteModel() m.x = pyo.Var() m.y = pyo.Var() @@ -922,6 +935,8 @@ def test_no_objective( opt.config.writer_config.linear_presolve = True else: opt.config.writer_config.linear_presolve = False + if (name, opt_class) not in dual_solvers: + check_duals = False m = pyo.ConcreteModel() m.x = pyo.Var() m.y = pyo.Var() @@ -983,9 +998,10 @@ def test_add_remove_cons( else: bound = res.objective_bound self.assertTrue(bound <= m.y.value) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a2 - a1))) - self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a2 - a1))) + self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) m.c3 = pyo.Constraint(expr=m.y >= a3 * m.x + b3) res = opt.solve(m) @@ -994,10 +1010,11 @@ def test_add_remove_cons( self.assertAlmostEqual(m.y.value, a1 * (b3 - b1) / (a1 - a3) + b1) self.assertAlmostEqual(res.incumbent_objective, m.y.value) self.assertTrue(res.objective_bound is None or res.objective_bound <= m.y.value) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a3 - a1))) - self.assertAlmostEqual(duals[m.c2], 0) - self.assertAlmostEqual(duals[m.c3], a1 / (a3 - a1)) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a3 - a1))) + self.assertAlmostEqual(duals[m.c2], 0) + self.assertAlmostEqual(duals[m.c3], a1 / (a3 - a1)) del m.c3 res = opt.solve(m) @@ -1006,9 +1023,10 @@ def test_add_remove_cons( self.assertAlmostEqual(m.y.value, a1 * (b2 - b1) / (a1 - a2) + b1) self.assertAlmostEqual(res.incumbent_objective, m.y.value) self.assertTrue(res.objective_bound is None or res.objective_bound <= m.y.value) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a2 - a1))) - self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.c1], -(1 + a1 / (a2 - a1))) + self.assertAlmostEqual(duals[m.c2], a1 / (a2 - a1)) @parameterized.expand(input=_load_tests(all_solvers)) def test_results_infeasible( @@ -1057,14 +1075,15 @@ def test_results_infeasible( NoSolutionError, '.*does not currently have a valid solution.*' ): res.solution_loader.load_vars() - with self.assertRaisesRegex( - NoDualsError, '.*does not currently have valid duals.*' - ): - res.solution_loader.get_duals() - with self.assertRaisesRegex( - NoReducedCostsError, '.*does not currently have valid reduced costs.*' - ): - res.solution_loader.get_reduced_costs() + if (name, opt_class) in dual_solvers: + with self.assertRaisesRegex( + NoDualsError, '.*does not currently have valid duals.*' + ): + res.solution_loader.get_duals() + with self.assertRaisesRegex( + NoReducedCostsError, '.*does not currently have valid reduced costs.*' + ): + res.solution_loader.get_reduced_costs() @parameterized.expand(input=_load_tests(all_solvers)) def test_trivial_constraints( @@ -1118,7 +1137,7 @@ def test_trivial_constraints( self.assertIn(res.termination_condition, acceptable_termination_conditions) self.assertIsNone(res.incumbent_objective) - @parameterized.expand(input=_load_tests(all_solvers)) + @parameterized.expand(input=_load_tests(dual_solvers)) def test_duals(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): opt: SolverBase = opt_class() if not opt.available(): @@ -1167,13 +1186,13 @@ def test_mutable_quadratic_coefficient( m.c = pyo.Constraint(expr=m.y >= (m.a * m.x + m.b) ** 2) res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0.41024548525899274, 4) - self.assertAlmostEqual(m.y.value, 0.34781038127030117, 4) + self.assertAlmostEqual(m.x.value, 0.41024548525899274, 3) + self.assertAlmostEqual(m.y.value, 0.34781038127030117, 3) m.a.value = 2 m.b.value = -0.5 res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0.10256137418973625, 4) - self.assertAlmostEqual(m.y.value, 0.0869525991355825, 4) + self.assertAlmostEqual(m.x.value, 0.10256137418973625, 3) + self.assertAlmostEqual(m.y.value, 0.0869525991355825, 3) @parameterized.expand(input=_load_tests(qcp_solvers)) def test_mutable_quadratic_objective_qcp( @@ -1198,14 +1217,14 @@ def test_mutable_quadratic_objective_qcp( m.ccon = pyo.Constraint(expr=m.y >= (m.a * m.x + m.b) ** 2) res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0.2719178742733325, 4) - self.assertAlmostEqual(m.y.value, 0.5301035741688002, 4) + self.assertAlmostEqual(m.x.value, 0.2719178742733325, 3) + self.assertAlmostEqual(m.y.value, 0.5301035741688002, 3) m.c.value = 3.5 m.d.value = -1 res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0.6962249634573562, 4) - self.assertAlmostEqual(m.y.value, 0.09227926676152151, 4) + self.assertAlmostEqual(m.x.value, 0.6962249634573562, 3) + self.assertAlmostEqual(m.y.value, 0.09227926676152151, 3) @parameterized.expand(input=_load_tests(qp_solvers)) def test_mutable_quadratic_objective_qp( @@ -1412,7 +1431,7 @@ def test_fixed_vars_4( else: opt.config.writer_config.linear_presolve = False m = pyo.ConcreteModel() - m.x = pyo.Var() + m.x = pyo.Var(bounds=(0, None)) m.y = pyo.Var() m.obj = pyo.Objective(expr=m.x**2 + m.y**2) m.c1 = pyo.Constraint(expr=m.x == 2 / m.y) @@ -1421,8 +1440,8 @@ def test_fixed_vars_4( self.assertAlmostEqual(m.x.value, 2) m.y.unfix() res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 2**0.5) - self.assertAlmostEqual(m.y.value, 2**0.5) + self.assertAlmostEqual(m.x.value, 2**0.5, 3) + self.assertAlmostEqual(m.y.value, 2**0.5, 3) @parameterized.expand(input=_load_tests(all_solvers)) def test_mutable_param_with_range( @@ -1506,9 +1525,10 @@ def test_mutable_param_with_range( res.objective_bound is None or res.objective_bound <= m.y.value + 1e-12 ) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.con1], (1 + a1 / (a2 - a1)), 6) - self.assertAlmostEqual(duals[m.con2], -a1 / (a2 - a1), 6) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.con1], (1 + a1 / (a2 - a1)), 6) + self.assertAlmostEqual(duals[m.con2], -a1 / (a2 - a1), 6) else: self.assertAlmostEqual(m.x.value, (c2 - c1) / (a1 - a2), 6) self.assertAlmostEqual(m.y.value, a1 * (c2 - c1) / (a1 - a2) + c1, 6) @@ -1517,9 +1537,10 @@ def test_mutable_param_with_range( res.objective_bound is None or res.objective_bound >= m.y.value - 1e-12 ) - duals = res.solution_loader.get_duals() - self.assertAlmostEqual(duals[m.con1], (1 + a1 / (a2 - a1)), 6) - self.assertAlmostEqual(duals[m.con2], -a1 / (a2 - a1), 6) + if (name, opt_class) in dual_solvers: + duals = res.solution_loader.get_duals() + self.assertAlmostEqual(duals[m.con1], (1 + a1 / (a2 - a1)), 6) + self.assertAlmostEqual(duals[m.con2], -a1 / (a2 - a1), 6) @parameterized.expand(input=_load_tests(all_solvers)) def test_add_and_remove_vars( @@ -1590,8 +1611,8 @@ def test_exp(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): m.obj = pyo.Objective(expr=m.x**2 + m.y**2) m.c1 = pyo.Constraint(expr=m.y >= pyo.exp(m.x)) res = opt.solve(m) - self.assertAlmostEqual(m.x.value, -0.42630274815985264) - self.assertAlmostEqual(m.y.value, 0.6529186341994245) + self.assertAlmostEqual(m.x.value, -0.42630274815985264, 4) + self.assertAlmostEqual(m.y.value, 0.6529186341994245, 4) @parameterized.expand(input=_load_tests(nlp_solvers)) def test_log(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): @@ -1609,8 +1630,8 @@ def test_log(self, name: str, opt_class: Type[SolverBase], use_presolve: bool): m.obj = pyo.Objective(expr=m.x**2 + m.y**2) m.c1 = pyo.Constraint(expr=m.y <= pyo.log(m.x)) res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0.6529186341994245) - self.assertAlmostEqual(m.y.value, -0.42630274815985264) + self.assertAlmostEqual(m.x.value, 0.6529186341994245, 3) + self.assertAlmostEqual(m.y.value, -0.42630274815985264, 3) @parameterized.expand(input=_load_tests(all_solvers)) def test_with_numpy( @@ -1720,24 +1741,25 @@ def test_solution_loader( self.assertNotIn(m.x, primals) self.assertIn(m.y, primals) self.assertAlmostEqual(primals[m.y], 1) - reduced_costs = res.solution_loader.get_reduced_costs() - self.assertIn(m.x, reduced_costs) - self.assertIn(m.y, reduced_costs) - self.assertAlmostEqual(reduced_costs[m.x], 1) - self.assertAlmostEqual(reduced_costs[m.y], 0) - reduced_costs = res.solution_loader.get_reduced_costs([m.y]) - self.assertNotIn(m.x, reduced_costs) - self.assertIn(m.y, reduced_costs) - self.assertAlmostEqual(reduced_costs[m.y], 0) - duals = res.solution_loader.get_duals() - self.assertIn(m.c1, duals) - self.assertIn(m.c2, duals) - self.assertAlmostEqual(duals[m.c1], 1) - self.assertAlmostEqual(duals[m.c2], 0) - duals = res.solution_loader.get_duals([m.c1]) - self.assertNotIn(m.c2, duals) - self.assertIn(m.c1, duals) - self.assertAlmostEqual(duals[m.c1], 1) + if (name, opt_class) in dual_solvers: + reduced_costs = res.solution_loader.get_reduced_costs() + self.assertIn(m.x, reduced_costs) + self.assertIn(m.y, reduced_costs) + self.assertAlmostEqual(reduced_costs[m.x], 1) + self.assertAlmostEqual(reduced_costs[m.y], 0) + reduced_costs = res.solution_loader.get_reduced_costs([m.y]) + self.assertNotIn(m.x, reduced_costs) + self.assertIn(m.y, reduced_costs) + self.assertAlmostEqual(reduced_costs[m.y], 0) + duals = res.solution_loader.get_duals() + self.assertIn(m.c1, duals) + self.assertIn(m.c2, duals) + self.assertAlmostEqual(duals[m.c1], 1) + self.assertAlmostEqual(duals[m.c2], 0) + duals = res.solution_loader.get_duals([m.c1]) + self.assertNotIn(m.c2, duals) + self.assertIn(m.c1, duals) + self.assertAlmostEqual(duals[m.c1], 1) @parameterized.expand(input=_load_tests(all_solvers)) def test_time_limit( @@ -2219,6 +2241,8 @@ def test_scaling(self, name: str, opt_class: Type[SolverBase], use_presolve: boo opt.config.writer_config.linear_presolve = True else: opt.config.writer_config.linear_presolve = False + if (name, opt_class) not in dual_solvers: + check_duals = False m = pyo.ConcreteModel() m.x = pyo.Var() From 37a31a7fc7b7fe8cb14179e4dc18df4dbdd8bc87 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 15 Aug 2025 01:50:18 -0600 Subject: [PATCH 064/104] scip direct --- pyomo/contrib/solver/plugins.py | 2 +- .../contrib/solver/solvers/scip/scip_direct.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index a4d6f5f9004..81e3677b19e 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -44,4 +44,4 @@ def load(): name='scip_direct', legacy_name='scip_direct_v2', doc='Direct interface pyscipopt', - ) + )(SCIPDirect) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 1ff470223bd..3926dd25a1c 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -16,6 +16,7 @@ from typing import Tuple, List, Optional, Sequence, Mapping, Dict from pyomo.common.collections import ComponentMap +from pyomo.core.expr.numvalue import is_constant from pyomo.common.numeric_types import native_numeric_types from pyomo.common.errors import InfeasibleConstraintException, ApplicationError from pyomo.common.timing import HierarchicalTimer @@ -67,6 +68,8 @@ ) from pyomo.common.config import ConfigValue from pyomo.common.tee import capture_output, TeeStream +from pyomo.core.base.units_container import _PyomoUnit +from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr logger = logging.getLogger(__name__) @@ -132,7 +135,15 @@ def _handle_negation(node, data, opt): def _handle_pow(node, data, opt): - return data[0] ** data[1] + x, y = data # x ** y = exp(log(x**y)) = exp(y*log(x)) + if is_constant(node.args[1]): + return x**y + else: + xlb, xub = compute_bounds_on_expr(node.args[0]) + if xlb > 0: + return scip.exp(y*scip.log(x)) + else: + return x**y # scip will probably raise an error here def _handle_product(node, data, opt): @@ -210,6 +221,10 @@ def _handle_named_expression(node, data, opt): return data[0] +def _handle_unit(node, data, opt): + return node.value + + _operator_map = { NegationExpression: _handle_negation, PowExpression: _handle_pow, @@ -237,6 +252,7 @@ def _handle_named_expression(node, data, opt): float: _handle_float, int: _handle_float, AutoLinkedBinaryVar: _handle_var, + _PyomoUnit: _handle_unit, } From a43a38bacc66aafa6af6c7305448e3f55a3fe263 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 16 Aug 2025 09:30:56 -0600 Subject: [PATCH 065/104] forgot to inherit from PersistentSolverBase --- .../solvers/gurobi/gurobi_persistent.py | 3 +- .../solver/tests/solvers/test_solvers.py | 139 ++++++++---------- 2 files changed, 64 insertions(+), 78 deletions(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 847ec958bdd..8d16bb9082e 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -28,6 +28,7 @@ from pyomo.contrib.solver.common.results import Results from pyomo.contrib.solver.common.util import IncompatibleModelError from pyomo.contrib.solver.common.solution_loader import SolutionLoaderBase +from pyomo.contrib.solver.common.base import PersistentSolverBase from pyomo.core.staleflag import StaleFlagManager from .gurobi_direct_base import ( GurobiDirectBase, @@ -575,7 +576,7 @@ def update_parameters(self, params: List[ParamData]): self.opt._update_parameters(params) -class GurobiPersistent(GurobiDirectQuadratic): +class GurobiPersistent(GurobiDirectQuadratic, PersistentSolverBase): _minimum_version = (7, 0, 0) def __init__(self, **kwds): diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 96e2e7b2c38..6965147d167 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -1221,55 +1221,50 @@ def test_mutable_quadratic_objective_qp( def test_fixed_vars( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): - for treat_fixed_vars_as_params in [True, False]: - opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=treat_fixed_vars_as_params) - if not opt.available(): - raise unittest.SkipTest(f'Solver {opt.name} not available.') - if any(name.startswith(i) for i in nl_solvers_set): - if use_presolve: - opt.config.writer_config.linear_presolve = True - else: - opt.config.writer_config.linear_presolve = False - m = pyo.ConcreteModel() - m.x = pyo.Var() - m.x.fix(0) - m.y = pyo.Var() - a1 = 1 - a2 = -1 - b1 = 1 - b2 = 2 - m.obj = pyo.Objective(expr=m.y) - m.c1 = pyo.Constraint(expr=m.y >= a1 * m.x + b1) - m.c2 = pyo.Constraint(expr=m.y >= a2 * m.x + b2) - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 2) - m.x.unfix() - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, (b2 - b1) / (a1 - a2)) - self.assertAlmostEqual(m.y.value, a1 * (b2 - b1) / (a1 - a2) + b1) - m.x.fix(0) - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 2) - m.x.value = 2 - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 2) - self.assertAlmostEqual(m.y.value, 3) - m.x.value = 0 - res = opt.solve(m) - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 2) + opt: SolverBase = opt_class() + if not opt.available(): + raise unittest.SkipTest(f'Solver {opt.name} not available.') + if any(name.startswith(i) for i in nl_solvers_set): + if use_presolve: + opt.config.writer_config.linear_presolve = True + else: + opt.config.writer_config.linear_presolve = False + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.x.fix(0) + m.y = pyo.Var() + a1 = 1 + a2 = -1 + b1 = 1 + b2 = 2 + m.obj = pyo.Objective(expr=m.y) + m.c1 = pyo.Constraint(expr=m.y >= a1 * m.x + b1) + m.c2 = pyo.Constraint(expr=m.y >= a2 * m.x + b2) + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 2) + m.x.unfix() + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, (b2 - b1) / (a1 - a2)) + self.assertAlmostEqual(m.y.value, a1 * (b2 - b1) / (a1 - a2) + b1) + m.x.fix(0) + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 2) + m.x.value = 2 + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 2) + self.assertAlmostEqual(m.y.value, 3) + m.x.value = 0 + res = opt.solve(m) + self.assertAlmostEqual(m.x.value, 0) + self.assertAlmostEqual(m.y.value, 2) @parameterized.expand(input=_load_tests(all_solvers)) def test_fixed_vars_2( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=True) if not opt.available(): raise unittest.SkipTest(f'Solver {opt.name} not available.') if any(name.startswith(i) for i in nl_solvers_set): @@ -1313,8 +1308,6 @@ def test_fixed_vars_3( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=True) if not opt.available(): raise unittest.SkipTest(f'Solver {opt.name} not available.') if any(name.startswith(i) for i in nl_solvers_set): @@ -1337,8 +1330,6 @@ def test_fixed_vars_4( self, name: str, opt_class: Type[SolverBase], use_presolve: bool ): opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=True) if not opt.available(): raise unittest.SkipTest(f'Solver {opt.name} not available.') if any(name.startswith(i) for i in nl_solvers_set): @@ -1892,10 +1883,7 @@ def test_fixed_binaries( res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 1) - if opt.is_persistent(): - opt: SolverBase = opt_class(treat_fixed_vars_as_params=False) - else: - opt = opt_class() + opt = opt_class() m.x.fix(0) res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 0) @@ -2049,33 +2037,30 @@ def test_bug_2(self, name: str, opt_class: Type[SolverBase], use_presolve: bool) This test is for a bug where an objective containing a fixed variable does not get updated properly when the variable is unfixed. """ - for fixed_var_option in [True, False]: - opt: SolverBase = opt_class() - if opt.is_persistent(): - opt = opt_class(treat_fixed_vars_as_params=fixed_var_option) - if not opt.available(): - raise unittest.SkipTest(f'Solver {opt.name} not available.') - if any(name.startswith(i) for i in nl_solvers_set): - if use_presolve: - opt.config.writer_config.linear_presolve = True - else: - opt.config.writer_config.linear_presolve = False - - m = pyo.ConcreteModel() - m.x = pyo.Var(bounds=(-10, 10)) - m.y = pyo.Var() - m.obj = pyo.Objective(expr=3 * m.y - m.x) - m.c = pyo.Constraint(expr=m.y >= m.x) - - m.x.fix(1) - res = opt.solve(m) - self.assertAlmostEqual(res.incumbent_objective, 2, 5) + opt: SolverBase = opt_class() + if not opt.available(): + raise unittest.SkipTest(f'Solver {opt.name} not available.') + if any(name.startswith(i) for i in nl_solvers_set): + if use_presolve: + opt.config.writer_config.linear_presolve = True + else: + opt.config.writer_config.linear_presolve = False - m.x.unfix() - m.x.setlb(-9) - m.x.setub(9) - res = opt.solve(m) - self.assertAlmostEqual(res.incumbent_objective, -18, 5) + m = pyo.ConcreteModel() + m.x = pyo.Var(bounds=(-10, 10)) + m.y = pyo.Var() + m.obj = pyo.Objective(expr=3 * m.y - m.x) + m.c = pyo.Constraint(expr=m.y >= m.x) + + m.x.fix(1) + res = opt.solve(m) + self.assertAlmostEqual(res.incumbent_objective, 2, 5) + + m.x.unfix() + m.x.setlb(-9) + m.x.setub(9) + res = opt.solve(m) + self.assertAlmostEqual(res.incumbent_objective, -18, 5) @parameterized.expand(input=_load_tests(nl_solvers)) def test_presolve_with_zero_coef( From e76baae7b130aa3d845e053c802a93ca5e76de26 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 16 Aug 2025 12:25:39 -0600 Subject: [PATCH 066/104] bug --- pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 8d16bb9082e..8145777ffb9 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -595,6 +595,7 @@ def __init__(self, **kwds): self._observer = _GurobiObserver(self) self._change_detector = ModelChangeDetector(observers=[self._observer]) self._constraint_ndx = 0 + self._should_update_parameters = False @property def auto_updates(self): @@ -683,6 +684,8 @@ def update(self): if self._needs_updated: self._update_gurobi_model() self._change_detector.update(timer=timer) + if self._should_update_parameters: + self._update_parameters([]) timer.stop('update') def _add_constraints(self, cons: List[ConstraintData]): @@ -915,6 +918,8 @@ def _update_variables(self, variables: List[VarData]): gurobipy_var.setAttr('lb', lb) gurobipy_var.setAttr('ub', ub) gurobipy_var.setAttr('vtype', vtype) + if var.fixed: + self._should_update_parameters = True self._needs_updated = True def _update_parameters(self, params: List[ParamData]): @@ -950,6 +955,8 @@ def _update_parameters(self, params: List[ParamData]): # parts have mutable coefficients self._solver_model.setObjective(new_gurobi_expr, sense=sense) + self._should_update_parameters = False + def _invalidate_last_results(self): if self._last_results_object is not None: self._last_results_object.solution_loader.invalidate() From 0b84dcc77e826022c705ae95d187dc29e0b24e1f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 18 Aug 2025 04:57:27 -0600 Subject: [PATCH 067/104] more expression types for scip --- .../solver/solvers/scip/scip_direct.py | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 3926dd25a1c..d72ce47ef02 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -42,6 +42,7 @@ NPV_SumExpression, NPV_UnaryFunctionExpression, ) +from pyomo.core.expr.numvalue import NumericConstant from pyomo.gdp.disjunct import AutoLinkedBinaryVar from pyomo.core.base.expression import ExpressionData, ScalarExpression from pyomo.core.expr.relational_expr import EqualityExpression, InequalityExpression, RangedExpression @@ -126,6 +127,10 @@ def _handle_param(node, data, opt): return scip_param +def _handle_constant(node, data, opt): + return node.value + + def _handle_float(node, data, opt): return float(node) @@ -167,6 +172,10 @@ def _handle_log(node, data, opt): return scip.log(data[0]) +def _handle_log10(node, data, opt): + return scip.log(data[0]) / math.log(10) + + def _handle_sin(node, data, opt): return scip.sin(data[0]) @@ -187,6 +196,12 @@ def _handle_tan(node, data, opt): return scip.sin(data[0]) / scip.cos(data[0]) +def _handle_tanh(node, data, opt): + x = data[0] + _exp = scip.exp + return (_exp(x) - _exp(-x)) / (_exp(x) + _exp(-x)) + + _unary_map = { 'exp': _handle_exp, 'log': _handle_log, @@ -195,6 +210,8 @@ def _handle_tan(node, data, opt): 'sqrt': _handle_sqrt, 'abs': _handle_abs, 'tan': _handle_tan, + 'log10': _handle_log10, + 'tanh': _handle_tanh, } @@ -253,6 +270,7 @@ def _handle_unit(node, data, opt): int: _handle_float, AutoLinkedBinaryVar: _handle_var, _PyomoUnit: _handle_unit, + NumericConstant: _handle_constant, } @@ -427,7 +445,7 @@ def solve(self, model: BlockData, **kwds) -> Results: scip_model.setParam(key, option) timer.start('optimize') - with capture_output(TeeStream(*ostreams), capture_fd=True): + with capture_output(TeeStream(*ostreams), capture_fd=False): scip_model.optimize() timer.stop('optimize') @@ -690,7 +708,7 @@ def _postsolve( if has_obj: try: - if scip_model.getObjVal() < scip_model.infinity(): + if scip_model.getNSols() > 0 and scip_model.getObjVal() < scip_model.infinity(): results.incumbent_objective = scip_model.getObjVal() else: results.incumbent_objective = None From c2a01777d43d47d7ea072c5b93ddc020e8b787ee Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 18 Aug 2025 09:13:12 -0600 Subject: [PATCH 068/104] bug --- pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 8145777ffb9..603e8e21800 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -432,7 +432,7 @@ def _add_variables(self, variables: List[VarData]): def _get_expr_from_pyomo_repn(self, repn): if repn.nonlinear_expr is not None: raise IncompatibleModelError( - f'GurobiDirectQuadratic only supports linear and quadratic expressions: {expr}.' + f'GurobiDirectQuadratic only supports linear and quadratic expressions: {repn}.' ) if len(repn.linear_vars) > 0: From 3180462477261e7c1eaa63e349b35abf080c6f05 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 20 Aug 2025 09:08:52 -0600 Subject: [PATCH 069/104] capture_fd for scip --- pyomo/contrib/solver/solvers/scip/scip_direct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index d72ce47ef02..8deca600b40 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -445,7 +445,7 @@ def solve(self, model: BlockData, **kwds) -> Results: scip_model.setParam(key, option) timer.start('optimize') - with capture_output(TeeStream(*ostreams), capture_fd=False): + with capture_output(TeeStream(*ostreams), capture_fd=True): scip_model.optimize() timer.stop('optimize') From 72912e0d9e5397a3403061aff3daf32a4e9b281c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Fri, 22 Aug 2025 08:49:40 -0600 Subject: [PATCH 070/104] working on persistent interface to scip --- .../solver/solvers/scip/scip_direct.py | 161 +++++++++++++++++- 1 file changed, 153 insertions(+), 8 deletions(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 8deca600b40..344b1741552 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -9,6 +9,7 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +from __future__ import annotations import datetime import io import logging @@ -49,7 +50,7 @@ from pyomo.core.staleflag import StaleFlagManager from pyomo.core.expr.visitor import StreamBasedExpressionVisitor from pyomo.common.dependencies import attempt_import -from pyomo.contrib.solver.common.base import SolverBase, Availability +from pyomo.contrib.solver.common.base import SolverBase, Availability, PersistentSolverBase from pyomo.contrib.solver.common.config import BranchAndBoundConfig from pyomo.contrib.solver.common.util import ( NoFeasibleSolutionError, @@ -71,6 +72,7 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.core.base.units_container import _PyomoUnit from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr +from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector logger = logging.getLogger(__name__) @@ -354,7 +356,72 @@ def load_import_suffixes(self, solution_id=None): load_import_suffixes(self._pyomo_model, self, solution_id=solution_id) -class SCIPDirect(SolverBase): +class ScipPersistentSolutionLoader(ScipDirectSolutionLoader): + def __init__( + self, + solver_model, + var_id_map, + var_map, + con_map, + pyomo_model, + opt, + ) -> None: + super().__init__( + solver_model, + var_id_map, + var_map, + con_map, + pyomo_model, + opt, + ) + self._valid = False + + def invalidate(self): + self._valid = False + + def _assert_solution_still_valid(self): + if not self._valid: + raise RuntimeError('The results in the solver are no longer valid.') + + def load_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> None: + self._assert_solution_still_valid() + return super().load_vars(vars_to_load, solution_id) + + def get_vars( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> Mapping[VarData, float]: + self._assert_solution_still_valid() + return super().get_vars(vars_to_load, solution_id) + + def get_duals( + self, cons_to_load: Sequence[ConstraintData] | None = None, solution_id=None + ) -> Dict[ConstraintData, float]: + self._assert_solution_still_valid() + return super().get_duals(cons_to_load) + + def get_reduced_costs( + self, vars_to_load: Sequence[VarData] | None = None, solution_id=None + ) -> Mapping[VarData, float]: + self._assert_solution_still_valid() + return super().get_reduced_costs(vars_to_load) + + def get_number_of_solutions(self) -> int: + self._assert_solution_still_valid() + return super().get_number_of_solutions() + + def get_solution_ids(self) -> List: + self._assert_solution_still_valid() + return super().get_solution_ids() + + def load_import_suffixes(self, solution_id=None): + self._assert_solution_still_valid() + super().load_import_suffixes(solution_id) + + + +class ScipDirect(SolverBase): _available = None _tc_map = None @@ -393,11 +460,11 @@ def available(self) -> Availability: return self._available if not scip_available: - SCIPDirect._available = Availability.NotFound + ScipDirect._available = Availability.NotFound elif self.version() < self._minimum_version: - SCIPDirect._available = Availability.BadVersion + ScipDirect._available = Availability.BadVersion else: - SCIPDirect._available = Availability.FullLicense + ScipDirect._available = Availability.FullLicense return self._available @@ -465,9 +532,9 @@ def solve(self, model: BlockData, **kwds) -> Results: return results def _get_tc_map(self): - if SCIPDirect._tc_map is None: + if ScipDirect._tc_map is None: tc = TerminationCondition - SCIPDirect._tc_map = { + ScipDirect._tc_map = { "unknown": tc.unknown, "userinterrupt": tc.interrupted, "nodelimit": tc.iterationLimit, @@ -487,7 +554,7 @@ def _get_tc_map(self): "inforunbd": tc.infeasibleOrUnbounded, "terminate": tc.unknown, } - return SCIPDirect._tc_map + return ScipDirect._tc_map def _get_infeasible_results(self): res = Results() @@ -753,3 +820,81 @@ def _mipstart(self): if pyomo_var.is_integer(): sol[scip_var] = pyomo_var.value self._solver_model.addSol(sol) + + +class _SCIPObserver(Observer): + def __init__(self, opt: ScipPersistent) -> None: + self.opt = opt + + def add_variables(self, variables: List[VarData]): + self.opt._add_variables(variables) + + def add_parameters(self, params: List[ParamData]): + pass + + def add_constraints(self, cons: List[ConstraintData]): + self.opt._add_constraints(cons) + + def add_sos_constraints(self, cons: List[SOSConstraintData]): + self.opt._add_sos_constraints(cons) + + def set_objective(self, obj: ObjectiveData | None): + self.opt._set_objective(obj) + + def remove_constraints(self, cons: List[ConstraintData]): + self.opt._remove_constraints(cons) + + def remove_sos_constraints(self, cons: List[SOSConstraintData]): + self.opt._remove_sos_constraints(cons) + + def remove_variables(self, variables: List[VarData]): + self.opt._remove_variables(variables) + + def remove_parameters(self, params: List[ParamData]): + pass + + def update_variables(self, variables: List[VarData]): + self.opt._update_variables(variables) + + def update_parameters(self, params: List[ParamData]): + self.opt._update_parameters(params) + + +class ScipPersistent(ScipDirect, PersistentSolverBase): + _minimum_version = (5, 5, 0) # this is probably conservative + + CONFIG = ScipConfig() + + def __init__(self, **kwds): + super().__init__(**kwds) + self._pyomo_model = None + self._objective = None + self._observer = _SCIPObserver(self) + self._change_detector = ModelChangeDetector(observers=[self._observer]) + + @property + def auto_updates(self): + return self._change_detector.config + + def _clear(self): + super()._clear() + self._pyomo_model = None + self._objective = None + + def _create_solver_model(self, model): + if model is self._pyomo_model: + self.update() + else: + self.set_instance(model=model) + + solution_loader = ScipPersistentSolutionLoader( + solver_model=self._solver_model, + var_id_map=self._vars, + var_map=self._pyomo_var_to_solver_var_map, + con_map=self._pyomo_con_to_solver_con_map, + pyomo_model=model, + opt=self, + ) + + has_obj = self._objective is not None: + return self._solver_model, solution_loader, has_obj \ No newline at end of file From a424cfb0ae4f81f9b90b8e7e229efccb7eeb93f2 Mon Sep 17 00:00:00 2001 From: Miranda Mundt Date: Mon, 25 Aug 2025 08:14:07 -0600 Subject: [PATCH 071/104] Minor changes - removing unused imports --- pyomo/contrib/observer/component_collector.py | 3 +-- pyomo/contrib/observer/model_observer.py | 9 +++------ pyomo/contrib/observer/tests/test_change_detector.py | 7 ++++--- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index d52ec46086c..e09fe666bbf 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -63,8 +63,7 @@ def exitNode(self, node, data): nt = type(node) if nt in collector_handlers: return collector_handlers[nt](node, self) - else: - return None + return None _visitor = _ComponentFromExprCollector() diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index bd89d4f600d..1452a3d673d 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -10,21 +10,18 @@ # __________________________________________________________________________ import abc -import datetime from typing import List, Sequence, Optional from pyomo.common.config import ConfigDict, ConfigValue from pyomo.core.base.constraint import ConstraintData, Constraint from pyomo.core.base.sos import SOSConstraintData, SOSConstraint from pyomo.core.base.var import VarData -from pyomo.core.base.param import ParamData, Param +from pyomo.core.base.param import ParamData from pyomo.core.base.objective import ObjectiveData -from pyomo.core.staleflag import StaleFlagManager from pyomo.common.collections import ComponentMap from pyomo.common.timing import HierarchicalTimer -from pyomo.contrib.solver.common.results import Results from pyomo.contrib.solver.common.util import get_objective -from .component_collector import collect_components_from_expr +from pyomo.contrib.observer.component_collector import collect_components_from_expr from pyomo.common.numeric_types import native_numeric_types @@ -624,7 +621,7 @@ def _check_for_var_changes(self): def _check_for_param_changes(self): params_to_update = [] - for pid, (p, val) in self._params.items(): + for _, (p, val) in self._params.items(): if p.value != val: params_to_update.append(p) return params_to_update diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index f3ce6e28b5c..802d4482acc 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -9,21 +9,22 @@ # This software is distributed under the 3-clause BSD License. # ___________________________________________________________________________ +import logging +from typing import List + +import pyomo.environ as pyo from pyomo.core.base.constraint import ConstraintData from pyomo.core.base.objective import ObjectiveData from pyomo.core.base.param import ParamData from pyomo.core.base.sos import SOSConstraintData from pyomo.core.base.var import VarData -import pyomo.environ as pyo from pyomo.common import unittest -from typing import List from pyomo.contrib.observer.model_observer import ( Observer, ModelChangeDetector, AutoUpdateConfig, ) from pyomo.common.collections import ComponentMap -import logging logger = logging.getLogger(__name__) From cfa1e9108f6edd05d238056e6f071122a3164d39 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 1 Sep 2025 17:10:19 -0600 Subject: [PATCH 072/104] minor fixes --- pyomo/contrib/solver/plugins.py | 4 ++-- pyomo/contrib/solver/solvers/scip/scip_direct.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 81e3677b19e..895b6387725 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -15,7 +15,7 @@ from .solvers.gurobi.gurobi_direct import GurobiDirect from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent from .solvers.highs import Highs -from .solvers.scip.scip_direct import SCIPDirect +from .solvers.scip.scip_direct import ScipDirect def load(): @@ -44,4 +44,4 @@ def load(): name='scip_direct', legacy_name='scip_direct_v2', doc='Direct interface pyscipopt', - )(SCIPDirect) + )(ScipDirect) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 344b1741552..1032affd597 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -896,5 +896,5 @@ def _create_solver_model(self, model): opt=self, ) - has_obj = self._objective is not None: + has_obj = self._objective is not None return self._solver_model, solution_loader, has_obj \ No newline at end of file From 33f831ac27389da9664cc519208a8288c20fe191 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 11:02:04 -0600 Subject: [PATCH 073/104] observer updates --- pyomo/contrib/observer/component_collector.py | 56 ++- pyomo/contrib/observer/model_observer.py | 410 +++++++++++------- .../observer/tests/test_change_detector.py | 46 +- 3 files changed, 315 insertions(+), 197 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index e09fe666bbf..2fe2170aa43 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -12,11 +12,27 @@ from pyomo.core.expr.visitor import StreamBasedExpressionVisitor from pyomo.core.expr.numeric_expr import ( ExternalFunctionExpression, - NPV_ExternalFunctionExpression, + NegationExpression, + PowExpression, + MaxExpression, + MinExpression, + ProductExpression, + MonomialTermExpression, + DivisionExpression, + SumExpression, + Expr_ifExpression, + UnaryFunctionExpression, + AbsExpression, +) +from pyomo.core.expr.relational_expr import ( + RangedExpression, + InequalityExpression, + EqualityExpression, ) from pyomo.core.base.var import VarData, ScalarVar from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.expression import ExpressionData, ScalarExpression +from pyomo.repn.util import ExitNodeDispatcher def handle_var(node, collector): @@ -39,16 +55,29 @@ def handle_external_function(node, collector): return None -collector_handlers = { - VarData: handle_var, - ScalarVar: handle_var, - ParamData: handle_param, - ScalarParam: handle_param, - ExpressionData: handle_named_expression, - ScalarExpression: handle_named_expression, - ExternalFunctionExpression: handle_external_function, - NPV_ExternalFunctionExpression: handle_external_function, -} +def handle_skip(node, collector): + return None + + +collector_handlers = ExitNodeDispatcher() +collector_handlers[VarData] = handle_var +collector_handlers[ParamData] = handle_param +collector_handlers[ExpressionData] = handle_named_expression +collector_handlers[ExternalFunctionExpression] = handle_external_function +collector_handlers[NegationExpression] = handle_skip +collector_handlers[PowExpression] = handle_skip +collector_handlers[MaxExpression] = handle_skip +collector_handlers[MinExpression] = handle_skip +collector_handlers[ProductExpression] = handle_skip +collector_handlers[MonomialTermExpression] = handle_skip +collector_handlers[DivisionExpression] = handle_skip +collector_handlers[SumExpression] = handle_skip +collector_handlers[Expr_ifExpression] = handle_skip +collector_handlers[UnaryFunctionExpression] = handle_skip +collector_handlers[AbsExpression] = handle_skip +collector_handlers[RangedExpression] = handle_skip +collector_handlers[InequalityExpression] = handle_skip +collector_handlers[EqualityExpression] = handle_skip class _ComponentFromExprCollector(StreamBasedExpressionVisitor): @@ -60,10 +89,7 @@ def __init__(self, **kwds): super().__init__(**kwds) def exitNode(self, node, data): - nt = type(node) - if nt in collector_handlers: - return collector_handlers[nt](node, self) - return None + return collector_handlers[node.__class__](node, self) _visitor = _ComponentFromExprCollector() diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 1452a3d673d..f429a8ad907 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -17,7 +17,9 @@ from pyomo.core.base.sos import SOSConstraintData, SOSConstraint from pyomo.core.base.var import VarData from pyomo.core.base.param import ParamData -from pyomo.core.base.objective import ObjectiveData +from pyomo.core.base.objective import ObjectiveData, Objective +from pyomo.core.base.block import BlockData +from pyomo.core.base.component import ActiveComponent from pyomo.common.collections import ComponentMap from pyomo.common.timing import HierarchicalTimer from pyomo.contrib.solver.common.util import get_objective @@ -75,21 +77,24 @@ def __init__( domain=bool, default=True, description=""" - If False, new/old constraints will not be automatically detected on subsequent - solves. Use False only when manually updating the solver with opt.add_constraints() - and opt.remove_constraints() or when you are certain constraints are not being - added to/removed from the model.""", + If False, new/old constraints will not be automatically detected on + subsequent solves. Use False only when manually updating the solver + with opt.add_constraints() and opt.remove_constraints() or when you + are certain constraints are not being added to/removed from the + model.""", ), ) - self.check_for_new_objective: bool = self.declare( - 'check_for_new_objective', + self.check_for_new_or_removed_objectives: bool = self.declare( + 'check_for_new_or_removed_objectives', ConfigValue( domain=bool, default=True, description=""" - If False, new/old objectives will not be automatically detected on subsequent - solves. Use False only when manually updating the solver with opt.set_objective() or - when you are certain objectives are not being added to / removed from the model.""", + If False, new/old objectives will not be automatically detected on + subsequent solves. Use False only when manually updating the solver + with opt.add_objectives() and opt.remove_objectives() or when you + are certain objectives are not being added to/removed from the + model.""", ), ) self.update_constraints: bool = self.declare( @@ -98,11 +103,12 @@ def __init__( domain=bool, default=True, description=""" - If False, changes to existing constraints will not be automatically detected on - subsequent solves. This includes changes to the lower, body, and upper attributes of - constraints. Use False only when manually updating the solver with - opt.remove_constraints() and opt.add_constraints() or when you are certain constraints - are not being modified.""", + If False, changes to existing constraints will not be automatically + detected on subsequent solves. This includes changes to the lower, + body, and upper attributes of constraints. Use False only when + manually updating the solver with opt.remove_constraints() and + opt.add_constraints() or when you are certain constraints are not + being modified.""", ), ) self.update_vars: bool = self.declare( @@ -111,11 +117,12 @@ def __init__( domain=bool, default=True, description=""" - If False, changes to existing variables will not be automatically detected on - subsequent solves. This includes changes to the lb, ub, domain, and fixed - attributes of variables. Use False only when manually updating the observer with - opt.update_variables() or when you are certain variables are not being modified. - Note that changes to values of fixed variables is handled by + If False, changes to existing variables will not be automatically + detected on subsequent solves. This includes changes to the lb, ub, + domain, and fixed attributes of variables. Use False only when + manually updating the observer with opt.update_variables() or when + you are certain variables are not being modified. Note that changes + to values of fixed variables is handled by update_parameters_and_fixed_vars.""", ), ) @@ -139,21 +146,22 @@ def __init__( default=True, description=""" If False, changes to Expressions will not be automatically detected on - subsequent solves. Use False only when manually updating the solver with - opt.remove_constraints() and opt.add_constraints() or when you are certain - Expressions are not being modified.""", + subsequent solves. Use False only when manually updating the solver + with opt.remove_constraints() and opt.add_constraints() or when you + are certain Expressions are not being modified.""", ), ) - self.update_objective: bool = self.declare( - 'update_objective', + self.update_objectives: bool = self.declare( + 'update_objectives', ConfigValue( domain=bool, default=True, description=""" If False, changes to objectives will not be automatically detected on - subsequent solves. This includes the expr and sense attributes of objectives. Use - False only when manually updating the solver with opt.set_objective() or when you are - certain objectives are not being modified.""", + subsequent solves. This includes the expr and sense attributes of + objectives. Use False only when manually updating the solver with + opt.set_objective() or when you are certain objectives are not being + modified.""", ), ) @@ -179,7 +187,11 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): pass @abc.abstractmethod - def set_objective(self, obj: Optional[ObjectiveData]): + def add_objectives(self, objs: List[ObjectiveData]): + pass + + @abc.abstractmethod + def remove_objectives(self, objs: List[ObjectiveData]): pass @abc.abstractmethod @@ -208,54 +220,61 @@ def update_parameters(self, params: List[ParamData]): class ModelChangeDetector: - def __init__(self, observers: Sequence[Observer], **kwds): + def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): """ Parameters ---------- observers: Sequence[Observer] The objects to notify when changes are made to the model """ + self._known_active_ctypes = {Constraint, SOSConstraint, Objective} self._observers: List[Observer] = list(observers) - self._model = None self._active_constraints = {} # maps constraint to expression self._active_sos = {} self._vars = {} # maps var id to (var, lb, ub, fixed, domain, value) self._params = {} # maps param id to param - self._objective = None - self._objective_expr = None - self._objective_sense = None - self._named_expressions = ( - {} - ) # maps constraint to list of tuples (named_expr, named_expr.expr) + self._objectives = {} # maps objective id to (objective, expression, sense) + + # maps constraints/objectives to list of tuples (named_expr, named_expr.expr) + self._named_expressions = {} + self._obj_named_expressions = {} + self._external_functions = ComponentMap() - self._obj_named_expressions = [] - self._referenced_variables = ( - {} - ) # var_id: [dict[constraints, None], dict[sos constraints, None], None or objective] - self._referenced_params = ( - {} - ) # param_id: [dict[constraints, None], dict[sos constraints, None], None or objective] + + # the dictionaries below are really just ordered sets, but we need to + # stick with built-in types for performance + + # var_id: ( + # dict[constraints, None], + # dict[sos constraints, None], + # dict[objectives, None], + # ) + self._referenced_variables = {} + + # param_id: ( + # dict[constraints, None], + # dict[sos constraints, None], + # dict[objectives, None], + # ) + self._referenced_params = {} + self._vars_referenced_by_con = {} - self._vars_referenced_by_obj = [] + self._vars_referenced_by_obj = {} self._params_referenced_by_con = {} - self._params_referenced_by_obj = [] - self._expr_types = None + self._params_referenced_by_obj = {} + self.config: AutoUpdateConfig = AutoUpdateConfig()( value=kwds, preserve_implicit=True ) - def set_instance(self, model): - saved_config = self.config - self.__init__(observers=self._observers) - self.config = saved_config self._model = model - self._add_block(model) + self._set_instance() def _add_variables(self, variables: List[VarData]): for v in variables: if id(v) in self._referenced_variables: raise ValueError(f'Variable {v.name} has already been added') - self._referenced_variables[id(v)] = [{}, {}, None] + self._referenced_variables[id(v)] = ({}, {}, {}) self._vars[id(v)] = ( v, v._lb, @@ -272,44 +291,42 @@ def _add_parameters(self, params: List[ParamData]): pid = id(p) if pid in self._referenced_params: raise ValueError(f'Parameter {p.name} has already been added') - self._referenced_params[pid] = [{}, {}, None] + self._referenced_params[pid] = ({}, {}, {}) self._params[id(p)] = (p, p.value) for obs in self._observers: obs.add_parameters(params) def _check_for_new_vars(self, variables: List[VarData]): - new_vars = {} + new_vars = [] for v in variables: - v_id = id(v) - if v_id not in self._referenced_variables: - new_vars[v_id] = v - self._add_variables(list(new_vars.values())) + if id(v) not in self._referenced_variables: + new_vars.append(v) + self._add_variables(new_vars) def _check_to_remove_vars(self, variables: List[VarData]): - vars_to_remove = {} + vars_to_remove = [] for v in variables: v_id = id(v) ref_cons, ref_sos, ref_obj = self._referenced_variables[v_id] - if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: - vars_to_remove[v_id] = v - self._remove_variables(list(vars_to_remove.values())) + if not ref_cons and not ref_sos and not ref_obj: + vars_to_remove.append(v) + self._remove_variables(vars_to_remove) def _check_for_new_params(self, params: List[ParamData]): - new_params = {} + new_params = [] for p in params: - pid = id(p) - if pid not in self._referenced_params: - new_params[pid] = p - self._add_parameters(list(new_params.values())) + if id(p) not in self._referenced_params: + new_params.append(p) + self._add_parameters(new_params) def _check_to_remove_params(self, params: List[ParamData]): - params_to_remove = {} + params_to_remove = [] for p in params: p_id = id(p) ref_cons, ref_sos, ref_obj = self._referenced_params[p_id] - if len(ref_cons) == 0 and len(ref_sos) == 0 and ref_obj is None: - params_to_remove[p_id] = p - self._remove_parameters(list(params_to_remove.values())) + if not ref_cons and not ref_sos and not ref_obj: + params_to_remove.append(p) + self._remove_parameters(params_to_remove) def _add_constraints(self, cons: List[ConstraintData]): vars_to_check = [] @@ -322,7 +339,8 @@ def _add_constraints(self, cons: List[ConstraintData]): named_exprs, variables, parameters, external_functions = tmp vars_to_check.extend(variables) params_to_check.extend(parameters) - self._named_expressions[con] = [(e, e.expr) for e in named_exprs] + if len(named_exprs) > 0: + self._named_expressions[con] = [(e, e.expr) for e in named_exprs] if len(external_functions) > 0: self._external_functions[con] = external_functions self._vars_referenced_by_con[con] = variables @@ -375,61 +393,103 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.add_sos_constraints(cons) - def _set_objective(self, obj: Optional[ObjectiveData]): - vars_to_remove_check = [] - params_to_remove_check = [] - if self._objective is not None: - for v in self._vars_referenced_by_obj: - self._referenced_variables[id(v)][2] = None - for p in self._params_referenced_by_obj: - self._referenced_params[id(p)][2] = None - vars_to_remove_check.extend(self._vars_referenced_by_obj) - params_to_remove_check.extend(self._params_referenced_by_obj) - self._external_functions.pop(self._objective, None) - if obj is not None: - self._objective = obj - self._objective_expr = obj.expr - self._objective_sense = obj.sense - tmp = collect_components_from_expr(obj.expr) - named_exprs, variables, parameters, external_functions = tmp - self._check_for_new_vars(variables) - self._check_for_new_params(parameters) - self._obj_named_expressions = [(i, i.expr) for i in named_exprs] + def _add_objectives(self, objs: List[ObjectiveData]): + vars_to_check = [] + params_to_check = [] + for obj in objs: + obj_id = id(obj) + self._objectives[obj_id] = (obj, obj.expr, obj.sense) + ( + named_exprs, + variables, + parameters, + external_functions, + ) = collect_components_from_expr(obj.expr) + vars_to_check.extend(variables) + params_to_check.extend(parameters) + if len(named_exprs) > 0: + self._obj_named_expressions[obj_id] = [(e, e.expr) for e in named_exprs] if len(external_functions) > 0: self._external_functions[obj] = external_functions - self._vars_referenced_by_obj = variables - self._params_referenced_by_obj = parameters + self._vars_referenced_by_obj[obj_id] = variables + self._params_referenced_by_obj[obj_id] = parameters + self._check_for_new_vars(vars_to_check) + self._check_for_new_params(params_to_check) + for obj in objs: + obj_id = id(obj) + variables = self._vars_referenced_by_obj[obj_id] + parameters = self._params_referenced_by_obj[obj_id] for v in variables: - self._referenced_variables[id(v)][2] = obj + self._referenced_variables[id(v)][2][obj_id] = None for p in parameters: - self._referenced_params[id(p)][2] = obj - else: - self._vars_referenced_by_obj = [] - self._params_referenced_by_obj = [] - self._objective = None - self._objective_expr = None - self._objective_sense = None - self._obj_named_expressions = [] + self._referenced_params[id(p)][2][obj_id] = None + for obs in self._observers: + obs.add_objectives(objs) + + def _remove_objectives(self, objs: List[ObjectiveData]): for obs in self._observers: - obs.set_objective(obj) - self._check_to_remove_vars(vars_to_remove_check) - self._check_to_remove_params(params_to_remove_check) + obs.remove_objectives(objs) + + vars_to_check = [] + params_to_check = [] + for obj in objs: + obj_id = id(obj) + if obj_id not in self._objectives: + raise ValueError( + f'cannot remove objective {obj.name} - it was not added' + ) + for v in self._vars_referenced_by_obj[obj_id]: + self._referenced_variables[id(v)][2].pop(obj_id) + for p in self._params_referenced_by_obj[obj_id]: + self._referenced_params[id(p)][2].pop(obj_id) + vars_to_check.extend(self._vars_referenced_by_obj[obj_id]) + params_to_check.extend(self._params_referenced_by_obj[obj_id]) + del self._objectives[obj_id] + del self._obj_named_expressions[obj_id] + self._external_functions.pop(obj, None) + del self._vars_referenced_by_obj[obj_id] + del self._params_referenced_by_obj[obj_id] + self._check_to_remove_vars(vars_to_check) + self._check_to_remove_params(params_to_check) + + def _check_for_unknown_active_components(self): + for ctype in self._model.collect_ctypes(): + if not issubclass(ctype, ActiveComponent): + continue + if ctype in self._known_active_ctypes: + continue + for comp in self._model.component_data_objects( + ctype, + active=True, + descend_into=True + ): + raise NotImplementedError( + f'ModelChangeDetector does not know how to ' + 'handle compents with ctype {ctype}' + ) + + def _set_instance(self): + self._check_for_unknown_active_components() - def _add_block(self, block): self._add_constraints( list( - block.component_data_objects(Constraint, descend_into=True, active=True) + self._model.component_data_objects(Constraint, descend_into=True, active=True) ) ) self._add_sos_constraints( list( - block.component_data_objects( - SOSConstraint, descend_into=True, active=True + self._model.component_data_objects( + SOSConstraint, descend_into=True, active=True, + ) + ) + ) + self._add_objectives( + list( + self._model.component_data_objects( + Objective, descend_into=True, active=True, ) ) ) - obj = get_objective(block) - self._set_objective(obj) def _remove_constraints(self, cons: List[ConstraintData]): for obs in self._observers: @@ -488,9 +548,9 @@ def _remove_variables(self, variables: List[VarData]): f'Cannot remove variable {v.name} - it has not been added' ) cons_using, sos_using, obj_using = self._referenced_variables[v_id] - if cons_using or sos_using or (obj_using is not None): + if cons_using or sos_using or obj_using: raise ValueError( - f'Cannot remove variable {v.name} - it is still being used by constraints or the objective' + f'Cannot remove variable {v.name} - it is still being used by constraints/objectives' ) del self._referenced_variables[v_id] del self._vars[v_id] @@ -505,9 +565,9 @@ def _remove_parameters(self, params: List[ParamData]): f'Cannot remove parameter {p.name} - it has not been added' ) cons_using, sos_using, obj_using = self._referenced_params[p_id] - if cons_using or sos_using or (obj_using is not None): + if cons_using or sos_using or obj_using: raise ValueError( - f'Cannot remove parameter {p.name} - it is still being used by constraints or the objective' + f'Cannot remove parameter {p.name} - it is still being used by constraints/objectives' ) del self._referenced_params[p_id] del self._params[p_id] @@ -600,14 +660,14 @@ def _check_for_modified_constraints(self): def _check_for_var_changes(self): vars_to_update = [] cons_to_update = {} - update_obj = False + objs_to_update = {} for vid, (v, _lb, _ub, _fixed, _domain_interval, _value) in self._vars.items(): if v.fixed != _fixed: vars_to_update.append(v) for c in self._referenced_variables[vid][0]: cons_to_update[c] = None - if self._referenced_variables[vid][2] is not None: - update_obj = True + for obj_id in self._referenced_variables[vid][2]: + objs_to_update[obj_id] = None elif v._lb is not _lb: vars_to_update.append(v) elif v._ub is not _ub: @@ -617,7 +677,8 @@ def _check_for_var_changes(self): elif v.value != _value: vars_to_update.append(v) cons_to_update = list(cons_to_update.keys()) - return vars_to_update, cons_to_update, update_obj + objs_to_update = [self._objectives[obj_id][0] for obj_id in objs_to_update.keys()] + return vars_to_update, cons_to_update, objs_to_update def _check_for_param_changes(self): params_to_update = [] @@ -633,39 +694,47 @@ def _check_for_named_expression_changes(self): if named_expr.expr is not old_expr: cons_to_update.append(con) break - update_obj = False - ne_list = self._obj_named_expressions - for named_expr, old_expr in ne_list: - if named_expr.expr is not old_expr: - update_obj = True - break - return cons_to_update, update_obj - - def _check_for_new_objective(self): - update_obj = False - new_obj = get_objective(self._model) - if new_obj is not self._objective: - update_obj = True - return new_obj, update_obj - - def _check_for_objective_changes(self): - update_obj = False - if self._objective is None: - return update_obj - if self._objective.expr is not self._objective_expr: - update_obj = True - elif self._objective.sense != self._objective_sense: - # we can definitely do something faster here than resetting the whole objective - update_obj = True - return update_obj + objs_to_update = [] + for obj_id, ne_list in self._obj_named_expressions.items(): + for named_expr, old_expr in ne_list: + if named_expr.expr is not old_expr: + objs_to_update.append(self._objectives[obj_id][0]) + break + return cons_to_update, objs_to_update + + def _check_for_new_or_removed_objectives(self): + new_objs = [] + old_objs = [] + current_objs_dict = { + id(obj): obj for obj in self._model.component_data_objects( + Objective, descend_into=True, active=True + ) + } + for obj_id, obj in current_objs_dict.items(): + if obj_id not in self._objectives: + new_objs.append(obj) + for obj_id, (obj, obj_expr, obj_sense) in self._objectives.items(): + if obj_id not in current_objs_dict: + old_objs.append(obj) + return new_objs, old_objs + + def _check_for_modified_objectives(self): + objs_to_update = [] + for obj_id, (obj, obj_expr, obj_sense) in self._objectives.items(): + if obj.expr is not obj_expr or obj.sense != obj_sense: + objs_to_update.append(obj) + return objs_to_update def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if timer is None: timer = HierarchicalTimer() config: AutoUpdateConfig = self.config(value=kwds, preserve_implicit=True) + self._check_for_unknown_active_components() + added_cons = set() added_sos = set() + added_objs = {} if config.check_for_new_or_removed_constraints: timer.start('sos') @@ -695,46 +764,51 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): added_sos.update(sos_to_update) timer.stop('sos') - need_to_set_objective = False + if config.check_for_new_or_removed_objectives: + timer.start('objective') + new_objs, old_objs = self._check_for_new_or_removed_objectives() + # many solvers require one objective, so we have to remove the + # old objective first + self._remove_objectives(old_objs) + self._add_objectives(new_objs) + added_objs.update((id(i), i) for i in new_objs) + timer.stop('objective') + + if config.update_objectives: + timer.start('objective') + objs_to_update = self._check_for_modified_objectives() + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) + added_objs.update((id(i), i) for i in objs_to_update) + timer.stop('objective') if config.update_vars: timer.start('vars') - vars_to_update, cons_to_update, update_obj = self._check_for_var_changes() + vars_to_update, cons_to_update, objs_to_update = self._check_for_var_changes() self._update_variables(vars_to_update) cons_to_update = [i for i in cons_to_update if i not in added_cons] + objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] self._remove_constraints(cons_to_update) self._add_constraints(cons_to_update) added_cons.update(cons_to_update) - if update_obj: - need_to_set_objective = True + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) + added_objs.update((id(i), i) for i in objs_to_update) timer.stop('vars') if config.update_named_expressions: timer.start('named expressions') - cons_to_update, update_obj = self._check_for_named_expression_changes() + cons_to_update, objs_to_update = self._check_for_named_expression_changes() cons_to_update = [i for i in cons_to_update if i not in added_cons] + objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] self._remove_constraints(cons_to_update) self._add_constraints(cons_to_update) added_cons.update(cons_to_update) - if update_obj: - need_to_set_objective = True + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) + added_objs.update((id(i), i) for i in objs_to_update) timer.stop('named expressions') - timer.start('objective') - new_obj = self._objective - if config.check_for_new_objective: - new_obj, update_obj = self._check_for_new_objective() - if update_obj: - need_to_set_objective = True - if config.update_objective: - update_obj = self._check_for_objective_changes() - if update_obj: - need_to_set_objective = True - - if need_to_set_objective: - self._set_objective(new_obj) - timer.stop('objective') - if config.update_parameters: timer.start('params') params_to_update = self._check_for_param_changes() diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 802d4482acc..a9d367fad01 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -82,15 +82,21 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): assert isinstance(c, SOSConstraintData) self._process(cons, 'add') - def set_objective(self, obj: ObjectiveData): - assert obj is None or isinstance(obj, ObjectiveData) - self._process([obj], 'set') + def add_objectives(self, objs: List[ObjectiveData]): + for obj in objs: + assert isinstance(obj, ObjectiveData) + self._process(objs, 'add') def remove_constraints(self, cons: List[ConstraintData]): for c in cons: assert isinstance(c, ConstraintData) self._process(cons, 'remove') + def remove_objectives(self, objs: List[ObjectiveData]): + for obj in objs: + assert isinstance(obj, ObjectiveData) + self._process(objs, 'remove') + def remove_sos_constraints(self, cons: List[SOSConstraintData]): for c in cons: assert isinstance(c, SOSConstraintData) @@ -125,19 +131,15 @@ def test_objective(self): m.p = pyo.Param(mutable=True, initialize=1) obs = ObserverChecker() - detector = ModelChangeDetector([obs]) + detector = ModelChangeDetector(m, [obs]) expected = ComponentMap() - expected[None] = make_count_dict() - expected[None]['set'] += 1 - - detector.set_instance(m) obs.check(expected) m.obj = pyo.Objective(expr=m.x**2 + m.p * m.y**2) detector.update() expected[m.obj] = make_count_dict() - expected[m.obj]['set'] += 1 + expected[m.obj]['add'] += 1 expected[m.x] = make_count_dict() expected[m.x]['add'] += 1 expected[m.y] = make_count_dict() @@ -154,13 +156,15 @@ def test_objective(self): m.x.fix(2) detector.update() expected[m.x]['update'] += 1 - expected[m.obj]['set'] += 1 + expected[m.obj]['remove'] += 1 + expected[m.obj]['add'] += 1 obs.check(expected) m.x.unfix() detector.update() expected[m.x]['update'] += 1 - expected[m.obj]['set'] += 1 + expected[m.obj]['remove'] += 1 + expected[m.obj]['add'] += 1 obs.check(expected) m.p.value = 2 @@ -171,9 +175,11 @@ def test_objective(self): m.obj.expr = m.x**2 + m.y**2 detector.update() expected[m.p]['remove'] += 1 - expected[m.obj]['set'] += 1 + expected[m.obj]['remove'] += 1 + expected[m.obj]['add'] += 1 obs.check(expected) + expected[m.obj]['remove'] += 1 del m.obj m.obj = pyo.Objective(expr=m.p * m.x) detector.update() @@ -181,7 +187,7 @@ def test_objective(self): expected[m.y]['remove'] += 1 # remember, m.obj is a different object now expected[m.obj] = make_count_dict() - expected[m.obj]['set'] += 1 + expected[m.obj]['add'] += 1 def test_constraints(self): m = pyo.ConcreteModel() @@ -190,7 +196,7 @@ def test_constraints(self): m.p = pyo.Param(mutable=True, initialize=1) obs = ObserverChecker() - detector = ModelChangeDetector([obs]) + detector = ModelChangeDetector(m, [obs]) expected = ComponentMap() expected[None] = make_count_dict() @@ -364,3 +370,15 @@ def test_named_expression(self): expected[m.p]['remove'] += 1 expected[m.p]['add'] += 1 obs.check(expected) + + def test_update_config(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.obj = pyo.Objective(expr=m.x**2 + m.y**2) + + obs = ObserverChecker() + detector = ModelChangeDetector([obs]) + + expected = ComponentMap() + expected[None] = make_count_dict() From 5f3f403bfabac655d04530794f1f5b5af5e161c2 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 12:38:07 -0600 Subject: [PATCH 074/104] observer fixes --- pyomo/contrib/observer/component_collector.py | 3 + pyomo/contrib/observer/model_observer.py | 6 +- .../observer/tests/test_change_detector.py | 95 ++++++++++++------- 3 files changed, 68 insertions(+), 36 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index 2fe2170aa43..638f85327b4 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -63,6 +63,7 @@ def handle_skip(node, collector): collector_handlers[VarData] = handle_var collector_handlers[ParamData] = handle_param collector_handlers[ExpressionData] = handle_named_expression +collector_handlers[ScalarExpression] = handle_named_expression collector_handlers[ExternalFunctionExpression] = handle_external_function collector_handlers[NegationExpression] = handle_skip collector_handlers[PowExpression] = handle_skip @@ -78,6 +79,8 @@ def handle_skip(node, collector): collector_handlers[RangedExpression] = handle_skip collector_handlers[InequalityExpression] = handle_skip collector_handlers[EqualityExpression] = handle_skip +collector_handlers[int] = handle_skip +collector_handlers[float] = handle_skip class _ComponentFromExprCollector(StreamBasedExpressionVisitor): diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index f429a8ad907..4b873dd858e 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -378,7 +378,6 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): params.append(p) vars_to_check.extend(variables) params_to_check.extend(params) - self._named_expressions[con] = [] self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params self._check_for_new_vars(vars_to_check) @@ -445,7 +444,7 @@ def _remove_objectives(self, objs: List[ObjectiveData]): vars_to_check.extend(self._vars_referenced_by_obj[obj_id]) params_to_check.extend(self._params_referenced_by_obj[obj_id]) del self._objectives[obj_id] - del self._obj_named_expressions[obj_id] + self._obj_named_expressions.pop(obj_id, None) self._external_functions.pop(obj, None) del self._vars_referenced_by_obj[obj_id] del self._params_referenced_by_obj[obj_id] @@ -508,7 +507,7 @@ def _remove_constraints(self, cons: List[ConstraintData]): vars_to_check.extend(self._vars_referenced_by_con[con]) params_to_check.extend(self._params_referenced_by_con[con]) del self._active_constraints[con] - del self._named_expressions[con] + self._named_expressions.pop(con, None) self._external_functions.pop(con, None) del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] @@ -532,7 +531,6 @@ def _remove_sos_constraints(self, cons: List[SOSConstraintData]): vars_to_check.extend(self._vars_referenced_by_con[con]) params_to_check.extend(self._params_referenced_by_con[con]) del self._active_sos[con] - del self._named_expressions[con] del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] self._check_to_remove_vars(vars_to_check) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index a9d367fad01..d6b5fcbcc62 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -156,8 +156,18 @@ def test_objective(self): m.x.fix(2) detector.update() expected[m.x]['update'] += 1 + # the variable gets updated + # the objective must get removed and added + # that causes x,y, and p to all get removed + # and added expected[m.obj]['remove'] += 1 expected[m.obj]['add'] += 1 + expected[m.x]['remove'] += 1 + expected[m.x]['add'] += 1 + expected[m.y]['remove'] += 1 + expected[m.y]['add'] += 1 + expected[m.p]['remove'] += 1 + expected[m.p]['add'] += 1 obs.check(expected) m.x.unfix() @@ -165,6 +175,12 @@ def test_objective(self): expected[m.x]['update'] += 1 expected[m.obj]['remove'] += 1 expected[m.obj]['add'] += 1 + expected[m.x]['remove'] += 1 + expected[m.x]['add'] += 1 + expected[m.y]['remove'] += 1 + expected[m.y]['add'] += 1 + expected[m.p]['remove'] += 1 + expected[m.p]['add'] += 1 obs.check(expected) m.p.value = 2 @@ -174,20 +190,27 @@ def test_objective(self): m.obj.expr = m.x**2 + m.y**2 detector.update() - expected[m.p]['remove'] += 1 expected[m.obj]['remove'] += 1 expected[m.obj]['add'] += 1 + expected[m.x]['remove'] += 1 + expected[m.x]['add'] += 1 + expected[m.y]['remove'] += 1 + expected[m.y]['add'] += 1 + expected[m.p]['remove'] += 1 obs.check(expected) expected[m.obj]['remove'] += 1 del m.obj - m.obj = pyo.Objective(expr=m.p * m.x) + m.obj2 = pyo.Objective(expr=m.p * m.x) detector.update() - expected[m.p]['add'] += 1 - expected[m.y]['remove'] += 1 # remember, m.obj is a different object now - expected[m.obj] = make_count_dict() - expected[m.obj]['add'] += 1 + expected[m.obj2] = make_count_dict() + expected[m.obj2]['add'] += 1 + expected[m.x]['remove'] += 1 + expected[m.x]['add'] += 1 + expected[m.y]['remove'] += 1 + expected[m.p]['add'] += 1 + obs.check(expected) def test_constraints(self): m = pyo.ConcreteModel() @@ -199,10 +222,6 @@ def test_constraints(self): detector = ModelChangeDetector(m, [obs]) expected = ComponentMap() - expected[None] = make_count_dict() - expected[None]['set'] += 1 - - detector.set_instance(m) obs.check(expected) m.obj = pyo.Objective(expr=m.y) @@ -217,15 +236,13 @@ def test_constraints(self): expected[m.c1] = make_count_dict() expected[m.c1]['add'] += 1 expected[m.obj] = make_count_dict() - expected[m.obj]['set'] += 1 + expected[m.obj]['add'] += 1 obs.check(expected) # now fix a variable and make sure the # constraint gets removed and added m.x.fix(1) - obs.pprint() detector.update() - obs.pprint() expected[m.c1]['remove'] += 1 expected[m.c1]['add'] += 1 # because x and p are only used in the @@ -248,8 +265,7 @@ def test_sos(self): m.c1 = pyo.SOSConstraint(var=m.x, sos=1) obs = ObserverChecker() - detector = ModelChangeDetector([obs]) - detector.set_instance(m) + detector = ModelChangeDetector(m, [obs]) expected = ComponentMap() expected[m.obj] = make_count_dict() @@ -257,7 +273,7 @@ def test_sos(self): expected[m.x[i]] = make_count_dict() expected[m.y] = make_count_dict() expected[m.c1] = make_count_dict() - expected[m.obj]['set'] += 1 + expected[m.obj]['add'] += 1 for i in m.a: expected[m.x[i]]['add'] += 1 expected[m.y]['add'] += 1 @@ -280,13 +296,9 @@ def test_vars_and_params_elsewhere(self): m2 = pyo.ConcreteModel() obs = ObserverChecker() - detector = ModelChangeDetector([obs]) + detector = ModelChangeDetector(m2, [obs]) expected = ComponentMap() - expected[None] = make_count_dict() - expected[None]['set'] += 1 - - detector.set_instance(m2) obs.check(expected) m2.obj = pyo.Objective(expr=m1.y) @@ -301,15 +313,13 @@ def test_vars_and_params_elsewhere(self): expected[m2.c1] = make_count_dict() expected[m2.c1]['add'] += 1 expected[m2.obj] = make_count_dict() - expected[m2.obj]['set'] += 1 + expected[m2.obj]['add'] += 1 obs.check(expected) # now fix a variable and make sure the # constraint gets removed and added m1.x.fix(1) - obs.pprint() detector.update() - obs.pprint() expected[m2.c1]['remove'] += 1 expected[m2.c1]['add'] += 1 # because x and p are only used in the @@ -330,13 +340,9 @@ def test_named_expression(self): m.p = pyo.Param(mutable=True, initialize=1) obs = ObserverChecker() - detector = ModelChangeDetector([obs]) + detector = ModelChangeDetector(m, [obs]) expected = ComponentMap() - expected[None] = make_count_dict() - expected[None]['set'] += 1 - - detector.set_instance(m) obs.check(expected) m.obj = pyo.Objective(expr=m.y) @@ -352,7 +358,7 @@ def test_named_expression(self): expected[m.c1] = make_count_dict() expected[m.c1]['add'] += 1 expected[m.obj] = make_count_dict() - expected[m.obj]['set'] += 1 + expected[m.obj]['add'] += 1 obs.check(expected) # now modify the named expression and make sure the @@ -376,9 +382,34 @@ def test_update_config(self): m.x = pyo.Var() m.y = pyo.Var() m.obj = pyo.Objective(expr=m.x**2 + m.y**2) + m.c1 = pyo.Constraint(expr=m.y >= pyo.exp(m.x)) obs = ObserverChecker() - detector = ModelChangeDetector([obs]) + detector = ModelChangeDetector(m, [obs]) expected = ComponentMap() - expected[None] = make_count_dict() + expected[m.x] = make_count_dict() + expected[m.y] = make_count_dict() + expected[m.obj] = make_count_dict() + expected[m.c1] = make_count_dict() + expected[m.x]['add'] += 1 + expected[m.y]['add'] += 1 + expected[m.obj]['add'] += 1 + expected[m.c1]['add'] += 1 + obs.check(expected) + + detector.config.check_for_new_or_removed_constraints = False + detector.config.update_constraints = False + m.c2 = pyo.Constraint(expr=m.y >= (m.x - 1)**2) + detector.update() + obs.check(expected) + + m.x.setlb(0) + detector.update() + expected[m.x]['update'] += 1 + obs.check(expected) + + detector.config.check_for_new_or_removed_constraints = True + detector.update() + expected[m.c2] = make_count_dict() + expected[m.c2]['add'] += 1 From 79e1b472c549c30918068ececab9aa21ca9601c8 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 12:46:02 -0600 Subject: [PATCH 075/104] observer config updates --- pyomo/contrib/observer/model_observer.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 4b873dd858e..335619a9f2e 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -12,7 +12,7 @@ import abc from typing import List, Sequence, Optional -from pyomo.common.config import ConfigDict, ConfigValue +from pyomo.common.config import ConfigDict, ConfigValue, document_configdict from pyomo.core.base.constraint import ConstraintData, Constraint from pyomo.core.base.sos import SOSConstraintData, SOSConstraint from pyomo.core.base.var import VarData @@ -48,6 +48,7 @@ """ +@document_configdict() class AutoUpdateConfig(ConfigDict): """ Control which parts of the model are automatically checked and/or updated upon re-solve @@ -71,6 +72,7 @@ def __init__( visibility=visibility, ) + # automatically detect new/removed constraints on subsequent solves self.check_for_new_or_removed_constraints: bool = self.declare( 'check_for_new_or_removed_constraints', ConfigValue( @@ -84,6 +86,7 @@ def __init__( model.""", ), ) + # automatically detect new/removed objectives on subsequent solves self.check_for_new_or_removed_objectives: bool = self.declare( 'check_for_new_or_removed_objectives', ConfigValue( @@ -97,6 +100,7 @@ def __init__( model.""", ), ) + # automatically detect changes to constraints on subsequent solves self.update_constraints: bool = self.declare( 'update_constraints', ConfigValue( @@ -111,6 +115,7 @@ def __init__( being modified.""", ), ) + # automatically detect changes to variables on subsequent solves self.update_vars: bool = self.declare( 'update_vars', ConfigValue( @@ -126,6 +131,7 @@ def __init__( update_parameters_and_fixed_vars.""", ), ) + # automatically detect changes to parameters on subsequent solves self.update_parameters: bool = self.declare( 'update_parameters', ConfigValue( @@ -139,6 +145,7 @@ def __init__( parameters are not being modified.""", ), ) + # automatically detect changes to named expressions on subsequent solves self.update_named_expressions: bool = self.declare( 'update_named_expressions', ConfigValue( @@ -151,6 +158,7 @@ def __init__( are certain Expressions are not being modified.""", ), ) + # automatically detect changes to objectives on subsequent solves self.update_objectives: bool = self.declare( 'update_objectives', ConfigValue( From 7275176961665ea610ee35c676f37b9bca2525a9 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 12:50:32 -0600 Subject: [PATCH 076/104] minor observer updates --- pyomo/contrib/observer/model_observer.py | 59 +++++++++++++++--------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 335619a9f2e..fcf18dcb310 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -745,28 +745,34 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if config.check_for_new_or_removed_constraints: timer.start('sos') new_sos, old_sos = self._check_for_new_or_removed_sos() - self._add_sos_constraints(new_sos) - self._remove_sos_constraints(old_sos) + if new_sos: + self._add_sos_constraints(new_sos) + if old_sos: + self._remove_sos_constraints(old_sos) added_sos.update(new_sos) timer.stop('sos') timer.start('cons') new_cons, old_cons = self._check_for_new_or_removed_constraints() - self._add_constraints(new_cons) - self._remove_constraints(old_cons) + if new_cons: + self._add_constraints(new_cons) + if old_cons: + self._remove_constraints(old_cons) added_cons.update(new_cons) timer.stop('cons') if config.update_constraints: timer.start('cons') cons_to_update = self._check_for_modified_constraints() - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + if cons_to_update: + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) added_cons.update(cons_to_update) timer.stop('cons') timer.start('sos') sos_to_update = self._check_for_modified_sos() - self._remove_sos_constraints(sos_to_update) - self._add_sos_constraints(sos_to_update) + if sos_to_update: + self._remove_sos_constraints(sos_to_update) + self._add_sos_constraints(sos_to_update) added_sos.update(sos_to_update) timer.stop('sos') @@ -775,30 +781,36 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): new_objs, old_objs = self._check_for_new_or_removed_objectives() # many solvers require one objective, so we have to remove the # old objective first - self._remove_objectives(old_objs) - self._add_objectives(new_objs) + if old_objs: + self._remove_objectives(old_objs) + if new_objs: + self._add_objectives(new_objs) added_objs.update((id(i), i) for i in new_objs) timer.stop('objective') if config.update_objectives: timer.start('objective') objs_to_update = self._check_for_modified_objectives() - self._remove_objectives(objs_to_update) - self._add_objectives(objs_to_update) + if objs_to_update: + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) added_objs.update((id(i), i) for i in objs_to_update) timer.stop('objective') if config.update_vars: timer.start('vars') vars_to_update, cons_to_update, objs_to_update = self._check_for_var_changes() - self._update_variables(vars_to_update) + if vars_to_update: + self._update_variables(vars_to_update) cons_to_update = [i for i in cons_to_update if i not in added_cons] objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + if cons_to_update: + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) added_cons.update(cons_to_update) - self._remove_objectives(objs_to_update) - self._add_objectives(objs_to_update) + if objs_to_update: + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) added_objs.update((id(i), i) for i in objs_to_update) timer.stop('vars') @@ -807,16 +819,19 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): cons_to_update, objs_to_update = self._check_for_named_expression_changes() cons_to_update = [i for i in cons_to_update if i not in added_cons] objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) + if cons_to_update: + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) added_cons.update(cons_to_update) - self._remove_objectives(objs_to_update) - self._add_objectives(objs_to_update) + if objs_to_update: + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) added_objs.update((id(i), i) for i in objs_to_update) timer.stop('named expressions') if config.update_parameters: timer.start('params') params_to_update = self._check_for_param_changes() - self._update_parameters(params_to_update) + if params_to_update: + self._update_parameters(params_to_update) timer.stop('params') From 823e15a410bbac6e6950ea7ae6008c017ca750bf Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 12:58:02 -0600 Subject: [PATCH 077/104] disable gc for expensive parts of observer --- pyomo/contrib/observer/model_observer.py | 244 ++++++++++++----------- 1 file changed, 130 insertions(+), 114 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index fcf18dcb310..48dac5ccb62 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -25,6 +25,7 @@ from pyomo.contrib.solver.common.util import get_objective from pyomo.contrib.observer.component_collector import collect_components_from_expr from pyomo.common.numeric_types import native_numeric_types +import gc """ @@ -476,27 +477,35 @@ def _check_for_unknown_active_components(self): ) def _set_instance(self): - self._check_for_unknown_active_components() - self._add_constraints( - list( - self._model.component_data_objects(Constraint, descend_into=True, active=True) + is_gc_enabled = gc.isenabled() + gc.disable() + + try: + self._check_for_unknown_active_components() + + self._add_constraints( + list( + self._model.component_data_objects(Constraint, descend_into=True, active=True) + ) ) - ) - self._add_sos_constraints( - list( - self._model.component_data_objects( - SOSConstraint, descend_into=True, active=True, + self._add_sos_constraints( + list( + self._model.component_data_objects( + SOSConstraint, descend_into=True, active=True, + ) ) ) - ) - self._add_objectives( - list( - self._model.component_data_objects( - Objective, descend_into=True, active=True, + self._add_objectives( + list( + self._model.component_data_objects( + Objective, descend_into=True, active=True, + ) ) ) - ) + finally: + if is_gc_enabled: + gc.enable() def _remove_constraints(self, cons: List[ConstraintData]): for obs in self._observers: @@ -736,102 +745,109 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): timer = HierarchicalTimer() config: AutoUpdateConfig = self.config(value=kwds, preserve_implicit=True) - self._check_for_unknown_active_components() - - added_cons = set() - added_sos = set() - added_objs = {} - - if config.check_for_new_or_removed_constraints: - timer.start('sos') - new_sos, old_sos = self._check_for_new_or_removed_sos() - if new_sos: - self._add_sos_constraints(new_sos) - if old_sos: - self._remove_sos_constraints(old_sos) - added_sos.update(new_sos) - timer.stop('sos') - timer.start('cons') - new_cons, old_cons = self._check_for_new_or_removed_constraints() - if new_cons: - self._add_constraints(new_cons) - if old_cons: - self._remove_constraints(old_cons) - added_cons.update(new_cons) - timer.stop('cons') - - if config.update_constraints: - timer.start('cons') - cons_to_update = self._check_for_modified_constraints() - if cons_to_update: - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) - added_cons.update(cons_to_update) - timer.stop('cons') - timer.start('sos') - sos_to_update = self._check_for_modified_sos() - if sos_to_update: - self._remove_sos_constraints(sos_to_update) - self._add_sos_constraints(sos_to_update) - added_sos.update(sos_to_update) - timer.stop('sos') - - if config.check_for_new_or_removed_objectives: - timer.start('objective') - new_objs, old_objs = self._check_for_new_or_removed_objectives() - # many solvers require one objective, so we have to remove the - # old objective first - if old_objs: - self._remove_objectives(old_objs) - if new_objs: - self._add_objectives(new_objs) - added_objs.update((id(i), i) for i in new_objs) - timer.stop('objective') - - if config.update_objectives: - timer.start('objective') - objs_to_update = self._check_for_modified_objectives() - if objs_to_update: - self._remove_objectives(objs_to_update) - self._add_objectives(objs_to_update) - added_objs.update((id(i), i) for i in objs_to_update) - timer.stop('objective') - - if config.update_vars: - timer.start('vars') - vars_to_update, cons_to_update, objs_to_update = self._check_for_var_changes() - if vars_to_update: - self._update_variables(vars_to_update) - cons_to_update = [i for i in cons_to_update if i not in added_cons] - objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] - if cons_to_update: - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) - added_cons.update(cons_to_update) - if objs_to_update: - self._remove_objectives(objs_to_update) - self._add_objectives(objs_to_update) - added_objs.update((id(i), i) for i in objs_to_update) - timer.stop('vars') - - if config.update_named_expressions: - timer.start('named expressions') - cons_to_update, objs_to_update = self._check_for_named_expression_changes() - cons_to_update = [i for i in cons_to_update if i not in added_cons] - objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] - if cons_to_update: - self._remove_constraints(cons_to_update) - self._add_constraints(cons_to_update) - added_cons.update(cons_to_update) - if objs_to_update: - self._remove_objectives(objs_to_update) - self._add_objectives(objs_to_update) - added_objs.update((id(i), i) for i in objs_to_update) - timer.stop('named expressions') - - if config.update_parameters: - timer.start('params') - params_to_update = self._check_for_param_changes() - if params_to_update: - self._update_parameters(params_to_update) - timer.stop('params') + is_gc_enabled = gc.isenabled() + gc.disable() + + try: + self._check_for_unknown_active_components() + + added_cons = set() + added_sos = set() + added_objs = {} + + if config.check_for_new_or_removed_constraints: + timer.start('sos') + new_sos, old_sos = self._check_for_new_or_removed_sos() + if new_sos: + self._add_sos_constraints(new_sos) + if old_sos: + self._remove_sos_constraints(old_sos) + added_sos.update(new_sos) + timer.stop('sos') + timer.start('cons') + new_cons, old_cons = self._check_for_new_or_removed_constraints() + if new_cons: + self._add_constraints(new_cons) + if old_cons: + self._remove_constraints(old_cons) + added_cons.update(new_cons) + timer.stop('cons') + + if config.update_constraints: + timer.start('cons') + cons_to_update = self._check_for_modified_constraints() + if cons_to_update: + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) + added_cons.update(cons_to_update) + timer.stop('cons') + timer.start('sos') + sos_to_update = self._check_for_modified_sos() + if sos_to_update: + self._remove_sos_constraints(sos_to_update) + self._add_sos_constraints(sos_to_update) + added_sos.update(sos_to_update) + timer.stop('sos') + + if config.check_for_new_or_removed_objectives: + timer.start('objective') + new_objs, old_objs = self._check_for_new_or_removed_objectives() + # many solvers require one objective, so we have to remove the + # old objective first + if old_objs: + self._remove_objectives(old_objs) + if new_objs: + self._add_objectives(new_objs) + added_objs.update((id(i), i) for i in new_objs) + timer.stop('objective') + + if config.update_objectives: + timer.start('objective') + objs_to_update = self._check_for_modified_objectives() + if objs_to_update: + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) + added_objs.update((id(i), i) for i in objs_to_update) + timer.stop('objective') + + if config.update_vars: + timer.start('vars') + vars_to_update, cons_to_update, objs_to_update = self._check_for_var_changes() + if vars_to_update: + self._update_variables(vars_to_update) + cons_to_update = [i for i in cons_to_update if i not in added_cons] + objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] + if cons_to_update: + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) + added_cons.update(cons_to_update) + if objs_to_update: + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) + added_objs.update((id(i), i) for i in objs_to_update) + timer.stop('vars') + + if config.update_named_expressions: + timer.start('named expressions') + cons_to_update, objs_to_update = self._check_for_named_expression_changes() + cons_to_update = [i for i in cons_to_update if i not in added_cons] + objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] + if cons_to_update: + self._remove_constraints(cons_to_update) + self._add_constraints(cons_to_update) + added_cons.update(cons_to_update) + if objs_to_update: + self._remove_objectives(objs_to_update) + self._add_objectives(objs_to_update) + added_objs.update((id(i), i) for i in objs_to_update) + timer.stop('named expressions') + + if config.update_parameters: + timer.start('params') + params_to_update = self._check_for_param_changes() + if params_to_update: + self._update_parameters(params_to_update) + timer.stop('params') + finally: + if is_gc_enabled: + gc.enable() From 73258f50838414992f2778a12b58183e2c186231 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 12:59:57 -0600 Subject: [PATCH 078/104] minor observer updates --- pyomo/contrib/observer/model_observer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 48dac5ccb62..0c4254d25f9 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -344,8 +344,12 @@ def _add_constraints(self, cons: List[ConstraintData]): if con in self._active_constraints: raise ValueError(f'Constraint {con.name} has already been added') self._active_constraints[con] = con.expr - tmp = collect_components_from_expr(con.expr) - named_exprs, variables, parameters, external_functions = tmp + ( + named_exprs, + variables, + parameters, + external_functions, + ) = collect_components_from_expr(con.expr) vars_to_check.extend(variables) params_to_check.extend(parameters) if len(named_exprs) > 0: From 5a8a5a683496c74e216becb256c2b6e8e42d5a88 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 13:01:11 -0600 Subject: [PATCH 079/104] minor observer updates --- pyomo/contrib/observer/model_observer.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 0c4254d25f9..dd6cd28892f 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -352,9 +352,9 @@ def _add_constraints(self, cons: List[ConstraintData]): ) = collect_components_from_expr(con.expr) vars_to_check.extend(variables) params_to_check.extend(parameters) - if len(named_exprs) > 0: + if named_exprs: self._named_expressions[con] = [(e, e.expr) for e in named_exprs] - if len(external_functions) > 0: + if external_functions: self._external_functions[con] = external_functions self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = parameters @@ -419,9 +419,9 @@ def _add_objectives(self, objs: List[ObjectiveData]): ) = collect_components_from_expr(obj.expr) vars_to_check.extend(variables) params_to_check.extend(parameters) - if len(named_exprs) > 0: + if named_exprs: self._obj_named_expressions[obj_id] = [(e, e.expr) for e in named_exprs] - if len(external_functions) > 0: + if external_functions: self._external_functions[obj] = external_functions self._vars_referenced_by_obj[obj_id] = variables self._params_referenced_by_obj[obj_id] = parameters From 321755a3460eac135be15e6cbf02b0b1ee7fae1c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 16:01:51 -0600 Subject: [PATCH 080/104] minor observer updates --- pyomo/contrib/observer/model_observer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index dd6cd28892f..e00331b47ed 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -687,14 +687,14 @@ def _check_for_var_changes(self): cons_to_update[c] = None for obj_id in self._referenced_variables[vid][2]: objs_to_update[obj_id] = None + elif _fixed and v.value != _value: + vars_to_update.append(v) elif v._lb is not _lb: vars_to_update.append(v) elif v._ub is not _ub: vars_to_update.append(v) elif _domain_interval != v.domain.get_interval(): vars_to_update.append(v) - elif v.value != _value: - vars_to_update.append(v) cons_to_update = list(cons_to_update.keys()) objs_to_update = [self._objectives[obj_id][0] for obj_id in objs_to_update.keys()] return vars_to_update, cons_to_update, objs_to_update From 2adefbc724afbbe63a19f1a03966a55579930e0c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 17:09:12 -0600 Subject: [PATCH 081/104] docstring for the model change detector --- pyomo/contrib/observer/model_observer.py | 121 +++++++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index e00331b47ed..647c8aef5d8 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -229,10 +229,131 @@ def update_parameters(self, params: List[ParamData]): class ModelChangeDetector: + """ + This class "watches" a pyomo model and notifies the observers when any + changes to the model are made (but only when ModelChangeDetector.update + is called). An example use case is for the persistent solver interfaces. + + The ModelChangeDetector considers the model to be defined by its set of + active components and any components used by those active components. For + example, the observers will not be notified of the addition of a variable + if that variable is not used in any constraints. + + The Observer/ModelChangeDetector are most useful when a small number + of changes are being relative to the size of the model. For example, + the persistent solver interfaces can be very efficient when repeatedly + solving the same model but with different values for mutable parameters. + + If you know that certain changes will not be made to the model, the + config can be modified to improve performance. For example, if you + know that no constraints will be added to or removed from the model, + then ``check_for_new_or_removed_constraints`` can be set to ``False``, + which will save some time when ``update`` is called. + + Here are some examples: + + >>> import pyomo.environ as pyo + >>> from pyomo.contrib.observer.model_observer import ( + ... AutoUpdateConfig, + ... Observer, + ... ModelChangeDetector, + ... ) + >>> class PrintObserver(Observer): + ... def add_variables(self, variables): + ... for i in variables: + ... print(f'{i} was added to the model') + ... def add_parameters(self, params): + ... for i in params: + ... print(f'{i} was added to the model') + ... def add_constraints(self, cons): + ... for i in cons: + ... print(f'{i} was added to the model') + ... def add_sos_constraints(self, cons): + ... for i in cons: + ... print(f'{i} was added to the model') + ... def add_objectives(self, objs): + ... for i in objs: + ... print(f'{i} was added to the model') + ... def remove_objectives(self, objs): + ... for i in objs: + ... print(f'{i} was removed from the model') + ... def remove_constraints(self, cons): + ... for i in cons: + ... print(f'{i} was removed from the model') + ... def remove_sos_constraints(self, cons): + ... for i in cons: + ... print(f'{i} was removed from the model') + ... def remove_variables(self, variables): + ... for i in variables: + ... print(f'{i} was removed from the model') + ... def remove_parameters(self, params): + ... for i in params: + ... print(f'{i} was removed from the model') + ... def update_variables(self, variables): + ... for i in variables: + ... print(f'{i} was modified') + ... def update_parameters(self, params): + ... for i in params: + ... print(f'{i} was modified') + >>> m = pyo.ConcreteModel() + >>> obs = PrintObserver() + >>> detector = ModelChangeDetector(m, [obs]) + >>> m.x = pyo.Var(bounds=()) + >>> m.y = pyo.Var() + >>> detector.update() # no output because the variables are not used + >>> m.obj = pyo.Objective(expr=m.x**2 + m.y**2) + >>> detector.update() + x was added to the model + y was added to the model + obj was added to the model + >>> del m.obj + >>> detector.update() + obj was removed from the model + x was removed from the model + y was removed from the model + >>> m.px = pyo.Param(mutable=True, initialize=1) + >>> m.py = pyo.Param(mutable=True, initialize=1) + >>> m.obj = pyo.Objective(expr=m.px*m.x + m.py*m.y) + >>> detector.update() + x was added to the model + y was added to the model + px was added to the model + py was added to the model + obj was added to the model + >>> detector.config.check_for_new_or_removed_constraints = False + >>> detector.config.check_for_new_or_removed_objectives = False + >>> detector.config.update_constraints = False + >>> detector.config.update_vars = False + >>> detector.config.update_parameters = True + >>> detector.config.update_named_expressions = False + >>> detector.config.update_objectives = False + >>> for i in range(10): + ... m.py.value = i + ... detector.update() # this will be faster because it is only checking for changes to parameters + py was modified + py was modified + py was modified + py was modified + py was modified + py was modified + py was modified + py was modified + py was modified + py was modified + >>> m.c = pyo.Constraint(expr=m.y >= pyo.exp(m.x)) + >>> detector.update() # no output because we did not check for new constraints + >>> detector.config.check_for_new_or_removed_constraints = True + >>> detector.update() + c was added to the model + + """ + def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): """ Parameters ---------- + model: BlockData + The model for which changes should be detected observers: Sequence[Observer] The objects to notify when changes are made to the model """ From df26c4e96f6bcf2e1c23aa5fd691fd9a86135c57 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 17:10:05 -0600 Subject: [PATCH 082/104] run black --- pyomo/contrib/observer/model_observer.py | 73 ++++++++++--------- .../observer/tests/test_change_detector.py | 4 +- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 647c8aef5d8..b1442a43ef7 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -230,31 +230,31 @@ def update_parameters(self, params: List[ParamData]): class ModelChangeDetector: """ - This class "watches" a pyomo model and notifies the observers when any + This class "watches" a pyomo model and notifies the observers when any changes to the model are made (but only when ModelChangeDetector.update is called). An example use case is for the persistent solver interfaces. - - The ModelChangeDetector considers the model to be defined by its set of + + The ModelChangeDetector considers the model to be defined by its set of active components and any components used by those active components. For example, the observers will not be notified of the addition of a variable - if that variable is not used in any constraints. + if that variable is not used in any constraints. - The Observer/ModelChangeDetector are most useful when a small number - of changes are being relative to the size of the model. For example, + The Observer/ModelChangeDetector are most useful when a small number + of changes are being relative to the size of the model. For example, the persistent solver interfaces can be very efficient when repeatedly solving the same model but with different values for mutable parameters. - If you know that certain changes will not be made to the model, the - config can be modified to improve performance. For example, if you + If you know that certain changes will not be made to the model, the + config can be modified to improve performance. For example, if you know that no constraints will be added to or removed from the model, then ``check_for_new_or_removed_constraints`` can be set to ``False``, which will save some time when ``update`` is called. - + Here are some examples: >>> import pyomo.environ as pyo >>> from pyomo.contrib.observer.model_observer import ( - ... AutoUpdateConfig, + ... AutoUpdateConfig, ... Observer, ... ModelChangeDetector, ... ) @@ -371,18 +371,18 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): self._external_functions = ComponentMap() - # the dictionaries below are really just ordered sets, but we need to + # the dictionaries below are really just ordered sets, but we need to # stick with built-in types for performance # var_id: ( # dict[constraints, None], - # dict[sos constraints, None], + # dict[sos constraints, None], # dict[objectives, None], # ) self._referenced_variables = {} # param_id: ( - # dict[constraints, None], + # dict[constraints, None], # dict[sos constraints, None], # dict[objectives, None], # ) @@ -465,12 +465,9 @@ def _add_constraints(self, cons: List[ConstraintData]): if con in self._active_constraints: raise ValueError(f'Constraint {con.name} has already been added') self._active_constraints[con] = con.expr - ( - named_exprs, - variables, - parameters, - external_functions, - ) = collect_components_from_expr(con.expr) + (named_exprs, variables, parameters, external_functions) = ( + collect_components_from_expr(con.expr) + ) vars_to_check.extend(variables) params_to_check.extend(parameters) if named_exprs: @@ -532,12 +529,9 @@ def _add_objectives(self, objs: List[ObjectiveData]): for obj in objs: obj_id = id(obj) self._objectives[obj_id] = (obj, obj.expr, obj.sense) - ( - named_exprs, - variables, - parameters, - external_functions, - ) = collect_components_from_expr(obj.expr) + (named_exprs, variables, parameters, external_functions) = ( + collect_components_from_expr(obj.expr) + ) vars_to_check.extend(variables) params_to_check.extend(parameters) if named_exprs: @@ -592,9 +586,7 @@ def _check_for_unknown_active_components(self): if ctype in self._known_active_ctypes: continue for comp in self._model.component_data_objects( - ctype, - active=True, - descend_into=True + ctype, active=True, descend_into=True ): raise NotImplementedError( f'ModelChangeDetector does not know how to ' @@ -611,20 +603,22 @@ def _set_instance(self): self._add_constraints( list( - self._model.component_data_objects(Constraint, descend_into=True, active=True) + self._model.component_data_objects( + Constraint, descend_into=True, active=True + ) ) ) self._add_sos_constraints( list( self._model.component_data_objects( - SOSConstraint, descend_into=True, active=True, + SOSConstraint, descend_into=True, active=True ) ) ) self._add_objectives( list( self._model.component_data_objects( - Objective, descend_into=True, active=True, + Objective, descend_into=True, active=True ) ) ) @@ -817,7 +811,9 @@ def _check_for_var_changes(self): elif _domain_interval != v.domain.get_interval(): vars_to_update.append(v) cons_to_update = list(cons_to_update.keys()) - objs_to_update = [self._objectives[obj_id][0] for obj_id in objs_to_update.keys()] + objs_to_update = [ + self._objectives[obj_id][0] for obj_id in objs_to_update.keys() + ] return vars_to_update, cons_to_update, objs_to_update def _check_for_param_changes(self): @@ -846,7 +842,8 @@ def _check_for_new_or_removed_objectives(self): new_objs = [] old_objs = [] current_objs_dict = { - id(obj): obj for obj in self._model.component_data_objects( + id(obj): obj + for obj in self._model.component_data_objects( Objective, descend_into=True, active=True ) } @@ -917,7 +914,7 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if config.check_for_new_or_removed_objectives: timer.start('objective') new_objs, old_objs = self._check_for_new_or_removed_objectives() - # many solvers require one objective, so we have to remove the + # many solvers require one objective, so we have to remove the # old objective first if old_objs: self._remove_objectives(old_objs) @@ -937,7 +934,9 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if config.update_vars: timer.start('vars') - vars_to_update, cons_to_update, objs_to_update = self._check_for_var_changes() + vars_to_update, cons_to_update, objs_to_update = ( + self._check_for_var_changes() + ) if vars_to_update: self._update_variables(vars_to_update) cons_to_update = [i for i in cons_to_update if i not in added_cons] @@ -954,7 +953,9 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): if config.update_named_expressions: timer.start('named expressions') - cons_to_update, objs_to_update = self._check_for_named_expression_changes() + cons_to_update, objs_to_update = ( + self._check_for_named_expression_changes() + ) cons_to_update = [i for i in cons_to_update if i not in added_cons] objs_to_update = [i for i in objs_to_update if id(i) not in added_objs] if cons_to_update: diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index d6b5fcbcc62..19609182daf 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -158,7 +158,7 @@ def test_objective(self): expected[m.x]['update'] += 1 # the variable gets updated # the objective must get removed and added - # that causes x,y, and p to all get removed + # that causes x,y, and p to all get removed # and added expected[m.obj]['remove'] += 1 expected[m.obj]['add'] += 1 @@ -400,7 +400,7 @@ def test_update_config(self): detector.config.check_for_new_or_removed_constraints = False detector.config.update_constraints = False - m.c2 = pyo.Constraint(expr=m.y >= (m.x - 1)**2) + m.c2 = pyo.Constraint(expr=m.y >= (m.x - 1) ** 2) detector.update() obs.check(expected) From 2dc566ec3d3f93150194642282470c75b57ac624 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 17:34:26 -0600 Subject: [PATCH 083/104] some comments for the observer --- pyomo/contrib/observer/model_observer.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index b1442a43ef7..0fd5a489b43 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -863,6 +863,30 @@ def _check_for_modified_objectives(self): return objs_to_update def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): + """ + Check for changes to the model and notify the observers. + + Parameters + ---------- + timer: Optional[HierarchicalTimer] + The timer to use for tracking how much time is spent detecting + different kinds of changes + """ + + """ + When possible, it is better to add new constraints before removing old + constraints. This prevents unnecessarily removing and adding variables. + If a constraint is removed, any variables that are used only by that + constraint will be removed. If there is a new constraint that uses + the same variable, then we don't actually need to remove the variable. + This is hard to avoid when we are modifying a constraint or changing + the objective. When the objective changes, we remove the old one + first just because most things don't handle multiple objectives. + + We check for changes to constraints/objectives before variables/parameters + so that we don't waste time updating a variable/parameter that is going to + get removed. + """ if timer is None: timer = HierarchicalTimer() config: AutoUpdateConfig = self.config(value=kwds, preserve_implicit=True) From 6b2ffbd84153a069880c39b964ed00e4a4e0f8b0 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 17:51:22 -0600 Subject: [PATCH 084/104] only need to descend into named expressions once --- pyomo/contrib/observer/component_collector.py | 5 ++++ .../tests/test_component_collector.py | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 pyomo/contrib/observer/tests/test_component_collector.py diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index 638f85327b4..c79c1182869 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -93,6 +93,11 @@ def __init__(self, **kwds): def exitNode(self, node, data): return collector_handlers[node.__class__](node, self) + + def beforeChild(self, node, child, child_idx): + if id(child) in self.named_expressions: + return False, None + return True, None _visitor = _ComponentFromExprCollector() diff --git a/pyomo/contrib/observer/tests/test_component_collector.py b/pyomo/contrib/observer/tests/test_component_collector.py new file mode 100644 index 00000000000..99d00074418 --- /dev/null +++ b/pyomo/contrib/observer/tests/test_component_collector.py @@ -0,0 +1,25 @@ +import pyomo.environ as pyo +from pyomo.common import unittest +from pyomo.contrib.observer.component_collector import collect_components_from_expr +from pyomo.common.collections import ComponentSet + + +class TestComponentCollector(unittest.TestCase): + def test_nested_named_expressions(self): + m = pyo.ConcreteModel() + m.x = pyo.Var() + m.y = pyo.Var() + m.z = pyo.Var() + m.e1 = pyo.Expression(expr=m.x + m.y) + m.e2 = pyo.Expression(expr=m.e1 + m.z) + e = m.e2 * pyo.exp(m.e2) + ( + named_exprs, + vars, + params, + external_funcs, + ) = collect_components_from_expr(e) + self.assertEqual(len(named_exprs), 2) + named_exprs = ComponentSet(named_exprs) + self.assertIn(m.e1, named_exprs) + self.assertIn(m.e2, named_exprs) From e68ee73049a9fa538b37c30b433333b2459478f1 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 17:51:43 -0600 Subject: [PATCH 085/104] only need to descend into named expressions once --- pyomo/contrib/observer/component_collector.py | 2 +- pyomo/contrib/observer/tests/test_component_collector.py | 7 +------ 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index c79c1182869..d30bb128758 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -93,7 +93,7 @@ def __init__(self, **kwds): def exitNode(self, node, data): return collector_handlers[node.__class__](node, self) - + def beforeChild(self, node, child, child_idx): if id(child) in self.named_expressions: return False, None diff --git a/pyomo/contrib/observer/tests/test_component_collector.py b/pyomo/contrib/observer/tests/test_component_collector.py index 99d00074418..ca272b06cfa 100644 --- a/pyomo/contrib/observer/tests/test_component_collector.py +++ b/pyomo/contrib/observer/tests/test_component_collector.py @@ -13,12 +13,7 @@ def test_nested_named_expressions(self): m.e1 = pyo.Expression(expr=m.x + m.y) m.e2 = pyo.Expression(expr=m.e1 + m.z) e = m.e2 * pyo.exp(m.e2) - ( - named_exprs, - vars, - params, - external_funcs, - ) = collect_components_from_expr(e) + (named_exprs, vars, params, external_funcs) = collect_components_from_expr(e) self.assertEqual(len(named_exprs), 2) named_exprs = ComponentSet(named_exprs) self.assertIn(m.e1, named_exprs) From 82e40b213a43ce2f2c64adbdece6e8d796af7bdd Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 19:47:56 -0600 Subject: [PATCH 086/104] update observer tests --- .../observer/tests/test_change_detector.py | 89 ++++++++++++++++--- 1 file changed, 78 insertions(+), 11 deletions(-) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index 19609182daf..c12398746e0 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -280,6 +280,18 @@ def test_sos(self): expected[m.c1]['add'] += 1 obs.check(expected) + detector.update() + obs.check(expected) + + m.c1.set_items([m.x[2], m.x[1], m.x[3]], [1, 2, 3]) + detector.update() + expected[m.c1]['remove'] += 1 + expected[m.c1]['add'] += 1 + for i in m.a: + expected[m.x[i]]['remove'] += 1 + expected[m.x[i]]['add'] += 1 + obs.check(expected) + for i in m.a: expected[m.x[i]]['remove'] += 1 expected[m.c1]['remove'] += 1 @@ -381,35 +393,90 @@ def test_update_config(self): m = pyo.ConcreteModel() m.x = pyo.Var() m.y = pyo.Var() - m.obj = pyo.Objective(expr=m.x**2 + m.y**2) - m.c1 = pyo.Constraint(expr=m.y >= pyo.exp(m.x)) + m.p = pyo.Param(initialize=1, mutable=True) obs = ObserverChecker() detector = ModelChangeDetector(m, [obs]) - expected = ComponentMap() + obs.check(expected) + + detector.config.check_for_new_or_removed_constraints = False + detector.config.check_for_new_or_removed_objectives = False + detector.config.update_constraints = False + detector.config.update_objectives = False + detector.config.update_vars = False + detector.config.update_parameters = False + detector.config.update_named_expressions = False + + m.e = pyo.Expression(expr=pyo.exp(m.x)) + m.obj = pyo.Objective(expr=m.x**2 + m.p*m.y**2) + m.c1 = pyo.Constraint(expr=m.y >= m.e + m.p) + + detector.update() + obs.check(expected) + + detector.config.check_for_new_or_removed_constraints = True + detector.update() expected[m.x] = make_count_dict() expected[m.y] = make_count_dict() - expected[m.obj] = make_count_dict() + expected[m.p] = make_count_dict() expected[m.c1] = make_count_dict() expected[m.x]['add'] += 1 expected[m.y]['add'] += 1 - expected[m.obj]['add'] += 1 + expected[m.p]['add'] += 1 expected[m.c1]['add'] += 1 obs.check(expected) - detector.config.check_for_new_or_removed_constraints = False - detector.config.update_constraints = False - m.c2 = pyo.Constraint(expr=m.y >= (m.x - 1) ** 2) + detector.config.check_for_new_or_removed_objectives = True detector.update() + expected[m.obj] = make_count_dict() + expected[m.obj]['add'] += 1 obs.check(expected) m.x.setlb(0) detector.update() + obs.check(expected) + + detector.config.update_vars = True + detector.update() expected[m.x]['update'] += 1 obs.check(expected) - detector.config.check_for_new_or_removed_constraints = True + m.p.value = 2 + detector.update() + obs.check(expected) + + detector.config.update_parameters = True + detector.update() + expected[m.p]['update'] += 1 + obs.check(expected) + + m.e.expr += 1 + detector.update() + obs.check(expected) + + detector.config.update_named_expressions = True + detector.update() + expected[m.c1]['remove'] += 1 + expected[m.c1]['add'] += 1 + obs.check(expected) + + m.obj.expr += 1 + detector.update() + obs.check(expected) + + detector.config.update_objectives = True + detector.update() + expected[m.obj]['remove'] += 1 + expected[m.obj]['add'] += 1 + obs.check(expected) + + m.c1 = m.y >= m.e + detector.update() + obs.check(expected) + + detector.config.update_constraints = True detector.update() - expected[m.c2] = make_count_dict() - expected[m.c2]['add'] += 1 + expected[m.c1]['remove'] += 1 + expected[m.c1]['add'] += 1 + obs.check(expected) From ae7e0311302b10974286d021dc4226d54b1da1e0 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Tue, 16 Sep 2025 19:49:07 -0600 Subject: [PATCH 087/104] run black --- pyomo/contrib/observer/tests/test_change_detector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/observer/tests/test_change_detector.py b/pyomo/contrib/observer/tests/test_change_detector.py index c12398746e0..5aea43bdd58 100644 --- a/pyomo/contrib/observer/tests/test_change_detector.py +++ b/pyomo/contrib/observer/tests/test_change_detector.py @@ -409,7 +409,7 @@ def test_update_config(self): detector.config.update_named_expressions = False m.e = pyo.Expression(expr=pyo.exp(m.x)) - m.obj = pyo.Objective(expr=m.x**2 + m.p*m.y**2) + m.obj = pyo.Objective(expr=m.x**2 + m.p * m.y**2) m.c1 = pyo.Constraint(expr=m.y >= m.e + m.p) detector.update() From 5778688d5d209bbce0062d74727e2fc060c47aa3 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Wed, 17 Sep 2025 06:20:09 -0600 Subject: [PATCH 088/104] fix docs --- pyomo/contrib/observer/model_observer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 0fd5a489b43..dddc96010c7 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -298,7 +298,7 @@ class ModelChangeDetector: >>> m = pyo.ConcreteModel() >>> obs = PrintObserver() >>> detector = ModelChangeDetector(m, [obs]) - >>> m.x = pyo.Var(bounds=()) + >>> m.x = pyo.Var() >>> m.y = pyo.Var() >>> detector.update() # no output because the variables are not used >>> m.obj = pyo.Objective(expr=m.x**2 + m.y**2) From cd1c4eff1cb8e79c7c7e9aead6f53d0fcf321772 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 18 Sep 2025 07:43:21 -0600 Subject: [PATCH 089/104] observer updates --- pyomo/contrib/observer/model_observer.py | 144 +++++++++++++++++- .../tests/test_component_collector.py | 11 ++ 2 files changed, 147 insertions(+), 8 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index dddc96010c7..32822e34104 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -73,7 +73,7 @@ def __init__( visibility=visibility, ) - # automatically detect new/removed constraints on subsequent solves + #: automatically detect new/removed constraints on subsequent solves self.check_for_new_or_removed_constraints: bool = self.declare( 'check_for_new_or_removed_constraints', ConfigValue( @@ -87,7 +87,7 @@ def __init__( model.""", ), ) - # automatically detect new/removed objectives on subsequent solves + #: automatically detect new/removed objectives on subsequent solves self.check_for_new_or_removed_objectives: bool = self.declare( 'check_for_new_or_removed_objectives', ConfigValue( @@ -101,7 +101,7 @@ def __init__( model.""", ), ) - # automatically detect changes to constraints on subsequent solves + #: automatically detect changes to constraints on subsequent solves self.update_constraints: bool = self.declare( 'update_constraints', ConfigValue( @@ -116,7 +116,7 @@ def __init__( being modified.""", ), ) - # automatically detect changes to variables on subsequent solves + #: automatically detect changes to variables on subsequent solves self.update_vars: bool = self.declare( 'update_vars', ConfigValue( @@ -132,7 +132,7 @@ def __init__( update_parameters_and_fixed_vars.""", ), ) - # automatically detect changes to parameters on subsequent solves + #: automatically detect changes to parameters on subsequent solves self.update_parameters: bool = self.declare( 'update_parameters', ConfigValue( @@ -146,7 +146,7 @@ def __init__( parameters are not being modified.""", ), ) - # automatically detect changes to named expressions on subsequent solves + #: automatically detect changes to named expressions on subsequent solves self.update_named_expressions: bool = self.declare( 'update_named_expressions', ConfigValue( @@ -159,7 +159,7 @@ def __init__( are certain Expressions are not being modified.""", ), ) - # automatically detect changes to objectives on subsequent solves + #: automatically detect changes to objectives on subsequent solves self.update_objectives: bool = self.declare( 'update_objectives', ConfigValue( @@ -181,50 +181,178 @@ def __init__(self): @abc.abstractmethod def add_variables(self, variables: List[VarData]): + """ + This method gets called by the ModelChangeDetector when new + "active" variables are detected in the model. This means variables + that are used within an active component such as a constraint or + an objective. + + Parameters + ---------- + variables: List[VarData] + The list of variables added to the model + """ pass @abc.abstractmethod def add_parameters(self, params: List[ParamData]): + """ + This method gets called by the ModelChangeDetector when new + "active" parameters are detected in the model. This means parameters + that are used within an active component such as a constraint or + an objective. + + Parameters + ---------- + params: List[ParamData] + The list of parameters added to the model + """ pass @abc.abstractmethod def add_constraints(self, cons: List[ConstraintData]): + """ + This method gets called by the ModelChangeDetector when new + active constraints are detected in the model. + + Parameters + ---------- + cons: List[ConstraintData] + The list of constraints added to the model + """ pass @abc.abstractmethod def add_sos_constraints(self, cons: List[SOSConstraintData]): + """ + This method gets called by the ModelChangeDetector when new + active SOS constraints are detected in the model. + + Parameters + ---------- + cons: List[SOSConstraintData] + The list of SOS constraints added to the model + """ pass @abc.abstractmethod def add_objectives(self, objs: List[ObjectiveData]): + """ + This method gets called by the ModelChangeDetector when new + active objectives are detected in the model. + + Parameters + ---------- + objs: List[ObjectiveData] + The list of objectives added to the model + """ pass @abc.abstractmethod def remove_objectives(self, objs: List[ObjectiveData]): + """ + This method gets called by the ModelChangeDetector when it detects + objectives that have been deactivated or removed from the model. + If the ModelChangeDetector detects changes in the underlying + expression for the objective, then ``remove_objectives`` will be + called followed by ``add_objectives``. + + Parameters + ---------- + objs: List[ObjectiveData] + The list of objectives that are no longer part of the model + """ pass @abc.abstractmethod def remove_constraints(self, cons: List[ConstraintData]): + """ + This method gets called by the ModelChangeDetector when it detects + constraints that have been deactivated or removed from the model. + If the ModelChangeDetector detects changes in the underlying + expression for the constraint, then ``remove_constraints`` will be + called followed by ``add_constraints``. + + Parameters + ---------- + cons: List[ConstraintData] + The list of constraints that are no longer part of the model + """ pass @abc.abstractmethod def remove_sos_constraints(self, cons: List[SOSConstraintData]): + """ + This method gets called by the ModelChangeDetector when it detects + SOS constraints that have been deactivated or removed from the model. + If the ModelChangeDetector detects changes in the underlying + data for the constraint, then ``remove_sos_constraints`` will be + called followed by ``add_sos_constraints``. + + Parameters + ---------- + cons: List[SOSConstraintData] + The list of SOS constraints that are no longer part of the model + """ pass @abc.abstractmethod def remove_variables(self, variables: List[VarData]): + """ + This method gets called by the ModelChangeDetector when it detects + variables that are no longer used in any active components ( + objectives or constraints). + + Parameters + ---------- + variables: List[VarData] + The list of variables that are no longer part of the model + """ pass @abc.abstractmethod def remove_parameters(self, params: List[ParamData]): + """ + This method gets called by the ModelChangeDetector when it detects + parameters that are no longer used in any active components ( + objectives or constraints). + + Parameters + ---------- + params: List[ParamData] + The list of parameters that are no longer part of the model + """ pass @abc.abstractmethod def update_variables(self, variables: List[VarData]): + """ + This method gets called by the ModelChangeDetector when it detects + variables that have been modified in some way (e.g., the bounds + change). This is only true for changes that are considered + "inputs" to the model. For example, the value of the variable is + considered an "output" (unless the variable is fixed), so changing + the value of an unfixed variable will not cause this method to be + called. + + Parameters + ---------- + variables: List[VarData] + The list of variables that have been modified + """ pass @abc.abstractmethod def update_parameters(self, params: List[ParamData]): + """ + This method gets called by the ModelChangeDetector when it detects + parameters that have been modified (i.e., the value changed). + + Parameters + ---------- + params: List[ParamData] + The list of parameters that have been modified + """ pass @@ -240,7 +368,7 @@ class ModelChangeDetector: if that variable is not used in any constraints. The Observer/ModelChangeDetector are most useful when a small number - of changes are being relative to the size of the model. For example, + of changes are being made relative to the size of the model. For example, the persistent solver interfaces can be very efficient when repeatedly solving the same model but with different values for mutable parameters. diff --git a/pyomo/contrib/observer/tests/test_component_collector.py b/pyomo/contrib/observer/tests/test_component_collector.py index ca272b06cfa..70d01a08ccf 100644 --- a/pyomo/contrib/observer/tests/test_component_collector.py +++ b/pyomo/contrib/observer/tests/test_component_collector.py @@ -1,3 +1,14 @@ +# ___________________________________________________________________________ +# +# Pyomo: Python Optimization Modeling Objects +# Copyright (c) 2008-2025 +# National Technology and Engineering Solutions of Sandia, LLC +# Under the terms of Contract DE-NA0003525 with National Technology and +# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain +# rights in this software. +# This software is distributed under the 3-clause BSD License. +# __________________________________________________________________________ + import pyomo.environ as pyo from pyomo.common import unittest from pyomo.contrib.observer.component_collector import collect_components_from_expr From 3f308935dc9e9dea431d9f287e96a753e8fb436e Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 18 Sep 2025 07:43:47 -0600 Subject: [PATCH 090/104] observer updates --- pyomo/contrib/observer/model_observer.py | 42 ++++++++++++------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 32822e34104..c5135504c48 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -182,10 +182,10 @@ def __init__(self): @abc.abstractmethod def add_variables(self, variables: List[VarData]): """ - This method gets called by the ModelChangeDetector when new + This method gets called by the ModelChangeDetector when new "active" variables are detected in the model. This means variables - that are used within an active component such as a constraint or - an objective. + that are used within an active component such as a constraint or + an objective. Parameters ---------- @@ -197,10 +197,10 @@ def add_variables(self, variables: List[VarData]): @abc.abstractmethod def add_parameters(self, params: List[ParamData]): """ - This method gets called by the ModelChangeDetector when new + This method gets called by the ModelChangeDetector when new "active" parameters are detected in the model. This means parameters - that are used within an active component such as a constraint or - an objective. + that are used within an active component such as a constraint or + an objective. Parameters ---------- @@ -212,8 +212,8 @@ def add_parameters(self, params: List[ParamData]): @abc.abstractmethod def add_constraints(self, cons: List[ConstraintData]): """ - This method gets called by the ModelChangeDetector when new - active constraints are detected in the model. + This method gets called by the ModelChangeDetector when new + active constraints are detected in the model. Parameters ---------- @@ -225,8 +225,8 @@ def add_constraints(self, cons: List[ConstraintData]): @abc.abstractmethod def add_sos_constraints(self, cons: List[SOSConstraintData]): """ - This method gets called by the ModelChangeDetector when new - active SOS constraints are detected in the model. + This method gets called by the ModelChangeDetector when new + active SOS constraints are detected in the model. Parameters ---------- @@ -238,8 +238,8 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): @abc.abstractmethod def add_objectives(self, objs: List[ObjectiveData]): """ - This method gets called by the ModelChangeDetector when new - active objectives are detected in the model. + This method gets called by the ModelChangeDetector when new + active objectives are detected in the model. Parameters ---------- @@ -253,8 +253,8 @@ def remove_objectives(self, objs: List[ObjectiveData]): """ This method gets called by the ModelChangeDetector when it detects objectives that have been deactivated or removed from the model. - If the ModelChangeDetector detects changes in the underlying - expression for the objective, then ``remove_objectives`` will be + If the ModelChangeDetector detects changes in the underlying + expression for the objective, then ``remove_objectives`` will be called followed by ``add_objectives``. Parameters @@ -269,8 +269,8 @@ def remove_constraints(self, cons: List[ConstraintData]): """ This method gets called by the ModelChangeDetector when it detects constraints that have been deactivated or removed from the model. - If the ModelChangeDetector detects changes in the underlying - expression for the constraint, then ``remove_constraints`` will be + If the ModelChangeDetector detects changes in the underlying + expression for the constraint, then ``remove_constraints`` will be called followed by ``add_constraints``. Parameters @@ -285,8 +285,8 @@ def remove_sos_constraints(self, cons: List[SOSConstraintData]): """ This method gets called by the ModelChangeDetector when it detects SOS constraints that have been deactivated or removed from the model. - If the ModelChangeDetector detects changes in the underlying - data for the constraint, then ``remove_sos_constraints`` will be + If the ModelChangeDetector detects changes in the underlying + data for the constraint, then ``remove_sos_constraints`` will be called followed by ``add_sos_constraints``. Parameters @@ -328,11 +328,11 @@ def remove_parameters(self, params: List[ParamData]): def update_variables(self, variables: List[VarData]): """ This method gets called by the ModelChangeDetector when it detects - variables that have been modified in some way (e.g., the bounds - change). This is only true for changes that are considered + variables that have been modified in some way (e.g., the bounds + change). This is only true for changes that are considered "inputs" to the model. For example, the value of the variable is considered an "output" (unless the variable is fixed), so changing - the value of an unfixed variable will not cause this method to be + the value of an unfixed variable will not cause this method to be called. Parameters From dc19b170db098ee7d515847149aacdda88a9ee0d Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 2 Oct 2025 03:55:23 -0600 Subject: [PATCH 091/104] observer: docstring updates --- pyomo/contrib/observer/model_observer.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index c5135504c48..6ee5302ddb8 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -378,7 +378,17 @@ class ModelChangeDetector: then ``check_for_new_or_removed_constraints`` can be set to ``False``, which will save some time when ``update`` is called. - Here are some examples: + We have discussed expanding the interface of the ``ModelChangeDetector`` + with methods to request extra information. For example, if the value + of a fixed variable changes, an observer may want to know all of the + constraints that use the variables. This class alredy has that + information, so the observer should not have to waste time recomputing + that. We have not yet added methods like this because we do not have + an immediate use case or need, and it's not yet clear waht those + methods should look like. If a need arises, please create an issue or + pull request. + + Here are some usage examples: >>> import pyomo.environ as pyo >>> from pyomo.contrib.observer.model_observer import ( @@ -718,7 +728,7 @@ def _check_for_unknown_active_components(self): ): raise NotImplementedError( f'ModelChangeDetector does not know how to ' - 'handle compents with ctype {ctype}' + 'handle components with ctype {ctype}' ) def _set_instance(self): From bb609590b75a63e8befaa1da91dca8f5b2d9df21 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 2 Oct 2025 04:05:11 -0600 Subject: [PATCH 092/104] observer: typos --- pyomo/contrib/observer/model_observer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 6ee5302ddb8..d72b3e06ca3 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -381,10 +381,10 @@ class ModelChangeDetector: We have discussed expanding the interface of the ``ModelChangeDetector`` with methods to request extra information. For example, if the value of a fixed variable changes, an observer may want to know all of the - constraints that use the variables. This class alredy has that + constraints that use the variables. This class already has that information, so the observer should not have to waste time recomputing that. We have not yet added methods like this because we do not have - an immediate use case or need, and it's not yet clear waht those + an immediate use case or need, and it's not yet clear what those methods should look like. If a need arises, please create an issue or pull request. From 576a21768c7c8acbdfb63d863c69fe603017a6c3 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Thu, 2 Oct 2025 07:22:54 -0600 Subject: [PATCH 093/104] run black --- pyomo/contrib/solver/tests/solvers/test_solvers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 192f1536800..74d3b7ccdbc 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -22,7 +22,10 @@ from pyomo.contrib.solver.common.base import SolverBase from pyomo.contrib.solver.common.config import SolverConfig from pyomo.contrib.solver.common.factory import SolverFactory -from pyomo.contrib.solver.solvers.gurobi_persistent import GurobiPersistent, GurobiDirectQuadratic +from pyomo.contrib.solver.solvers.gurobi_persistent import ( + GurobiPersistent, + GurobiDirectQuadratic, +) from pyomo.contrib.solver.solvers.gurobi.gurobi_direct import GurobiDirect from pyomo.contrib.solver.solvers.highs import Highs from pyomo.contrib.solver.solvers.ipopt import Ipopt From ce99fb2597611449874d78d02678d35fb21eaf47 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 4 Oct 2025 15:56:11 -0600 Subject: [PATCH 094/104] observer improvements --- pyomo/contrib/observer/component_collector.py | 7 +- pyomo/contrib/observer/model_observer.py | 166 +++++++++++------- .../solvers/gurobi/gurobi_persistent.py | 166 ++++++++++++------ .../tests/solvers/test_gurobi_persistent.py | 10 +- .../solver/tests/solvers/test_solvers.py | 10 +- 5 files changed, 229 insertions(+), 130 deletions(-) diff --git a/pyomo/contrib/observer/component_collector.py b/pyomo/contrib/observer/component_collector.py index d30bb128758..22e66aa6c80 100644 --- a/pyomo/contrib/observer/component_collector.py +++ b/pyomo/contrib/observer/component_collector.py @@ -33,6 +33,7 @@ from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.expression import ExpressionData, ScalarExpression from pyomo.repn.util import ExitNodeDispatcher +from pyomo.common.numeric_types import native_numeric_types def handle_var(node, collector): @@ -79,8 +80,6 @@ def handle_skip(node, collector): collector_handlers[RangedExpression] = handle_skip collector_handlers[InequalityExpression] = handle_skip collector_handlers[EqualityExpression] = handle_skip -collector_handlers[int] = handle_skip -collector_handlers[float] = handle_skip class _ComponentFromExprCollector(StreamBasedExpressionVisitor): @@ -92,6 +91,10 @@ def __init__(self, **kwds): super().__init__(**kwds) def exitNode(self, node, data): + if type(node) in native_numeric_types: + # we need this here to handle numpy + # (we can't put numpy in the dispatcher?) + return None return collector_handlers[node.__class__](node, self) def beforeChild(self, node, child, child_idx): diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index b0a5b07fdfd..71c1a7c5460 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -16,16 +16,18 @@ from pyomo.core.base.constraint import ConstraintData, Constraint from pyomo.core.base.sos import SOSConstraintData, SOSConstraint from pyomo.core.base.var import VarData -from pyomo.core.base.param import ParamData +from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.objective import ObjectiveData, Objective -from pyomo.core.base.block import BlockData +from pyomo.core.base.block import BlockData, Block from pyomo.core.base.component import ActiveComponent +from pyomo.core.base.suffix import Suffix from pyomo.common.collections import ComponentMap from pyomo.common.timing import HierarchicalTimer from pyomo.contrib.solver.common.util import get_objective from pyomo.contrib.observer.component_collector import collect_components_from_expr from pyomo.common.numeric_types import native_numeric_types import gc +import warnings """ @@ -49,6 +51,9 @@ """ +_param_types = {ParamData, ScalarParam} + + @document_configdict() class AutoUpdateConfig(ConfigDict): """ @@ -492,7 +497,7 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): observers: Sequence[Observer] The objects to notify when changes are made to the model """ - self._known_active_ctypes = {Constraint, SOSConstraint, Objective} + self._known_active_ctypes = {Constraint, SOSConstraint, Objective, Block} self._observers: List[Observer] = list(observers) self._active_constraints = {} # maps constraint to expression self._active_sos = {} @@ -520,12 +525,14 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): # dict[constraints, None], # dict[sos constraints, None], # dict[objectives, None], + # dict[var_id, None], # ) self._referenced_params = {} self._vars_referenced_by_con = {} self._vars_referenced_by_obj = {} self._params_referenced_by_con = {} + self._params_referenced_by_var = {} # for when parameters show up in variable bounds self._params_referenced_by_obj = {} self.config: AutoUpdateConfig = AutoUpdateConfig()( @@ -536,11 +543,13 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): self._set_instance() def add_variables(self, variables: List[VarData]): + params_to_check = {} for v in variables: - if id(v) in self._referenced_variables: + vid = id(v) + if vid in self._referenced_variables: raise ValueError(f'Variable {v.name} has already been added') - self._referenced_variables[id(v)] = ({}, {}, {}) - self._vars[id(v)] = ( + self._referenced_variables[vid] = ({}, {}, {}) + self._vars[vid] = ( v, v._lb, v._ub, @@ -548,6 +557,31 @@ def add_variables(self, variables: List[VarData]): v.domain.get_interval(), v.value, ) + ref_params = set() + for bnd in (v._lb, v._ub): + if bnd is None or type(bnd) in native_numeric_types: + continue + (named_exprs, _vars, parameters, external_functions) = ( + collect_components_from_expr(bnd) + ) + if _vars: + raise NotImplementedError('ModelChangeDetector does not support variables in the bounds of other variables') + if named_exprs: + raise NotImplementedError('ModelChangeDetector does not support Expressions in the bounds of other variables') + if external_functions: + raise NotImplementedError('ModelChangeDetector does not support external functions in the bounds of other variables') + params_to_check.update((id(p), p) for p in parameters) + if vid not in self._params_referenced_by_var: + self._params_referenced_by_var[vid] = [] + self._params_referenced_by_var[vid].extend(p for p in parameters if id(p) not in ref_params) + ref_params.update(id(p) for p in parameters) + self._check_for_new_params(list(params_to_check.values())) + for v in variables: + if id(v) not in self._params_referenced_by_var: + continue + parameters = self._params_referenced_by_var[id(v)] + for p in parameters: + self._referenced_params[id(p)][3][id(v)] = None for obs in self._observers: obs.add_variables(variables) @@ -556,46 +590,46 @@ def add_parameters(self, params: List[ParamData]): pid = id(p) if pid in self._referenced_params: raise ValueError(f'Parameter {p.name} has already been added') - self._referenced_params[pid] = ({}, {}, {}) + self._referenced_params[pid] = ({}, {}, {}, {}) self._params[id(p)] = (p, p.value) for obs in self._observers: obs.add_parameters(params) def _check_for_new_vars(self, variables: List[VarData]): - new_vars = [] + new_vars = {} for v in variables: if id(v) not in self._referenced_variables: - new_vars.append(v) - self.add_variables(new_vars) + new_vars[id(v)] = v + self.add_variables(list(new_vars.values())) def _check_to_remove_vars(self, variables: List[VarData]): - vars_to_remove = [] + vars_to_remove = {} for v in variables: v_id = id(v) ref_cons, ref_sos, ref_obj = self._referenced_variables[v_id] if not ref_cons and not ref_sos and not ref_obj: - vars_to_remove.append(v) - self._remove_variables(vars_to_remove) + vars_to_remove[v_id] = v + self.remove_variables(list(vars_to_remove.values())) def _check_for_new_params(self, params: List[ParamData]): - new_params = [] + new_params = {} for p in params: if id(p) not in self._referenced_params: - new_params.append(p) - self.add_parameters(new_params) + new_params[id(p)] = p + self.add_parameters(list(new_params.values())) def _check_to_remove_params(self, params: List[ParamData]): - params_to_remove = [] + params_to_remove = {} for p in params: p_id = id(p) - ref_cons, ref_sos, ref_obj = self._referenced_params[p_id] - if not ref_cons and not ref_sos and not ref_obj: - params_to_remove.append(p) - self._remove_parameters(params_to_remove) + ref_cons, ref_sos, ref_obj, ref_vars = self._referenced_params[p_id] + if not ref_cons and not ref_sos and not ref_obj and not ref_vars: + params_to_remove[p_id] = p + self.remove_parameters(list(params_to_remove.values())) def add_constraints(self, cons: List[ConstraintData]): - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for con in cons: if con in self._active_constraints: raise ValueError(f'Constraint {con.name} has already been added') @@ -603,16 +637,16 @@ def add_constraints(self, cons: List[ConstraintData]): (named_exprs, variables, parameters, external_functions) = ( collect_components_from_expr(con.expr) ) - vars_to_check.extend(variables) - params_to_check.extend(parameters) + vars_to_check.update((id(v), v) for v in variables) + params_to_check.update((id(p), p) for p in parameters) if named_exprs: self._named_expressions[con] = [(e, e.expr) for e in named_exprs] if external_functions: self._external_functions[con] = external_functions self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = parameters - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) + self._check_for_new_vars(list(vars_to_check.values())) + self._check_for_new_params(list(params_to_check.values())) for con in cons: variables = self._vars_referenced_by_con[con] parameters = self._params_referenced_by_con[con] @@ -624,8 +658,8 @@ def add_constraints(self, cons: List[ConstraintData]): obs.add_constraints(cons) def add_sos_constraints(self, cons: List[SOSConstraintData]): - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for con in cons: if con in self._active_sos: raise ValueError(f'Constraint {con.name} has already been added') @@ -642,12 +676,12 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): continue if p.is_parameter_type(): params.append(p) - vars_to_check.extend(variables) - params_to_check.extend(params) + vars_to_check.update((id(v), v) for v in variables) + params_to_check.update((id(p), p) for p in params) self._vars_referenced_by_con[con] = variables self._params_referenced_by_con[con] = params - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) + self._check_for_new_vars(list(vars_to_check.values())) + self._check_for_new_params(list(params_to_check.values())) for con in cons: variables = self._vars_referenced_by_con[con] params = self._params_referenced_by_con[con] @@ -659,24 +693,24 @@ def add_sos_constraints(self, cons: List[SOSConstraintData]): obs.add_sos_constraints(cons) def add_objectives(self, objs: List[ObjectiveData]): - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for obj in objs: obj_id = id(obj) self._objectives[obj_id] = (obj, obj.expr, obj.sense) (named_exprs, variables, parameters, external_functions) = ( collect_components_from_expr(obj.expr) ) - vars_to_check.extend(variables) - params_to_check.extend(parameters) + vars_to_check.update((id(v), v) for v in variables) + params_to_check.update((id(p), p) for p in parameters) if named_exprs: self._obj_named_expressions[obj_id] = [(e, e.expr) for e in named_exprs] if external_functions: self._external_functions[obj] = external_functions self._vars_referenced_by_obj[obj_id] = variables self._params_referenced_by_obj[obj_id] = parameters - self._check_for_new_vars(vars_to_check) - self._check_for_new_params(params_to_check) + self._check_for_new_vars(list(vars_to_check.values())) + self._check_for_new_params(list(params_to_check.values())) for obj in objs: obj_id = id(obj) variables = self._vars_referenced_by_obj[obj_id] @@ -692,8 +726,8 @@ def remove_objectives(self, objs: List[ObjectiveData]): for obs in self._observers: obs.remove_objectives(objs) - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for obj in objs: obj_id = id(obj) if obj_id not in self._objectives: @@ -704,15 +738,15 @@ def remove_objectives(self, objs: List[ObjectiveData]): self._referenced_variables[id(v)][2].pop(obj_id) for p in self._params_referenced_by_obj[obj_id]: self._referenced_params[id(p)][2].pop(obj_id) - vars_to_check.extend(self._vars_referenced_by_obj[obj_id]) - params_to_check.extend(self._params_referenced_by_obj[obj_id]) + vars_to_check.update((id(v), v) for v in self._vars_referenced_by_obj[obj_id]) + params_to_check.update((id(p), p) for p in self._params_referenced_by_obj[obj_id]) del self._objectives[obj_id] self._obj_named_expressions.pop(obj_id, None) self._external_functions.pop(obj, None) del self._vars_referenced_by_obj[obj_id] del self._params_referenced_by_obj[obj_id] - self._check_to_remove_vars(vars_to_check) - self._check_to_remove_params(params_to_check) + self._check_to_remove_vars(list(vars_to_check.values())) + self._check_to_remove_params(list(params_to_check.values())) def _check_for_unknown_active_components(self): for ctype in self._model.collect_ctypes(): @@ -723,9 +757,12 @@ def _check_for_unknown_active_components(self): for comp in self._model.component_data_objects( ctype, active=True, descend_into=True ): + if isinstance(comp, Suffix): + warnings.warn('ModelChangeDetector does not detect changes to suffixes') + continue raise NotImplementedError( - f'ModelChangeDetector does not know how to ' - 'handle components with ctype {ctype}' + 'ModelChangeDetector does not know how to ' + f'handle components with ctype {ctype}' ) def _set_instance(self): @@ -764,8 +801,8 @@ def _set_instance(self): def remove_constraints(self, cons: List[ConstraintData]): for obs in self._observers: obs.remove_constraints(cons) - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for con in cons: if con not in self._active_constraints: raise ValueError( @@ -775,21 +812,21 @@ def remove_constraints(self, cons: List[ConstraintData]): self._referenced_variables[id(v)][0].pop(con) for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][0].pop(con) - vars_to_check.extend(self._vars_referenced_by_con[con]) - params_to_check.extend(self._params_referenced_by_con[con]) + vars_to_check.update((id(v), v) for v in self._vars_referenced_by_con[con]) + params_to_check.update((id(p), p) for p in self._params_referenced_by_con[con]) del self._active_constraints[con] self._named_expressions.pop(con, None) self._external_functions.pop(con, None) del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] - self._check_to_remove_vars(vars_to_check) - self._check_to_remove_params(params_to_check) + self._check_to_remove_vars(list(vars_to_check.values())) + self._check_to_remove_params(list(params_to_check.values())) def remove_sos_constraints(self, cons: List[SOSConstraintData]): for obs in self._observers: obs.remove_sos_constraints(cons) - vars_to_check = [] - params_to_check = [] + vars_to_check = {} + params_to_check = {} for con in cons: if con not in self._active_sos: raise ValueError( @@ -799,23 +836,29 @@ def remove_sos_constraints(self, cons: List[SOSConstraintData]): self._referenced_variables[id(v)][1].pop(con) for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][1].pop(con) - vars_to_check.extend(self._vars_referenced_by_con[con]) - params_to_check.extend(self._params_referenced_by_con[con]) + vars_to_check.update((id(v), v) for v in self._vars_referenced_by_con[con]) + params_to_check.update((id(p), p) for p in self._params_referenced_by_con[con]) del self._active_sos[con] del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] - self._check_to_remove_vars(vars_to_check) - self._check_to_remove_params(params_to_check) + self._check_to_remove_vars(list(vars_to_check.values())) + self._check_to_remove_params(list(params_to_check.values())) def remove_variables(self, variables: List[VarData]): for obs in self._observers: obs.remove_variables(variables) + params_to_check = {} for v in variables: v_id = id(v) if v_id not in self._referenced_variables: raise ValueError( f'Cannot remove variable {v.name} - it has not been added' ) + if v_id in self._params_referenced_by_var: + for p in self._params_referenced_by_var[v_id]: + self._referenced_params[id(p)][3].pop(v_id) + params_to_check.update((id(p), p) for p in self._params_referenced_by_var[v_id]) + self._params_referenced_by_var.pop(v_id) cons_using, sos_using, obj_using = self._referenced_variables[v_id] if cons_using or sos_using or obj_using: raise ValueError( @@ -823,6 +866,7 @@ def remove_variables(self, variables: List[VarData]): ) del self._referenced_variables[v_id] del self._vars[v_id] + self._check_to_remove_params(list(params_to_check.values())) def remove_parameters(self, params: List[ParamData]): for obs in self._observers: @@ -833,8 +877,8 @@ def remove_parameters(self, params: List[ParamData]): raise ValueError( f'Cannot remove parameter {p.name} - it has not been added' ) - cons_using, sos_using, obj_using = self._referenced_params[p_id] - if cons_using or sos_using or obj_using: + cons_using, sos_using, obj_using, vars_using = self._referenced_params[p_id] + if cons_using or sos_using or obj_using or vars_using: raise ValueError( f'Cannot remove parameter {p.name} - it is still being used by constraints/objectives' ) diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 603e8e21800..74952af2b49 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -15,6 +15,7 @@ from collections.abc import Iterable from pyomo.common.collections import ComponentSet, OrderedSet +from pyomo.common.errors import PyomoException from pyomo.common.shutdown import python_is_shutting_down from pyomo.common.timing import HierarchicalTimer from pyomo.core.base.objective import ObjectiveData @@ -33,13 +34,14 @@ from .gurobi_direct_base import ( GurobiDirectBase, gurobipy, + GurobiConfig, _load_vars, _get_primals, _get_duals, _get_reduced_costs, ) from pyomo.contrib.solver.common.util import get_objective -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector +from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig logger = logging.getLogger(__name__) @@ -539,7 +541,7 @@ def _set_objective(self, obj): class _GurobiObserver(Observer): - def __init__(self, opt: GurobiPersistentQuadratic) -> None: + def __init__(self, opt: GurobiPersistent) -> None: self.opt = opt def add_variables(self, variables: List[VarData]): @@ -554,8 +556,11 @@ def add_constraints(self, cons: List[ConstraintData]): def add_sos_constraints(self, cons: List[SOSConstraintData]): self.opt._add_sos_constraints(cons) - def set_objective(self, obj: ObjectiveData | None): - self.opt._set_objective(obj) + def add_objectives(self, objs: List[ObjectiveData]): + self.opt._add_objectives(objs) + + def remove_objectives(self, objs: List[ObjectiveData]): + self.opt._remove_objectives(objs) def remove_constraints(self, cons: List[ConstraintData]): self.opt._remove_constraints(cons) @@ -576,8 +581,31 @@ def update_parameters(self, params: List[ParamData]): self.opt._update_parameters(params) +class GurobiPersistentConfig(GurobiConfig): + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + GurobiConfig.__init__( + self, + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + self.auto_updates: bool = self.declare( + 'auto_updates', AutoUpdateConfig() + ) + + class GurobiPersistent(GurobiDirectQuadratic, PersistentSolverBase): _minimum_version = (7, 0, 0) + CONFIG = GurobiPersistentConfig() def __init__(self, **kwds): super().__init__(**kwds) @@ -592,15 +620,11 @@ def __init__(self, **kwds): self._constraints_added_since_update = OrderedSet() self._vars_added_since_update = ComponentSet() self._last_results_object: Optional[Results] = None - self._observer = _GurobiObserver(self) - self._change_detector = ModelChangeDetector(observers=[self._observer]) + self._observer = None + self._change_detector = None self._constraint_ndx = 0 self._should_update_parameters = False - @property - def auto_updates(self): - return self._change_detector.config - def _clear(self): super()._clear() self._pyomo_model = None @@ -669,8 +693,10 @@ def set_instance(self, pyomo_model): self._clear() self._pyomo_model = pyomo_model self._solver_model = gurobipy.Model(env=self.env()) + self._observer = _GurobiObserver(self) timer.start('set_instance') - self._change_detector.set_instance(pyomo_model) + self._change_detector = ModelChangeDetector(model=self._pyomo_model, observers=[self._observer], **dict(self.config.auto_updates)) + self._change_detector.config = self.config.auto_updates timer.stop('set_instance') def update(self): @@ -804,59 +830,83 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): self._constraints_added_since_update.update(cons) self._needs_updated = True - def _set_objective(self, obj): - self._invalidate_last_results() - if obj is None: - sense = gurobipy.GRB.MINIMIZE - gurobi_expr = 0 - repn_constant = 0 - self._mutable_objective = None - else: - if obj.sense == minimize: - sense = gurobipy.GRB.MINIMIZE - elif obj.sense == maximize: - sense = gurobipy.GRB.MAXIMIZE + def _remove_objectives(self, objs: List[ObjectiveData]): + for obj in objs: + if obj is not self._objective: + raise RuntimeError( + 'tried to remove an objective that has not been added: ' \ + f'{str(obj)}' + ) else: - raise ValueError(f'Objective sense is not recognized: {obj.sense}') + self._invalidate_last_results() + self._solver_model.setObjective(0, sense=gurobipy.GRB.MINIMIZE) + # see PR #2454 + self._solver_model.update() + self._objective = None + self._needs_updated = False + + def _add_objectives(self, objs: List[ObjectiveData]): + if len(objs) > 1: + raise NotImplementedError( + 'the persistent interface to gurobi currently ' \ + f'only supports single-objective problems; got {len(objs)}: ' + f'{[str(i) for i in objs]}' + ) + + if len(objs) == 0: + return + + obj = objs[0] - repn = generate_standard_repn( - obj.expr, quadratic=True, compute_values=False + if self._objective is not None: + raise NotImplementedError( + 'the persistent interface to gurobi currently ' \ + 'only supports single-objective problems; tried to add ' \ + f'an objective ({str(obj)}), but there is already an ' \ + f'active objective ({str(self._objective)})' ) - repn_constant = value(repn.constant) - gurobi_expr = self._get_expr_from_pyomo_repn(repn) - mutable_constant = _MutableConstant(repn.constant, None, None) + self._invalidate_last_results() - mlc_list = [] - for c, v in zip(repn.linear_coefs, repn.linear_vars): - if not is_constant(c): - mlc = _MutableLinearCoefficient( - c, - None, - None, - id(v), - self._pyomo_var_to_solver_var_map, - self._solver_model, - ) - mlc_list.append(mlc) + if obj.sense == minimize: + sense = gurobipy.GRB.MINIMIZE + elif obj.sense == maximize: + sense = gurobipy.GRB.MAXIMIZE + else: + raise ValueError(f'Objective sense is not recognized: {obj.sense}') - mqc_list = [] - for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): - if not is_constant(coef): - mqc = _MutableQuadraticCoefficient( - coef, id(x), id(y), self._pyomo_var_to_solver_var_map - ) - mqc_list.append(mqc) + repn = generate_standard_repn( + obj.expr, quadratic=True, compute_values=False + ) + repn_constant = value(repn.constant) + gurobi_expr = self._get_expr_from_pyomo_repn(repn) - self._mutable_objective = _MutableObjective( - self._solver_model, mutable_constant, mlc_list, mqc_list - ) + mutable_constant = _MutableConstant(repn.constant, None, None) + + mlc_list = [] + for c, v in zip(repn.linear_coefs, repn.linear_vars): + if not is_constant(c): + mlc = _MutableLinearCoefficient( + c, + None, + None, + id(v), + self._pyomo_var_to_solver_var_map, + self._solver_model, + ) + mlc_list.append(mlc) - # hack - # see PR #2454 - if self._objective is not None: - self._solver_model.setObjective(0) - self._solver_model.update() + mqc_list = [] + for coef, (x, y) in zip(repn.quadratic_coefs, repn.quadratic_vars): + if not is_constant(coef): + mqc = _MutableQuadraticCoefficient( + coef, id(x), id(y), self._pyomo_var_to_solver_var_map + ) + mqc_list.append(mqc) + + self._mutable_objective = _MutableObjective( + self._solver_model, mutable_constant, mlc_list, mqc_list + ) self._solver_model.setObjective(gurobi_expr + repn_constant, sense=sense) self._objective = obj @@ -1366,8 +1416,8 @@ def add_constraints(self, cons): def add_sos_constraints(self, cons): self._change_detector.add_sos_constraints(cons) - def set_objective(self, obj): - self._change_detector.set_objective(obj) + def set_objective(self, obj: ObjectiveData): + self._change_detector.add_objectives([obj]) def remove_constraints(self, cons): self._change_detector.remove_constraints(cons) diff --git a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py index 96cd1498956..24b53a19f2b 100644 --- a/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py +++ b/pyomo/contrib/solver/tests/solvers/test_gurobi_persistent.py @@ -499,11 +499,11 @@ def test_zero_time_limit(self): class TestManualMode(unittest.TestCase): def setUp(self): opt = GurobiPersistent() - opt.auto_updates.check_for_new_or_removed_constraints = False - opt.auto_updates.update_parameters = False - opt.auto_updates.update_vars = False - opt.auto_updates.update_constraints = False - opt.auto_updates.update_named_expressions = False + opt.config.auto_updates.check_for_new_or_removed_constraints = False + opt.config.auto_updates.update_parameters = False + opt.config.auto_updates.update_vars = False + opt.config.auto_updates.update_constraints = False + opt.config.auto_updates.update_named_expressions = False self.opt = opt def test_basics(self): diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 74d3b7ccdbc..e7ff00f7f41 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -22,7 +22,7 @@ from pyomo.contrib.solver.common.base import SolverBase from pyomo.contrib.solver.common.config import SolverConfig from pyomo.contrib.solver.common.factory import SolverFactory -from pyomo.contrib.solver.solvers.gurobi_persistent import ( +from pyomo.contrib.solver.solvers.gurobi.gurobi_persistent import ( GurobiPersistent, GurobiDirectQuadratic, ) @@ -1537,9 +1537,7 @@ def test_add_and_remove_vars( opt.config.auto_updates.update_vars = False opt.config.auto_updates.update_constraints = False opt.config.auto_updates.update_named_expressions = False - opt.config.auto_updates.check_for_new_or_removed_params = False opt.config.auto_updates.check_for_new_or_removed_constraints = False - opt.config.auto_updates.check_for_new_or_removed_vars = False opt.config.load_solutions = False res = opt.solve(m) self.assertEqual(res.solution_status, SolutionStatus.optimal) @@ -1864,7 +1862,11 @@ def test_objective_changes( res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 3) if opt.is_persistent(): - opt.config.auto_updates.check_for_new_objective = False + # hack until we get everything ported to the observer + try: + opt.config.auto_updates.check_for_new_or_removed_objectives = False + except: + opt.config.auto_updates.check_for_new_objective = False m.e.expr = 4 res = opt.solve(m) self.assertAlmostEqual(res.incumbent_objective, 4) From 066e4fdf80f401194d63355495ba43f10da073e8 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sat, 4 Oct 2025 15:56:58 -0600 Subject: [PATCH 095/104] run black --- pyomo/contrib/observer/model_observer.py | 44 ++++++++++++++----- .../solvers/gurobi/gurobi_persistent.py | 30 +++++++------ 2 files changed, 50 insertions(+), 24 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 71c1a7c5460..2da340aab4f 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -532,7 +532,9 @@ def __init__(self, model: BlockData, observers: Sequence[Observer], **kwds): self._vars_referenced_by_con = {} self._vars_referenced_by_obj = {} self._params_referenced_by_con = {} - self._params_referenced_by_var = {} # for when parameters show up in variable bounds + self._params_referenced_by_var = ( + {} + ) # for when parameters show up in variable bounds self._params_referenced_by_obj = {} self.config: AutoUpdateConfig = AutoUpdateConfig()( @@ -565,15 +567,23 @@ def add_variables(self, variables: List[VarData]): collect_components_from_expr(bnd) ) if _vars: - raise NotImplementedError('ModelChangeDetector does not support variables in the bounds of other variables') + raise NotImplementedError( + 'ModelChangeDetector does not support variables in the bounds of other variables' + ) if named_exprs: - raise NotImplementedError('ModelChangeDetector does not support Expressions in the bounds of other variables') + raise NotImplementedError( + 'ModelChangeDetector does not support Expressions in the bounds of other variables' + ) if external_functions: - raise NotImplementedError('ModelChangeDetector does not support external functions in the bounds of other variables') + raise NotImplementedError( + 'ModelChangeDetector does not support external functions in the bounds of other variables' + ) params_to_check.update((id(p), p) for p in parameters) if vid not in self._params_referenced_by_var: self._params_referenced_by_var[vid] = [] - self._params_referenced_by_var[vid].extend(p for p in parameters if id(p) not in ref_params) + self._params_referenced_by_var[vid].extend( + p for p in parameters if id(p) not in ref_params + ) ref_params.update(id(p) for p in parameters) self._check_for_new_params(list(params_to_check.values())) for v in variables: @@ -738,8 +748,12 @@ def remove_objectives(self, objs: List[ObjectiveData]): self._referenced_variables[id(v)][2].pop(obj_id) for p in self._params_referenced_by_obj[obj_id]: self._referenced_params[id(p)][2].pop(obj_id) - vars_to_check.update((id(v), v) for v in self._vars_referenced_by_obj[obj_id]) - params_to_check.update((id(p), p) for p in self._params_referenced_by_obj[obj_id]) + vars_to_check.update( + (id(v), v) for v in self._vars_referenced_by_obj[obj_id] + ) + params_to_check.update( + (id(p), p) for p in self._params_referenced_by_obj[obj_id] + ) del self._objectives[obj_id] self._obj_named_expressions.pop(obj_id, None) self._external_functions.pop(obj, None) @@ -758,7 +772,9 @@ def _check_for_unknown_active_components(self): ctype, active=True, descend_into=True ): if isinstance(comp, Suffix): - warnings.warn('ModelChangeDetector does not detect changes to suffixes') + warnings.warn( + 'ModelChangeDetector does not detect changes to suffixes' + ) continue raise NotImplementedError( 'ModelChangeDetector does not know how to ' @@ -813,7 +829,9 @@ def remove_constraints(self, cons: List[ConstraintData]): for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][0].pop(con) vars_to_check.update((id(v), v) for v in self._vars_referenced_by_con[con]) - params_to_check.update((id(p), p) for p in self._params_referenced_by_con[con]) + params_to_check.update( + (id(p), p) for p in self._params_referenced_by_con[con] + ) del self._active_constraints[con] self._named_expressions.pop(con, None) self._external_functions.pop(con, None) @@ -837,7 +855,9 @@ def remove_sos_constraints(self, cons: List[SOSConstraintData]): for p in self._params_referenced_by_con[con]: self._referenced_params[id(p)][1].pop(con) vars_to_check.update((id(v), v) for v in self._vars_referenced_by_con[con]) - params_to_check.update((id(p), p) for p in self._params_referenced_by_con[con]) + params_to_check.update( + (id(p), p) for p in self._params_referenced_by_con[con] + ) del self._active_sos[con] del self._vars_referenced_by_con[con] del self._params_referenced_by_con[con] @@ -857,7 +877,9 @@ def remove_variables(self, variables: List[VarData]): if v_id in self._params_referenced_by_var: for p in self._params_referenced_by_var[v_id]: self._referenced_params[id(p)][3].pop(v_id) - params_to_check.update((id(p), p) for p in self._params_referenced_by_var[v_id]) + params_to_check.update( + (id(p), p) for p in self._params_referenced_by_var[v_id] + ) self._params_referenced_by_var.pop(v_id) cons_using, sos_using, obj_using = self._referenced_variables[v_id] if cons_using or sos_using or obj_using: diff --git a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py index 74952af2b49..315d8c6dc4a 100644 --- a/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py +++ b/pyomo/contrib/solver/solvers/gurobi/gurobi_persistent.py @@ -41,7 +41,11 @@ _get_reduced_costs, ) from pyomo.contrib.solver.common.util import get_objective -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig +from pyomo.contrib.observer.model_observer import ( + Observer, + ModelChangeDetector, + AutoUpdateConfig, +) logger = logging.getLogger(__name__) @@ -598,9 +602,7 @@ def __init__( implicit_domain=implicit_domain, visibility=visibility, ) - self.auto_updates: bool = self.declare( - 'auto_updates', AutoUpdateConfig() - ) + self.auto_updates: bool = self.declare('auto_updates', AutoUpdateConfig()) class GurobiPersistent(GurobiDirectQuadratic, PersistentSolverBase): @@ -695,7 +697,11 @@ def set_instance(self, pyomo_model): self._solver_model = gurobipy.Model(env=self.env()) self._observer = _GurobiObserver(self) timer.start('set_instance') - self._change_detector = ModelChangeDetector(model=self._pyomo_model, observers=[self._observer], **dict(self.config.auto_updates)) + self._change_detector = ModelChangeDetector( + model=self._pyomo_model, + observers=[self._observer], + **dict(self.config.auto_updates), + ) self._change_detector.config = self.config.auto_updates timer.stop('set_instance') @@ -834,7 +840,7 @@ def _remove_objectives(self, objs: List[ObjectiveData]): for obj in objs: if obj is not self._objective: raise RuntimeError( - 'tried to remove an objective that has not been added: ' \ + 'tried to remove an objective that has not been added: ' f'{str(obj)}' ) else: @@ -848,7 +854,7 @@ def _remove_objectives(self, objs: List[ObjectiveData]): def _add_objectives(self, objs: List[ObjectiveData]): if len(objs) > 1: raise NotImplementedError( - 'the persistent interface to gurobi currently ' \ + 'the persistent interface to gurobi currently ' f'only supports single-objective problems; got {len(objs)}: ' f'{[str(i) for i in objs]}' ) @@ -860,9 +866,9 @@ def _add_objectives(self, objs: List[ObjectiveData]): if self._objective is not None: raise NotImplementedError( - 'the persistent interface to gurobi currently ' \ - 'only supports single-objective problems; tried to add ' \ - f'an objective ({str(obj)}), but there is already an ' \ + 'the persistent interface to gurobi currently ' + 'only supports single-objective problems; tried to add ' + f'an objective ({str(obj)}), but there is already an ' f'active objective ({str(self._objective)})' ) @@ -875,9 +881,7 @@ def _add_objectives(self, objs: List[ObjectiveData]): else: raise ValueError(f'Objective sense is not recognized: {obj.sense}') - repn = generate_standard_repn( - obj.expr, quadratic=True, compute_values=False - ) + repn = generate_standard_repn(obj.expr, quadratic=True, compute_values=False) repn_constant = value(repn.constant) gurobi_expr = self._get_expr_from_pyomo_repn(repn) From e6331dfa67f60224a075f0fb1a8c4bc57baf43df Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 5 Oct 2025 21:28:15 -0600 Subject: [PATCH 096/104] persistent interface to scip --- .../solver/solvers/scip/scip_direct.py | 223 ++++++++++++++++-- 1 file changed, 204 insertions(+), 19 deletions(-) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 1032affd597..cf0c71606f8 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -25,6 +25,7 @@ from pyomo.core.base.var import VarData, ScalarVar from pyomo.core.base.param import ParamData, ScalarParam from pyomo.core.base.constraint import Constraint, ConstraintData +from pyomo.core.base.objective import ObjectiveData from pyomo.core.base.sos import SOSConstraint, SOSConstraintData from pyomo.core.kernel.objective import minimize, maximize from pyomo.core.expr.numeric_expr import ( @@ -72,7 +73,7 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.core.base.units_container import _PyomoUnit from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector +from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig logger = logging.getLogger(__name__) @@ -420,7 +421,6 @@ def load_import_suffixes(self, solution_id=None): super().load_import_suffixes(solution_id) - class ScipDirect(SolverBase): _available = None @@ -822,7 +822,27 @@ def _mipstart(self): self._solver_model.addSol(sol) -class _SCIPObserver(Observer): +class ScipPersistentConfig(ScipConfig): + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + ScipConfig.__init__( + self, + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + self.auto_updates: bool = self.declare('auto_updates', AutoUpdateConfig()) + + +class _ScipObserver(Observer): def __init__(self, opt: ScipPersistent) -> None: self.opt = opt @@ -838,8 +858,11 @@ def add_constraints(self, cons: List[ConstraintData]): def add_sos_constraints(self, cons: List[SOSConstraintData]): self.opt._add_sos_constraints(cons) - def set_objective(self, obj: ObjectiveData | None): - self.opt._set_objective(obj) + def add_objectives(self, objs: List[ObjectiveData]): + self.opt._add_objectives(objs) + + def remove_objectives(self, objs: List[ObjectiveData]): + self.opt._remove_objectives(objs) def remove_constraints(self, cons: List[ConstraintData]): self.opt._remove_constraints(cons) @@ -862,39 +885,201 @@ def update_parameters(self, params: List[ParamData]): class ScipPersistent(ScipDirect, PersistentSolverBase): _minimum_version = (5, 5, 0) # this is probably conservative - - CONFIG = ScipConfig() + CONFIG = ScipPersistentConfig() def __init__(self, **kwds): super().__init__(**kwds) self._pyomo_model = None - self._objective = None - self._observer = _SCIPObserver(self) - self._change_detector = ModelChangeDetector(observers=[self._observer]) - - @property - def auto_updates(self): - return self._change_detector.config + self._observer = None + self._change_detector = None + self._last_results_object: Optional[Results] = None def _clear(self): super()._clear() self._pyomo_model = None self._objective = None + self._observer = None + self._change_detector = None - def _create_solver_model(self, model): - if model is self._pyomo_model: + def _create_solver_model(self, pyomo_model): + if pyomo_model is self._pyomo_model: self.update() else: - self.set_instance(model=model) + self.set_instance(pyomo_model=pyomo_model) solution_loader = ScipPersistentSolutionLoader( solver_model=self._solver_model, var_id_map=self._vars, var_map=self._pyomo_var_to_solver_var_map, con_map=self._pyomo_con_to_solver_con_map, - pyomo_model=model, + pyomo_model=pyomo_model, opt=self, ) has_obj = self._objective is not None - return self._solver_model, solution_loader, has_obj \ No newline at end of file + return self._solver_model, solution_loader, has_obj + + def solve(self, model, **kwds) -> Results: + res = super().solve(model, **kwds) + return res + + def update(self): + if self.config.timer is None: + timer = HierarchicalTimer() + else: + timer = self.config.timer + if self._pyomo_model is None: + raise RuntimeError('must call set_instance or solve before update') + timer.start('update') + self._change_detector.update(timer=timer) + timer.stop('update') + + def set_instance(self, pyomo_model): + if self.config.timer is None: + timer = HierarchicalTimer() + else: + timer = self.config.timer + self._clear() + self._pyomo_model = pyomo_model + self._solver_model = scip.Model() + self._observer = _ScipObserver(self) + timer.start('set_instance') + self._change_detector = ModelChangeDetector( + model=self._pyomo_model, + observers=[self._observer], + **dict(self.config.auto_updates), + ) + self._change_detector.config = self.config.auto_updates + timer.stop('set_instance') + + def _invalidate_last_results(self): + if self._last_results_object is not None: + self._last_results_object.solution_loader.invalidate() + + def _add_variables(self, variables: List[VarData]): + self._invalidate_last_results() + for v in variables: + self._add_var(v) + + def _add_constraints(self, cons: List[ConstraintData]): + self._invalidate_last_results() + super()._add_constraints(cons) + + def _add_sos_constraints(self, cons: List[SOSConstraintData]): + self._invalidate_last_results() + return super()._add_sos_constraints(cons) + + def _add_objectives(self, objs: List[ObjectiveData]): + if len(objs) > 1: + raise NotImplementedError( + 'the persistent interface to gurobi currently ' + f'only supports single-objective problems; got {len(objs)}: ' + f'{[str(i) for i in objs]}' + ) + + if len(objs) == 0: + return + + obj = objs[0] + + if self._objective is not None: + raise NotImplementedError( + 'the persistent interface to gurobi currently ' + 'only supports single-objective problems; tried to add ' + f'an objective ({str(obj)}), but there is already an ' + f'active objective ({str(self._objective)})' + ) + + self._invalidate_last_results() + self._set_objective(obj) + + def _remove_objectives(self, objs: List[ObjectiveData]): + for obj in objs: + if obj is not self._objective: + raise RuntimeError( + 'tried to remove an objective that has not been added: ' + f'{str(obj)}' + ) + else: + self._invalidate_last_results() + self._set_objective(None) + + def _remove_constraints(self, cons: List[ConstraintData]): + for con in cons: + scip_con = self._pyomo_con_to_solver_con_map.pop(con) + self._solver_model.delCons(scip_con) + + def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + for con in cons: + scip_con = self._pyomo_con_to_solver_con_map.pop(con) + self._solver_model.delCons(scip_con) + + def _remove_variables(self, variables: List[VarData]): + for v in variables: + vid = id(v) + scip_var = self._pyomo_var_to_solver_var_map.pop(vid) + self._solver_model.delVar(scip_var) + self._vars.pop(vid) + + def _update_variables(self, variables: List[VarData]): + for v in variables: + vid = id(v) + scip_var = self._pyomo_var_to_solver_var_map[vid] + vtype = self._scip_vtype_from_var(v) + lb, ub = self._scip_lb_ub_from_var(v) + self._solver_model.chgVarLb(scip_var, lb) + self._solver_model.chgVarUb(scip_var, ub) + self._solver_model.chgVarType(scip_var, vtype) + + def _update_parameters(self, params: List[ParamData]): + for p in params: + pid = id(p) + scip_var = self._pyomo_param_to_solver_param_map[pid] + lb = ub = p.value + self._solver_model.chgVarLb(scip_var, lb) + self._solver_model.chgVarUb(scip_var, ub) + + def add_variables(self, variables): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.add_variables(variables) + + def add_constraints(self, cons): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.add_constraints(cons) + + def add_sos_constraints(self, cons): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.add_sos_constraints(cons) + + def set_objective(self, obj: ObjectiveData): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.add_objectives([obj]) + + def remove_constraints(self, cons): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.remove_constraints(cons) + + def remove_sos_constraints(self, cons): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.remove_sos_constraints(cons) + + def remove_variables(self, variables): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.remove_variables(variables) + + def update_variables(self, variables): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.update_variables(variables) + + def update_parameters(self, params): + if self._change_detector is None: + raise RuntimeError('call set_instance first') + self._change_detector.update_parameters(params) From 98e2c9a9904b688eb4cb735474bbfd0ea092370f Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 5 Oct 2025 21:40:47 -0600 Subject: [PATCH 097/104] update docs --- .../reference/topical/solvers/scip_persistent.rst | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 doc/OnlineDocs/reference/topical/solvers/scip_persistent.rst diff --git a/doc/OnlineDocs/reference/topical/solvers/scip_persistent.rst b/doc/OnlineDocs/reference/topical/solvers/scip_persistent.rst deleted file mode 100644 index 63ed55b74e3..00000000000 --- a/doc/OnlineDocs/reference/topical/solvers/scip_persistent.rst +++ /dev/null @@ -1,7 +0,0 @@ -SCIPPersistent -================ - -.. autoclass:: pyomo.solvers.plugins.solvers.scip_persistent.SCIPPersistent - :members: - :inherited-members: - :show-inheritance: \ No newline at end of file From f0be4ffd5a9cfed6b51d8aa22453358426d873cd Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 5 Oct 2025 21:41:07 -0600 Subject: [PATCH 098/104] update docs --- doc/OnlineDocs/reference/topical/solvers/index.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/OnlineDocs/reference/topical/solvers/index.rst b/doc/OnlineDocs/reference/topical/solvers/index.rst index 628f9cfdab0..400032df076 100644 --- a/doc/OnlineDocs/reference/topical/solvers/index.rst +++ b/doc/OnlineDocs/reference/topical/solvers/index.rst @@ -9,4 +9,3 @@ Solver Interfaces gurobi_direct.rst gurobi_persistent.rst xpress_persistent.rst - scip_persistent.rst From 7ec95a8f4f2c4a9e678cc61aa97c71be20411171 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Sun, 5 Oct 2025 21:46:55 -0600 Subject: [PATCH 099/104] persistent interface to scip --- pyomo/contrib/solver/plugins.py | 7 ++++++- .../solver/tests/solvers/test_solvers.py | 17 +++++++++++------ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 2c7bab3bf03..4ac74ecf560 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -15,7 +15,7 @@ from .solvers.gurobi.gurobi_direct import GurobiDirect from .solvers.gurobi.gurobi_persistent import GurobiDirectQuadratic, GurobiPersistent from .solvers.highs import Highs -from .solvers.scip.scip_direct import ScipDirect +from .solvers.scip.scip_direct import ScipDirect, ScipPersistent def load(): @@ -45,3 +45,8 @@ def load(): legacy_name='scip_direct_v2', doc='Direct interface pyscipopt', )(ScipDirect) + SolverFactory.register( + name='scip_persistent', + legacy_name='scip_persistent_v2', + doc='Persistent interface pyscipopt', + )(ScipPersistent) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index b49d80baa37..6bd7d01e679 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -34,7 +34,7 @@ SolutionStatus, Results, ) -from pyomo.contrib.solver.solvers.scip.scip_direct import SCIPDirect +from pyomo.contrib.solver.solvers.scip.scip_direct import ScipDirect, ScipPersistent from pyomo.contrib.solver.common.util import ( NoDualsError, NoOptimalSolutionError, @@ -60,30 +60,35 @@ ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), ('highs', Highs), - ('scip_direct', SCIPDirect), + ('scip_direct', ScipDirect), + ('scip_persistent', ScipPersistent), ] mip_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct', GurobiDirect), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('highs', Highs), - ('scip_direct', SCIPDirect), + ('scip_direct', ScipDirect), + ('scip_persistent', ScipPersistent), ] nlp_solvers = [ ('ipopt', Ipopt), - ('scip_direct', SCIPDirect), + ('scip_direct', ScipDirect), + ('scip_persistent', ScipPersistent), ] qcp_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), ('ipopt', Ipopt), - ('scip_direct', SCIPDirect), + ('scip_direct', ScipDirect), + ('scip_persistent', ScipPersistent), ] qp_solvers = qcp_solvers + [("highs", Highs)] miqcqp_solvers = [ ('gurobi_persistent', GurobiPersistent), ('gurobi_direct_quadratic', GurobiDirectQuadratic), - ('scip_direct', SCIPDirect), + ('scip_direct', ScipDirect), + ('scip_persistent', ScipPersistent), ] nl_solvers = [('ipopt', Ipopt)] nl_solvers_set = {i[0] for i in nl_solvers} From 75903d6cbbfbcd19f96bb709535e6b2a12b0086c Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 6 Oct 2025 08:44:16 -0600 Subject: [PATCH 100/104] persistent interface to scip --- pyomo/contrib/observer/model_observer.py | 9 ++ .../solver/solvers/scip/scip_direct.py | 135 +++++++++++++----- 2 files changed, 112 insertions(+), 32 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 2da340aab4f..77356ac1b57 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -1202,3 +1202,12 @@ def update(self, timer: Optional[HierarchicalTimer] = None, **kwds): finally: if is_gc_enabled: gc.enable() + + def get_variables_impacted_by_param(self, p: ParamData): + return [self._vars[vid][0] for vid in self._referenced_params[id(p)][3]] + + def get_constraints_impacted_by_param(self, p: ParamData): + return list(self._referenced_params[id(p)][0]) + + def get_constraints_impacted_by_var(self, v: VarData): + return list(self._referenced_variables[id(v)][0]) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index cf0c71606f8..99de1d80125 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -110,7 +110,7 @@ def __init__( ) -def _handle_var(node, data, opt): +def _handle_var(node, data, opt, visitor): if id(node) not in opt._pyomo_var_to_solver_var_map: scip_var = opt._add_var(node) else: @@ -118,7 +118,13 @@ def _handle_var(node, data, opt): return scip_var -def _handle_param(node, data, opt): +def _handle_param(node, data, opt, visitor): + # for the persistent interface, we create scip variables in place + # of parameters. However, this makes things complicated for range + # constraints because scip does not allow variables in the + # lower and upper parts of range constraints + if visitor.in_range: + return node.value if not opt.is_persistent(): return node.value if not node.mutable: @@ -130,19 +136,19 @@ def _handle_param(node, data, opt): return scip_param -def _handle_constant(node, data, opt): +def _handle_constant(node, data, opt, visitor): return node.value -def _handle_float(node, data, opt): +def _handle_float(node, data, opt, visitor): return float(node) -def _handle_negation(node, data, opt): +def _handle_negation(node, data, opt, visitor): return -data[0] -def _handle_pow(node, data, opt): +def _handle_pow(node, data, opt, visitor): x, y = data # x ** y = exp(log(x**y)) = exp(y*log(x)) if is_constant(node.args[1]): return x**y @@ -154,52 +160,52 @@ def _handle_pow(node, data, opt): return x**y # scip will probably raise an error here -def _handle_product(node, data, opt): +def _handle_product(node, data, opt, visitor): assert len(data) == 2 return data[0] * data[1] -def _handle_division(node, data, opt): +def _handle_division(node, data, opt, visitor): return data[0] / data[1] -def _handle_sum(node, data, opt): +def _handle_sum(node, data, opt, visitor): return sum(data) -def _handle_exp(node, data, opt): +def _handle_exp(node, data, opt, visitor): return scip.exp(data[0]) -def _handle_log(node, data, opt): +def _handle_log(node, data, opt, visitor): return scip.log(data[0]) -def _handle_log10(node, data, opt): +def _handle_log10(node, data, opt, visitor): return scip.log(data[0]) / math.log(10) -def _handle_sin(node, data, opt): +def _handle_sin(node, data, opt, visitor): return scip.sin(data[0]) -def _handle_cos(node, data, opt): +def _handle_cos(node, data, opt, visitor): return scip.cos(data[0]) -def _handle_sqrt(node, data, opt): +def _handle_sqrt(node, data, opt, visitor): return scip.sqrt(data[0]) -def _handle_abs(node, data, opt): +def _handle_abs(node, data, opt, visitor): return abs(data[0]) -def _handle_tan(node, data, opt): +def _handle_tan(node, data, opt, visitor): return scip.sin(data[0]) / scip.cos(data[0]) -def _handle_tanh(node, data, opt): +def _handle_tanh(node, data, opt, visitor): x = data[0] _exp = scip.exp return (_exp(x) - _exp(-x)) / (_exp(x) + _exp(-x)) @@ -218,30 +224,32 @@ def _handle_tanh(node, data, opt): } -def _handle_unary(node, data, opt): +def _handle_unary(node, data, opt, visitor): if node.getname() in _unary_map: - return _unary_map[node.getname()](node, data, opt) + return _unary_map[node.getname()](node, data, opt, visitor) else: raise NotImplementedError(f'unable to handle unary expression: {str(node)}') -def _handle_equality(node, data, opt): +def _handle_equality(node, data, opt, visitor): return data[0] == data[1] -def _handle_ranged(node, data, opt): +def _handle_ranged(node, data, opt, visitor): + # note that the lower and upper parts of the + # range constraint cannot have variables return data[0] <= (data[1] <= data[2]) -def _handle_inequality(node, data, opt): +def _handle_inequality(node, data, opt, visitor): return data[0] <= data[1] -def _handle_named_expression(node, data, opt): +def _handle_named_expression(node, data, opt, visitor): return data[0] -def _handle_unit(node, data, opt): +def _handle_unit(node, data, opt, visitor): return node.value @@ -281,16 +289,26 @@ class _PyomoToScipVisitor(StreamBasedExpressionVisitor): def __init__(self, solver, **kwds): super().__init__(**kwds) self.solver = solver + self.in_range = False + + def initializeWalker(self, expr): + self.in_range = False + return True, None def exitNode(self, node, data): nt = type(node) if nt in _operator_map: - return _operator_map[nt](node, data, self.solver) + return _operator_map[nt](node, data, self.solver, self) elif nt in native_numeric_types: _operator_map[nt] = _handle_float - return _handle_float(node, data, self.solver) + return _handle_float(node, data, self.solver, self) else: raise NotImplementedError(f'unrecognized expression type: {nt}') + + def enterNode(self, node): + if type(node) is RangedExpression: + self.in_range = True + return None, [] logger = logging.getLogger("pyomo.solvers") @@ -375,7 +393,7 @@ def __init__( pyomo_model, opt, ) - self._valid = False + self._valid = True def invalidate(self): self._valid = False @@ -513,6 +531,7 @@ def solve(self, model: BlockData, **kwds) -> Results: timer.start('optimize') with capture_output(TeeStream(*ostreams), capture_fd=True): + # scip_model.writeProblem(filename='foo.lp') scip_model.optimize() timer.stop('optimize') @@ -723,7 +742,7 @@ def _set_objective(self, obj): vtype="C" ) - if self._objective is not None: + if self._obj_con is not None: self._solver_model.delCons(self._obj_con) if obj is None: @@ -850,7 +869,7 @@ def add_variables(self, variables: List[VarData]): self.opt._add_variables(variables) def add_parameters(self, params: List[ParamData]): - pass + self.opt._add_parameters(params) def add_constraints(self, cons: List[ConstraintData]): self.opt._add_constraints(cons) @@ -874,7 +893,7 @@ def remove_variables(self, variables: List[VarData]): self.opt._remove_variables(variables) def remove_parameters(self, params: List[ParamData]): - pass + self.opt._remove_parameters(params) def update_variables(self, variables: List[VarData]): self.opt._update_variables(variables) @@ -893,13 +912,22 @@ def __init__(self, **kwds): self._observer = None self._change_detector = None self._last_results_object: Optional[Results] = None - + self._needs_reopt = False + self._range_constraints = set() + def _clear(self): super()._clear() self._pyomo_model = None self._objective = None self._observer = None self._change_detector = None + self._needs_reopt = False + + def _check_reopt(self): + if self._needs_reopt: + # self._solver_model.freeReoptSolve() # when is it safe to use this one??? + self._solver_model.freeTransform() + self._needs_reopt = False def _create_solver_model(self, pyomo_model): if pyomo_model is self._pyomo_model: @@ -921,6 +949,7 @@ def _create_solver_model(self, pyomo_model): def solve(self, model, **kwds) -> Results: res = super().solve(model, **kwds) + self._needs_reopt = True return res def update(self): @@ -957,19 +986,32 @@ def _invalidate_last_results(self): self._last_results_object.solution_loader.invalidate() def _add_variables(self, variables: List[VarData]): + self._check_reopt() self._invalidate_last_results() for v in variables: self._add_var(v) + def _add_parameters(self, params: List[ParamData]): + self._check_reopt() + self._invalidate_last_results() + for p in params: + self._add_param(p) + def _add_constraints(self, cons: List[ConstraintData]): + self._check_reopt() self._invalidate_last_results() + for con in cons: + if type(con.expr) is RangedExpression: + self._range_constraints.add(con) super()._add_constraints(cons) def _add_sos_constraints(self, cons: List[SOSConstraintData]): + self._check_reopt() self._invalidate_last_results() return super()._add_sos_constraints(cons) def _add_objectives(self, objs: List[ObjectiveData]): + self._check_reopt() if len(objs) > 1: raise NotImplementedError( 'the persistent interface to gurobi currently ' @@ -994,6 +1036,7 @@ def _add_objectives(self, objs: List[ObjectiveData]): self._set_objective(obj) def _remove_objectives(self, objs: List[ObjectiveData]): + self._check_reopt() for obj in objs: if obj is not self._objective: raise RuntimeError( @@ -1005,23 +1048,41 @@ def _remove_objectives(self, objs: List[ObjectiveData]): self._set_objective(None) def _remove_constraints(self, cons: List[ConstraintData]): + self._check_reopt() + self._invalidate_last_results() for con in cons: scip_con = self._pyomo_con_to_solver_con_map.pop(con) self._solver_model.delCons(scip_con) + self._range_constraints.discard(con) def _remove_sos_constraints(self, cons: List[SOSConstraintData]): + self._check_reopt() + self._invalidate_last_results() for con in cons: scip_con = self._pyomo_con_to_solver_con_map.pop(con) self._solver_model.delCons(scip_con) def _remove_variables(self, variables: List[VarData]): + self._check_reopt() + self._invalidate_last_results() for v in variables: vid = id(v) scip_var = self._pyomo_var_to_solver_var_map.pop(vid) self._solver_model.delVar(scip_var) self._vars.pop(vid) + def _remove_parameters(self, params: List[ParamData]): + self._check_reopt() + self._invalidate_last_results() + for p in params: + pid = id(p) + scip_var = self._pyomo_param_to_solver_param_map.pop(pid) + self._solver_model.delVar(scip_var) + self._params.pop(pid) + def _update_variables(self, variables: List[VarData]): + self._check_reopt() + self._invalidate_last_results() for v in variables: vid = id(v) scip_var = self._pyomo_var_to_solver_var_map[vid] @@ -1032,12 +1093,22 @@ def _update_variables(self, variables: List[VarData]): self._solver_model.chgVarType(scip_var, vtype) def _update_parameters(self, params: List[ParamData]): + self._check_reopt() + self._invalidate_last_results() for p in params: pid = id(p) scip_var = self._pyomo_param_to_solver_param_map[pid] lb = ub = p.value self._solver_model.chgVarLb(scip_var, lb) self._solver_model.chgVarUb(scip_var, ub) + impacted_vars = self._change_detector.get_variables_impacted_by_param(p) + if impacted_vars: + self._update_variables(impacted_vars) + impacted_cons = self._change_detector.get_constraints_impacted_by_param(p) + for con in impacted_cons: + if con in self._range_constraints: + self._remove_constraints([con]) + self._add_constraints([con]) def add_variables(self, variables): if self._change_detector is None: From 0051024e7a44ab7d4c3df43c96890c49669e3b69 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 6 Oct 2025 10:46:14 -0600 Subject: [PATCH 101/104] updating tests --- .../solver/tests/solvers/test_solvers.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 6bd7d01e679..3665de4521a 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -2381,7 +2381,8 @@ def test_param_updates(self, name: str, opt_class: Type[SolverBase]): m.obj = pyo.Objective(expr=m.y) m.c1 = pyo.Constraint(expr=(0, m.y - m.a1 * m.x - m.b1, None)) m.c2 = pyo.Constraint(expr=(None, -m.y + m.a2 * m.x + m.b2, 0)) - m.dual = pyo.Suffix(direction=pyo.Suffix.IMPORT) + if (name, opt_class) in dual_solvers: + m.dual = pyo.Suffix(direction=pyo.Suffix.IMPORT) params_to_test = [(1, -1, 2, 1), (1, -2, 2, 1), (1, -1, 3, 1)] for a1, a2, b1, b2 in params_to_test: @@ -2393,8 +2394,9 @@ def test_param_updates(self, name: str, opt_class: Type[SolverBase]): pyo.assert_optimal_termination(res) self.assertAlmostEqual(m.x.value, (b2 - b1) / (a1 - a2)) self.assertAlmostEqual(m.y.value, a1 * (b2 - b1) / (a1 - a2) + b1) - self.assertAlmostEqual(m.dual[m.c1], (1 + a1 / (a2 - a1))) - self.assertAlmostEqual(m.dual[m.c2], a1 / (a2 - a1)) + if (name, opt_class) in dual_solvers: + self.assertAlmostEqual(m.dual[m.c1], (1 + a1 / (a2 - a1))) + self.assertAlmostEqual(m.dual[m.c2], a1 / (a2 - a1)) @parameterized.expand(input=all_solvers) def test_load_solutions(self, name: str, opt_class: Type[SolverBase]): @@ -2405,11 +2407,14 @@ def test_load_solutions(self, name: str, opt_class: Type[SolverBase]): m.x = pyo.Var() m.obj = pyo.Objective(expr=m.x) m.c = pyo.Constraint(expr=(-1, m.x, 1)) - m.dual = pyo.Suffix(direction=pyo.Suffix.IMPORT) + if (name, opt_class) in dual_solvers: + m.dual = pyo.Suffix(direction=pyo.Suffix.IMPORT) res = opt.solve(m, load_solutions=False) pyo.assert_optimal_termination(res) self.assertIsNone(m.x.value) - self.assertNotIn(m.c, m.dual) + if (name, opt_class) in dual_solvers: + self.assertNotIn(m.c, m.dual) m.solutions.load_from(res) self.assertAlmostEqual(m.x.value, -1) - self.assertAlmostEqual(m.dual[m.c], 1) + if (name, opt_class) in dual_solvers: + self.assertAlmostEqual(m.dual[m.c], 1) From b037b9c356cd9caac3bf04924bece2c247962f91 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 6 Oct 2025 10:57:00 -0600 Subject: [PATCH 102/104] forgot to delete/revert some files --- .../solver/solvers/scip/scip_persistent.py | 192 ----------- pyomo/solvers/tests/checks/test_SCIPDirect.py | 310 ----------------- .../tests/checks/test_SCIPPersistent.py | 318 ------------------ pyomo/solvers/tests/solvers.py | 15 - pyomo/solvers/tests/testcases.py | 9 - 5 files changed, 844 deletions(-) delete mode 100644 pyomo/contrib/solver/solvers/scip/scip_persistent.py delete mode 100644 pyomo/solvers/tests/checks/test_SCIPDirect.py delete mode 100644 pyomo/solvers/tests/checks/test_SCIPPersistent.py diff --git a/pyomo/contrib/solver/solvers/scip/scip_persistent.py b/pyomo/contrib/solver/solvers/scip/scip_persistent.py deleted file mode 100644 index bc64edc28a8..00000000000 --- a/pyomo/contrib/solver/solvers/scip/scip_persistent.py +++ /dev/null @@ -1,192 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ -from pyomo.solvers.plugins.solvers.scip_direct import SCIPDirect -from pyomo.solvers.plugins.solvers.persistent_solver import PersistentSolver -from pyomo.opt.base import SolverFactory - - -@SolverFactory.register("scip_persistent", doc="Persistent python interface to SCIP") -class SCIPPersistent(PersistentSolver, SCIPDirect): - """ - A class that provides a persistent interface to SCIP. Direct solver interfaces do not use any file io. - Rather, they interface directly with the python bindings for the specific solver. Persistent solver interfaces - are similar except that they "remember" their model. Thus, persistent solver interfaces allow incremental changes - to the solver model (e.g., the gurobi python model or the cplex python model). Note that users are responsible - for notifying the persistent solver interfaces when changes are made to the corresponding pyomo model. - - Keyword Arguments - ----------------- - model: ConcreteModel - Passing a model to the constructor is equivalent to calling the set_instance method. - type: str - String indicating the class type of the solver instance. - name: str - String representing either the class type of the solver instance or an assigned name. - doc: str - Documentation for the solver - options: dict - Dictionary of solver options - """ - - def __init__(self, **kwds): - kwds["type"] = "scip_persistent" - PersistentSolver.__init__(self, **kwds) - SCIPDirect._init(self) - - self._pyomo_model = kwds.pop("model", None) - if self._pyomo_model is not None: - self.set_instance(self._pyomo_model, **kwds) - - def _remove_constraint(self, solver_conname): - con = self._solver_con_to_pyomo_con_map[solver_conname] - scip_con = self._pyomo_con_to_solver_con_expr_map[con] - self._solver_model.delCons(scip_con) - del self._pyomo_con_to_solver_con_expr_map[con] - - def _remove_sos_constraint(self, solver_sos_conname): - con = self._solver_con_to_pyomo_con_map[solver_sos_conname] - scip_con = self._pyomo_con_to_solver_con_expr_map[con] - self._solver_model.delCons(scip_con) - del self._pyomo_con_to_solver_con_expr_map[con] - - def _remove_var(self, solver_varname): - var = self._solver_var_to_pyomo_var_map[solver_varname] - scip_var = self._pyomo_var_to_solver_var_expr_map[var] - self._solver_model.delVar(scip_var) - del self._pyomo_var_to_solver_var_expr_map[var] - - def _warm_start(self): - SCIPDirect._warm_start(self) - - def update_var(self, var): - """Update a single variable in the solver's model. - - This will update bounds, fix/unfix the variable as needed, and - update the variable type. - - Parameters - ---------- - var: Var (scalar Var or single _VarData) - - """ - # see PR #366 for discussion about handling indexed - # objects and keeping compatibility with the - # pyomo.kernel objects - # if var.is_indexed(): - # for child_var in var.values(): - # self.compile_var(child_var) - # return - if var not in self._pyomo_var_to_solver_var_map: - raise ValueError( - f"The Var provided to compile_var needs to be added first: {var}" - ) - scip_var = self._pyomo_var_to_solver_var_map[var] - vtype = self._scip_vtype_from_var(var) - lb, ub = self._scip_lb_ub_from_var(var) - - self._solver_model.chgVarLb(scip_var, lb) - self._solver_model.chgVarUb(scip_var, ub) - self._solver_model.chgVarType(scip_var, vtype) - - def write(self, filename, filetype=""): - """ - Write the model to a file (e.g., an lp file). - - Parameters - ---------- - filename: str - Name of the file to which the model should be written. - filetype: str - The file type (e.g., lp). - """ - self._solver_model.writeProblem(filename + filetype) - - def set_scip_param(self, param, val): - """ - Set a SCIP parameter. - - Parameters - ---------- - param: str - The SCIP parameter to set. Options include any SCIP parameter. - Please see the SCIP documentation for options. - Link at: https://www.scipopt.org/doc/html/PARAMETERS.php - val: any - The value to set the parameter to. See SCIP documentation for possible values. - """ - self._solver_model.setParam(param, val) - - def get_scip_param(self, param): - """ - Get the value of the SCIP parameter. - - Parameters - ---------- - param: str or int or float - The SCIP parameter to get the value of. See SCIP documentation for possible options. - Link at: https://www.scipopt.org/doc/html/PARAMETERS.php - """ - return self._solver_model.getParam(param) - - def _add_column(self, var, obj_coef, constraints, coefficients): - """Add a column to the solver's model - - This will add the Pyomo variable var to the solver's - model, and put the coefficients on the associated - constraints in the solver model. If the obj_coef is - not zero, it will add obj_coef*var to the objective - of the solver's model. - - Parameters - ---------- - var: Var (scalar Var or single _VarData) - obj_coef: float - constraints: list of solver constraints - coefficients: list of coefficients to put on var in the associated constraint - """ - - # Set-up add var - varname = self._symbol_map.getSymbol(var, self._labeler) - vtype = self._scip_vtype_from_var(var) - lb, ub = self._scip_lb_ub_from_var(var) - - # Add the variable to the model and then to all the constraints - scip_var = self._solver_model.addVar(lb=lb, ub=ub, vtype=vtype, name=varname) - self._pyomo_var_to_solver_var_expr_map[var] = scip_var - self._solver_var_to_pyomo_var_map[varname] = var - self._referenced_variables[var] = len(coefficients) - - # Get the SCIP cons by passing through two dictionaries - pyomo_cons = [self._solver_con_to_pyomo_con_map[con] for con in constraints] - scip_cons = [ - self._pyomo_con_to_solver_con_expr_map[pyomo_con] - for pyomo_con in pyomo_cons - ] - - for i, scip_con in enumerate(scip_cons): - if not scip_con.isLinear(): - raise ValueError( - "_add_column functionality not supported for non-linear constraints" - ) - self._solver_model.addConsCoeff(scip_con, scip_var, coefficients[i]) - con = self._solver_con_to_pyomo_con_map[scip_con.name] - self._vars_referenced_by_con[con].add(var) - - sense = self._solver_model.getObjectiveSense() - self._solver_model.setObjective(obj_coef * scip_var, sense=sense, clear=False) - - def reset(self): - """This function is necessary to call before making any changes to the - SCIP model after optimizing. It frees solution run specific information - that is not automatically done when changes to an already solved model - are made. Making changes to an already optimized model, e.g. adding additional - constraints will raise an error unless this function is called.""" - self._solver_model.freeTransform() diff --git a/pyomo/solvers/tests/checks/test_SCIPDirect.py b/pyomo/solvers/tests/checks/test_SCIPDirect.py deleted file mode 100644 index 186de0eaf58..00000000000 --- a/pyomo/solvers/tests/checks/test_SCIPDirect.py +++ /dev/null @@ -1,310 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ - -import sys - -import pyomo.common.unittest as unittest - -from pyomo.environ import ( - ConcreteModel, - AbstractModel, - Var, - Objective, - Block, - Constraint, - Suffix, - NonNegativeIntegers, - NonNegativeReals, - Integers, - Binary, - value, -) -from pyomo.opt import SolverFactory, TerminationCondition, SolutionStatus - -try: - import pyscipopt - - scip_available = True -except ImportError: - scip_available = False - - -@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") -class SCIPDirectTests(unittest.TestCase): - def setUp(self): - self.stderr = sys.stderr - sys.stderr = None - - def tearDown(self): - sys.stderr = self.stderr - - def test_infeasible_lp(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var(within=NonNegativeReals) - model.C1 = Constraint(expr=model.X == 1) - model.C2 = Constraint(expr=model.X == 2) - model.O = Objective(expr=model.X) - - results = opt.solve(model) - - self.assertEqual( - results.solver.termination_condition, TerminationCondition.infeasible - ) - - def test_unbounded_lp(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var() - model.O = Objective(expr=model.X) - - results = opt.solve(model) - - self.assertIn( - results.solver.termination_condition, - ( - TerminationCondition.unbounded, - TerminationCondition.infeasibleOrUnbounded, - ), - ) - - def test_optimal_lp(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var(within=NonNegativeReals) - model.O = Objective(expr=model.X) - - results = opt.solve(model, load_solutions=False) - - self.assertEqual(results.solution.status, SolutionStatus.optimal) - - def test_infeasible_mip(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var(within=NonNegativeIntegers) - model.C1 = Constraint(expr=model.X == 1) - model.C2 = Constraint(expr=model.X == 2) - model.O = Objective(expr=model.X) - - results = opt.solve(model) - - self.assertEqual( - results.solver.termination_condition, TerminationCondition.infeasible - ) - - def test_unbounded_mip(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = AbstractModel() - model.X = Var(within=Integers) - model.O = Objective(expr=model.X) - - instance = model.create_instance() - results = opt.solve(instance) - - self.assertIn( - results.solver.termination_condition, - ( - TerminationCondition.unbounded, - TerminationCondition.infeasibleOrUnbounded, - ), - ) - - def test_optimal_mip(self): - with SolverFactory("scip_direct", solver_io="python") as opt: - model = ConcreteModel() - model.X = Var(within=NonNegativeIntegers) - model.O = Objective(expr=model.X) - - results = opt.solve(model, load_solutions=False) - - self.assertEqual(results.solution.status, SolutionStatus.optimal) - - -@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") -class TestAddVar(unittest.TestCase): - def test_add_single_variable(self): - """Test that the variable is added correctly to `solver_model`.""" - model = ConcreteModel() - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNVars(), 0) - - model.X = Var(within=Binary) - - opt._add_var(model.X) - - self.assertEqual(opt._solver_model.getNVars(), 1) - self.assertEqual(opt._solver_model.getVars()[0].vtype(), "BINARY") - - def test_add_block_containing_single_variable(self): - """Test that the variable is added correctly to `solver_model`.""" - model = ConcreteModel() - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNVars(), 0) - - model.X = Var(within=Binary) - - opt._add_block(model) - - self.assertEqual(opt._solver_model.getNVars(), 1) - self.assertEqual(opt._solver_model.getVars()[0].vtype(), "BINARY") - - def test_add_block_containing_multiple_variables(self): - """Test that: - - The variable is added correctly to `solver_model` - - Fixed variable bounds are set correctly - """ - model = ConcreteModel() - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNVars(), 0) - - model.X1 = Var(within=Binary) - model.X2 = Var(within=NonNegativeReals) - model.X3 = Var(within=NonNegativeIntegers) - - model.X3.fix(5) - - opt._add_block(model) - - self.assertEqual(opt._solver_model.getNVars(), 3) - scip_vars = opt._solver_model.getVars() - vtypes = [scip_var.vtype() for scip_var in scip_vars] - assert "BINARY" in vtypes and "CONTINUOUS" in vtypes and "INTEGER" in vtypes - lbs = [scip_var.getLbGlobal() for scip_var in scip_vars] - ubs = [scip_var.getUbGlobal() for scip_var in scip_vars] - assert 0 in lbs and 5 in lbs - assert ( - 1 in ubs - and 5 in ubs - and any([opt._solver_model.isInfinity(ub) for ub in ubs]) - ) - - -@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") -class TestAddCon(unittest.TestCase): - def test_add_single_constraint(self): - model = ConcreteModel() - model.X = Var(within=Binary) - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNConss(), 0) - - model.C = Constraint(expr=model.X == 1) - - opt._add_constraint(model.C) - - self.assertEqual(opt._solver_model.getNConss(), 1) - con = opt._solver_model.getConss()[0] - self.assertEqual(con.isLinear(), 1) - self.assertEqual(opt._solver_model.getRhs(con), 1) - - def test_add_block_containing_single_constraint(self): - model = ConcreteModel() - model.X = Var(within=Binary) - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNConss(), 0) - - model.B = Block() - model.B.C = Constraint(expr=model.X == 1) - - opt._add_block(model.B) - - self.assertEqual(opt._solver_model.getNConss(), 1) - con = opt._solver_model.getConss()[0] - self.assertEqual(con.isLinear(), 1) - self.assertEqual(opt._solver_model.getRhs(con), 1) - - def test_add_block_containing_multiple_constraints(self): - model = ConcreteModel() - model.X = Var(within=Binary) - - opt = SolverFactory("scip_direct", solver_io="python") - opt._set_instance(model) - - self.assertEqual(opt._solver_model.getNConss(), 0) - - model.B = Block() - model.B.C1 = Constraint(expr=model.X == 1) - model.B.C2 = Constraint(expr=model.X <= 1) - model.B.C3 = Constraint(expr=model.X >= 1) - - opt._add_block(model.B) - - self.assertEqual(opt._solver_model.getNConss(), 3) - - -@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") -class TestLoadVars(unittest.TestCase): - def setUp(self): - opt = SolverFactory("scip_direct", solver_io="python") - model = ConcreteModel() - model.X = Var(within=NonNegativeReals, initialize=0) - model.Y = Var(within=NonNegativeReals, initialize=0) - - model.C1 = Constraint(expr=2 * model.X + model.Y >= 8) - model.C2 = Constraint(expr=model.X + 3 * model.Y >= 6) - - model.O = Objective(expr=model.X + model.Y) - - opt.solve(model, load_solutions=False, save_results=False) - - self._model = model - self._opt = opt - - def test_all_vars_are_loaded(self): - self.assertTrue(self._model.X.stale) - self.assertTrue(self._model.Y.stale) - self.assertEqual(value(self._model.X), 0) - self.assertEqual(value(self._model.Y), 0) - - self._opt.load_vars() - - self.assertFalse(self._model.X.stale) - self.assertFalse(self._model.Y.stale) - self.assertAlmostEqual(value(self._model.X), 3.6) - self.assertAlmostEqual(value(self._model.Y), 0.8) - - def test_only_specified_vars_are_loaded(self): - self.assertTrue(self._model.X.stale) - self.assertTrue(self._model.Y.stale) - self.assertEqual(value(self._model.X), 0) - self.assertEqual(value(self._model.Y), 0) - - self._opt.load_vars([self._model.X]) - - self.assertFalse(self._model.X.stale) - self.assertTrue(self._model.Y.stale) - self.assertAlmostEqual(value(self._model.X), 3.6) - self.assertEqual(value(self._model.Y), 0) - - self._opt.load_vars([self._model.Y]) - - self.assertFalse(self._model.X.stale) - self.assertFalse(self._model.Y.stale) - self.assertAlmostEqual(value(self._model.X), 3.6) - self.assertAlmostEqual(value(self._model.Y), 0.8) - - -if __name__ == "__main__": - unittest.main() diff --git a/pyomo/solvers/tests/checks/test_SCIPPersistent.py b/pyomo/solvers/tests/checks/test_SCIPPersistent.py deleted file mode 100644 index 61cf7385352..00000000000 --- a/pyomo/solvers/tests/checks/test_SCIPPersistent.py +++ /dev/null @@ -1,318 +0,0 @@ -# ___________________________________________________________________________ -# -# Pyomo: Python Optimization Modeling Objects -# Copyright (c) 2008-2025 -# National Technology and Engineering Solutions of Sandia, LLC -# Under the terms of Contract DE-NA0003525 with National Technology and -# Engineering Solutions of Sandia, LLC, the U.S. Government retains certain -# rights in this software. -# This software is distributed under the 3-clause BSD License. -# ___________________________________________________________________________ - -import pyomo.environ -import pyomo.common.unittest as unittest - -from pyomo.core import ( - ConcreteModel, - Var, - Objective, - Constraint, - NonNegativeReals, - NonNegativeIntegers, - Reals, - Binary, - SOSConstraint, - Set, - sin, - cos, - exp, - log, -) -from pyomo.opt import SolverFactory - -try: - import pyscipopt - - scip_available = True -except ImportError: - scip_available = False - - -@unittest.skipIf(not scip_available, "The SCIP python bindings are not available") -class TestQuadraticObjective(unittest.TestCase): - def test_quadratic_objective_linear_surrogate_is_set(self): - m = ConcreteModel() - m.X = Var(bounds=(-2, 2)) - m.Y = Var(bounds=(-2, 2)) - m.Z = Var(within=Reals) - m.O = Objective(expr=m.Z) - m.C1 = Constraint(expr=m.Y >= 2 * m.X - 1) - m.C2 = Constraint(expr=m.Y >= -m.X + 2) - m.C3 = Constraint(expr=m.Z >= m.X**2 + m.Y**2) - opt = SolverFactory("scip_persistent") - opt.set_instance(m) - opt.solve() - - self.assertAlmostEqual(m.X.value, 1, places=3) - self.assertAlmostEqual(m.Y.value, 1, places=3) - - opt.reset() - - opt.remove_constraint(m.C3) - del m.C3 - m.C3 = Constraint(expr=m.Z >= m.X**2) - opt.add_constraint(m.C3) - opt.solve() - self.assertAlmostEqual(m.X.value, 0, places=3) - self.assertAlmostEqual(m.Y.value, 2, places=3) - - def test_add_and_remove_sos(self): - m = ConcreteModel() - m.I = Set(initialize=[1, 2, 3]) - m.X = Var(m.I, bounds=(-2, 2)) - - m.C = SOSConstraint(var=m.X, sos=1) - - m.O = Objective(expr=m.X[1] + m.X[2]) - - opt = SolverFactory("scip_persistent") - - opt.set_instance(m) - opt.solve() - - zero_val_var = 0 - for i in range(1, 4): - if -0.001 < m.X[i].value < 0.001: - zero_val_var += 1 - assert zero_val_var == 2 - - opt.reset() - - opt.remove_sos_constraint(m.C) - del m.C - - m.C = SOSConstraint(var=m.X, sos=2) - opt.add_sos_constraint(m.C) - - opt.solve() - - zero_val_var = 0 - for i in range(1, 4): - if -0.001 < m.X[i].value < 0.001: - zero_val_var += 1 - assert zero_val_var == 1 - - def test_get_and_set_param(self): - m = ConcreteModel() - m.X = Var(bounds=(-2, 2)) - m.O = Objective(expr=m.X) - m.C3 = Constraint(expr=m.X <= 2) - opt = SolverFactory("scip_persistent") - opt.set_instance(m) - - opt.set_scip_param("limits/time", 60) - - assert opt.get_scip_param("limits/time") == 60 - - def test_non_linear(self): - - PI = 3.141592653589793238462643 - NWIRES = 11 - DIAMETERS = [ - 0.207, - 0.225, - 0.244, - 0.263, - 0.283, - 0.307, - 0.331, - 0.362, - 0.394, - 0.4375, - 0.500, - ] - PRELOAD = 300.0 - MAXWORKLOAD = 1000.0 - MAXDEFLECT = 6.0 - DEFLECTPRELOAD = 1.25 - MAXFREELEN = 14.0 - MAXCOILDIAM = 3.0 - MAXSHEARSTRESS = 189000.0 - SHEARMOD = 11500000.0 - - m = ConcreteModel() - m.coil = Var(within=NonNegativeReals) - m.wire = Var(within=NonNegativeReals) - m.defl = Var( - bounds=(DEFLECTPRELOAD / (MAXWORKLOAD - PRELOAD), MAXDEFLECT / PRELOAD) - ) - m.ncoils = Var(within=NonNegativeIntegers) - m.const1 = Var(within=NonNegativeReals) - m.const2 = Var(within=NonNegativeReals) - m.volume = Var(within=NonNegativeReals) - m.I = Set(initialize=[i for i in range(NWIRES)]) - m.y = Var(m.I, within=Binary) - - m.O = Objective(expr=m.volume) - - m.c1 = Constraint( - expr=PI / 2 * (m.ncoils + 2) * m.coil * m.wire**2 - m.volume == 0 - ) - - m.c2 = Constraint(expr=m.coil / m.wire - m.const1 == 0) - - m.c3 = Constraint( - expr=(4 * m.const1 - 1) / (4 * m.const1 - 4) + 0.615 / m.const1 - m.const2 - == 0 - ) - - m.c4 = Constraint( - expr=8.0 * MAXWORKLOAD / PI * m.const1 * m.const2 - - MAXSHEARSTRESS * m.wire**2 - <= 0 - ) - - m.c5 = Constraint( - expr=8 / SHEARMOD * m.ncoils * m.const1**3 / m.wire - m.defl == 0 - ) - - m.c6 = Constraint( - expr=MAXWORKLOAD * m.defl + 1.05 * m.ncoils * m.wire + 2.1 * m.wire - <= MAXFREELEN - ) - - m.c7 = Constraint(expr=m.coil + m.wire <= MAXCOILDIAM) - - m.c8 = Constraint( - expr=sum(m.y[i] * DIAMETERS[i] for i in range(NWIRES)) - m.wire == 0 - ) - - m.c9 = Constraint(expr=sum(m.y[i] for i in range(NWIRES)) == 1) - - opt = SolverFactory("scip_persistent") - opt.set_instance(m) - - opt.solve() - - self.assertAlmostEqual(m.volume.value, 1.6924910128, places=2) - - def test_non_linear_unary_expressions(self): - - m = ConcreteModel() - m.X = Var(bounds=(1, 2)) - m.Y = Var(within=Reals) - - m.O = Objective(expr=m.Y) - - m.C = Constraint(expr=exp(m.X) == m.Y) - - opt = SolverFactory("scip_persistent") - opt.set_instance(m) - - opt.solve() - self.assertAlmostEqual(m.X.value, 1, places=3) - self.assertAlmostEqual(m.Y.value, exp(1), places=3) - - opt.reset() - opt.remove_constraint(m.C) - del m.C - - m.C = Constraint(expr=log(m.X) == m.Y) - opt.add_constraint(m.C) - opt.solve() - self.assertAlmostEqual(m.X.value, 1, places=3) - self.assertAlmostEqual(m.Y.value, 0, places=3) - - opt.reset() - opt.remove_constraint(m.C) - del m.C - - m.C = Constraint(expr=sin(m.X) == m.Y) - opt.add_constraint(m.C) - opt.solve() - self.assertAlmostEqual(m.X.value, 1, places=3) - self.assertAlmostEqual(m.Y.value, sin(1), places=3) - - opt.reset() - opt.remove_constraint(m.C) - del m.C - - m.C = Constraint(expr=cos(m.X) == m.Y) - opt.add_constraint(m.C) - opt.solve() - self.assertAlmostEqual(m.X.value, 2, places=3) - self.assertAlmostEqual(m.Y.value, cos(2), places=3) - - def test_add_column(self): - m = ConcreteModel() - m.x = Var(within=NonNegativeReals) - m.c = Constraint(expr=(0, m.x, 1)) - m.obj = Objective(expr=-m.x) - - opt = SolverFactory("scip_persistent") - opt.set_instance(m) - opt.solve() - self.assertAlmostEqual(m.x.value, 1) - - m.y = Var(within=NonNegativeReals) - - opt.reset() - - opt.add_column(m, m.y, -3, [m.c], [2]) - opt.solve() - - self.assertAlmostEqual(m.x.value, 0) - self.assertAlmostEqual(m.y.value, 0.5) - - def test_add_column_exceptions(self): - m = ConcreteModel() - m.x = Var() - m.c = Constraint(expr=(0, m.x, 1)) - m.ci = Constraint([1, 2], rule=lambda m, i: (0, m.x, i + 1)) - m.cd = Constraint(expr=(0, -m.x, 1)) - m.cd.deactivate() - m.obj = Objective(expr=-m.x) - - opt = SolverFactory("scip_persistent") - - # set_instance not called - self.assertRaises(RuntimeError, opt.add_column, m, m.x, 0, [m.c], [1]) - - opt.set_instance(m) - - m2 = ConcreteModel() - m2.y = Var() - m2.c = Constraint(expr=(0, m.x, 1)) - - # different model than attached to opt - self.assertRaises(RuntimeError, opt.add_column, m2, m2.y, 0, [], []) - # pyomo var attached to different model - self.assertRaises(RuntimeError, opt.add_column, m, m2.y, 0, [], []) - - z = Var() - # pyomo var floating - self.assertRaises(RuntimeError, opt.add_column, m, z, -2, [m.c, z], [1]) - - m.y = Var() - # len(coefficients) == len(constraints) - self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c], [1, 2]) - self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c, z], [1]) - - # add indexed constraint - self.assertRaises(AttributeError, opt.add_column, m, m.y, -2, [m.ci], [1]) - # add something not a _ConstraintData - self.assertRaises(AttributeError, opt.add_column, m, m.y, -2, [m.x], [1]) - - # constraint not on solver model - self.assertRaises(KeyError, opt.add_column, m, m.y, -2, [m2.c], [1]) - - # inactive constraint - self.assertRaises(KeyError, opt.add_column, m, m.y, -2, [m.cd], [1]) - - opt.add_var(m.y) - # var already in solver model - self.assertRaises(RuntimeError, opt.add_column, m, m.y, -2, [m.c], [1]) - - -if __name__ == "__main__": - unittest.main() diff --git a/pyomo/solvers/tests/solvers.py b/pyomo/solvers/tests/solvers.py index e9967cd1ce2..e5058e8894b 100644 --- a/pyomo/solvers/tests/solvers.py +++ b/pyomo/solvers/tests/solvers.py @@ -369,21 +369,6 @@ def test_solver_cases(*args): name='scip', io='nl', capabilities=_scip_capabilities, import_suffixes=[] ) - # - # SCIP PERSISTENT - # - - _scip_persistent_capabilities = set( - ["linear", "integer", "quadratic_constraint", "sos1", "sos2"] - ) - - _test_solver_cases["scip_persistent", "python"] = initialize( - name="scip_persistent", - io="python", - capabilities=_scip_persistent_capabilities, - import_suffixes=[], - ) - # # CONOPT # diff --git a/pyomo/solvers/tests/testcases.py b/pyomo/solvers/tests/testcases.py index c1725bedee7..696936ddf05 100644 --- a/pyomo/solvers/tests/testcases.py +++ b/pyomo/solvers/tests/testcases.py @@ -248,15 +248,6 @@ "inside NL files. A ticket has been filed.", ) -# -# SCIP Persistent -# - -ExpectedFailures["scip_persistent", "python", "LP_trivial_constraints"] = ( - lambda v: v <= _trunk_version, - "SCIP does not allow empty constraints with no variables to be added to the Model.", -) - # # BARON # From c200e2e996920c0eccb2e12d3b74376847e3b448 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 6 Oct 2025 15:09:16 -0600 Subject: [PATCH 103/104] run black --- pyomo/contrib/observer/model_observer.py | 2 +- pyomo/contrib/solver/plugins.py | 8 +- .../solver/solvers/scip/scip_direct.py | 138 +++++++++--------- .../solver/tests/solvers/test_solvers.py | 3 +- 4 files changed, 75 insertions(+), 76 deletions(-) diff --git a/pyomo/contrib/observer/model_observer.py b/pyomo/contrib/observer/model_observer.py index 77356ac1b57..9bb9c917fc5 100644 --- a/pyomo/contrib/observer/model_observer.py +++ b/pyomo/contrib/observer/model_observer.py @@ -1208,6 +1208,6 @@ def get_variables_impacted_by_param(self, p: ParamData): def get_constraints_impacted_by_param(self, p: ParamData): return list(self._referenced_params[id(p)][0]) - + def get_constraints_impacted_by_var(self, v: VarData): return list(self._referenced_variables[id(v)][0]) diff --git a/pyomo/contrib/solver/plugins.py b/pyomo/contrib/solver/plugins.py index 4ac74ecf560..ff24148dd73 100644 --- a/pyomo/contrib/solver/plugins.py +++ b/pyomo/contrib/solver/plugins.py @@ -41,12 +41,12 @@ def load(): name='highs', legacy_name='highs_v2', doc='Persistent interface to HiGHS' )(Highs) SolverFactory.register( - name='scip_direct', - legacy_name='scip_direct_v2', + name='scip_direct', + legacy_name='scip_direct_v2', doc='Direct interface pyscipopt', )(ScipDirect) SolverFactory.register( - name='scip_persistent', - legacy_name='scip_persistent_v2', + name='scip_persistent', + legacy_name='scip_persistent_v2', doc='Persistent interface pyscipopt', )(ScipPersistent) diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 99de1d80125..7e39d6e8595 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -47,11 +47,19 @@ from pyomo.core.expr.numvalue import NumericConstant from pyomo.gdp.disjunct import AutoLinkedBinaryVar from pyomo.core.base.expression import ExpressionData, ScalarExpression -from pyomo.core.expr.relational_expr import EqualityExpression, InequalityExpression, RangedExpression +from pyomo.core.expr.relational_expr import ( + EqualityExpression, + InequalityExpression, + RangedExpression, +) from pyomo.core.staleflag import StaleFlagManager from pyomo.core.expr.visitor import StreamBasedExpressionVisitor from pyomo.common.dependencies import attempt_import -from pyomo.contrib.solver.common.base import SolverBase, Availability, PersistentSolverBase +from pyomo.contrib.solver.common.base import ( + SolverBase, + Availability, + PersistentSolverBase, +) from pyomo.contrib.solver.common.config import BranchAndBoundConfig from pyomo.contrib.solver.common.util import ( NoFeasibleSolutionError, @@ -73,7 +81,11 @@ from pyomo.common.tee import capture_output, TeeStream from pyomo.core.base.units_container import _PyomoUnit from pyomo.contrib.fbbt.fbbt import compute_bounds_on_expr -from pyomo.contrib.observer.model_observer import Observer, ModelChangeDetector, AutoUpdateConfig +from pyomo.contrib.observer.model_observer import ( + Observer, + ModelChangeDetector, + AutoUpdateConfig, +) logger = logging.getLogger(__name__) @@ -120,8 +132,8 @@ def _handle_var(node, data, opt, visitor): def _handle_param(node, data, opt, visitor): # for the persistent interface, we create scip variables in place - # of parameters. However, this makes things complicated for range - # constraints because scip does not allow variables in the + # of parameters. However, this makes things complicated for range + # constraints because scip does not allow variables in the # lower and upper parts of range constraints if visitor.in_range: return node.value @@ -155,7 +167,7 @@ def _handle_pow(node, data, opt, visitor): else: xlb, xub = compute_bounds_on_expr(node.args[0]) if xlb > 0: - return scip.exp(y*scip.log(x)) + return scip.exp(y * scip.log(x)) else: return x**y # scip will probably raise an error here @@ -236,7 +248,7 @@ def _handle_equality(node, data, opt, visitor): def _handle_ranged(node, data, opt, visitor): - # note that the lower and upper parts of the + # note that the lower and upper parts of the # range constraint cannot have variables return data[0] <= (data[1] <= data[2]) @@ -304,7 +316,7 @@ def exitNode(self, node, data): return _handle_float(node, data, self.solver, self) else: raise NotImplementedError(f'unrecognized expression type: {nt}') - + def enterNode(self, node): if type(node) is RangedExpression: self.in_range = True @@ -316,13 +328,7 @@ def enterNode(self, node): class ScipDirectSolutionLoader(SolutionLoaderBase): def __init__( - self, - solver_model, - var_id_map, - var_map, - con_map, - pyomo_model, - opt, + self, solver_model, var_id_map, var_map, con_map, pyomo_model, opt ) -> None: super().__init__() self._solver_model = solver_model @@ -342,7 +348,9 @@ def get_solution_ids(self) -> List: def load_vars( self, vars_to_load: Optional[Sequence[VarData]] = None, solution_id=None ) -> None: - for v, val in self.get_vars(vars_to_load=vars_to_load, solution_id=solution_id).items(): + for v, val in self.get_vars( + vars_to_load=vars_to_load, solution_id=solution_id + ).items(): v.value = val def get_vars( @@ -377,22 +385,9 @@ def load_import_suffixes(self, solution_id=None): class ScipPersistentSolutionLoader(ScipDirectSolutionLoader): def __init__( - self, - solver_model, - var_id_map, - var_map, - con_map, - pyomo_model, - opt, + self, solver_model, var_id_map, var_map, con_map, pyomo_model, opt ) -> None: - super().__init__( - solver_model, - var_id_map, - var_map, - con_map, - pyomo_model, - opt, - ) + super().__init__(solver_model, var_id_map, var_map, con_map, pyomo_model, opt) self._valid = True def invalidate(self): @@ -401,7 +396,7 @@ def invalidate(self): def _assert_solution_still_valid(self): if not self._valid: raise RuntimeError('The results in the solver are no longer valid.') - + def load_vars( self, vars_to_load: Sequence[VarData] | None = None, solution_id=None ) -> None: @@ -454,11 +449,15 @@ def __init__(self, **kwds): self._params = {} # param id to param self._pyomo_var_to_solver_var_map = {} # var id to scip var self._pyomo_con_to_solver_con_map = {} - self._pyomo_param_to_solver_param_map = {} # param id to scip var with equal bounds + self._pyomo_param_to_solver_param_map = ( + {} + ) # param id to scip var with equal bounds self._pyomo_sos_to_solver_sos_map = {} self._expr_visitor = _PyomoToScipVisitor(self) self._objective = None # pyomo objective - self._obj_var = None # a scip variable because the objective cannot be nonlinear + self._obj_var = ( + None # a scip variable because the objective cannot be nonlinear + ) self._obj_con = None # a scip constraint (obj_var >= obj_expr) def _clear(self): @@ -476,7 +475,7 @@ def _clear(self): def available(self) -> Availability: if self._available is not None: return self._available - + if not scip_available: ScipDirect._available = Availability.NotFound elif self.version() < self._minimum_version: @@ -485,7 +484,7 @@ def available(self) -> Availability: ScipDirect._available = Availability.FullLicense return self._available - + def version(self) -> Tuple: return tuple(int(i) for i in scip.__version__.split('.')) @@ -493,9 +492,7 @@ def solve(self, model: BlockData, **kwds) -> Results: start_timestamp = datetime.datetime.now(datetime.timezone.utc) orig_config = self.config if not self.available(): - raise ApplicationError( - f'{self.name} is not available: {self.available()}' - ) + raise ApplicationError(f'{self.name} is not available: {self.available()}') try: config = self.config(value=kwds, preserve_implicit=True) @@ -546,7 +543,9 @@ def solve(self, model: BlockData, **kwds) -> Results: results.solver_log = ostreams[0].getvalue() end_timestamp = datetime.datetime.now(datetime.timezone.utc) results.timing_info.start_timestamp = start_timestamp - results.timing_info.wall_time = (end_timestamp - start_timestamp).total_seconds() + results.timing_info.wall_time = ( + end_timestamp - start_timestamp + ).total_seconds() results.timing_info.timer = timer return results @@ -616,7 +615,7 @@ def _add_var(self, var): self._vars[id(var)] = var self._pyomo_var_to_solver_var_map[id(var)] = scip_var return scip_var - + def _add_param(self, p): vtype = "C" lb = ub = p.value @@ -646,9 +645,7 @@ def _create_solver_model(self, model): self._solver_model = scip.Model() timer.start('collect constraints') cons = list( - model.component_data_objects( - Constraint, descend_into=True, active=True - ) + model.component_data_objects(Constraint, descend_into=True, active=True) ) timer.stop('collect constraints') timer.start('translate constraints') @@ -656,9 +653,7 @@ def _create_solver_model(self, model): timer.stop('translate constraints') timer.start('sos') sos = list( - model.component_data_objects( - SOSConstraint, descend_into=True, active=True - ) + model.component_data_objects(SOSConstraint, descend_into=True, active=True) ) self._add_sos_constraints(sos) timer.stop('sos') @@ -688,7 +683,9 @@ def _add_constraint(self, con): def _add_sos_constraint(self, con): level = con.level if level not in [1, 2]: - raise ValueError(f"{self.name} does not support SOS level {level} constraints") + raise ValueError( + f"{self.name} does not support SOS level {level} constraints" + ) scip_vars = [] weights = [] @@ -701,13 +698,9 @@ def _add_sos_constraint(self, con): weights.append(w) if level == 1: - scip_cons = self._solver_model.addConsSOS1( - scip_vars, weights=weights - ) + scip_cons = self._solver_model.addConsSOS1(scip_vars, weights=weights) else: - scip_cons = self._solver_model.addConsSOS2( - scip_vars, weights=weights - ) + scip_cons = self._solver_model.addConsSOS2(scip_vars, weights=weights) self._pyomo_con_to_solver_con_map[con] = scip_cons def _scip_vtype_from_var(self, var): @@ -737,9 +730,9 @@ def _scip_vtype_from_var(self, var): def _set_objective(self, obj): if self._obj_var is None: self._obj_var = self._solver_model.addVar( - lb=-self._solver_model.infinity(), - ub=self._solver_model.infinity(), - vtype="C" + lb=-self._solver_model.infinity(), + ub=self._solver_model.infinity(), + vtype="C", ) if self._obj_con is not None: @@ -766,19 +759,21 @@ def _set_objective(self, obj): self._objective = obj def _postsolve( - self, - scip_model, - solution_loader: ScipDirectSolutionLoader, - has_obj + self, scip_model, solution_loader: ScipDirectSolutionLoader, has_obj ): results = Results() results.solution_loader = solution_loader - results.timing_info.scip_time = scip_model.getSolvingTime() - results.termination_condition = self._get_tc_map().get(scip_model.getStatus(), TerminationCondition.unknown) - + results.timing_info.scip_time = scip_model.getSolvingTime() + results.termination_condition = self._get_tc_map().get( + scip_model.getStatus(), TerminationCondition.unknown + ) + if solution_loader.get_number_of_solutions() > 0: - if results.termination_condition == TerminationCondition.convergenceCriteriaSatisfied: + if ( + results.termination_condition + == TerminationCondition.convergenceCriteriaSatisfied + ): results.solution_status = SolutionStatus.optimal else: results.solution_status = SolutionStatus.feasible @@ -786,15 +781,18 @@ def _postsolve( results.solution_status = SolutionStatus.noSolution if ( - results.termination_condition + results.termination_condition != TerminationCondition.convergenceCriteriaSatisfied and self.config.raise_exception_on_nonoptimal_result ): raise NoOptimalSolutionError() - + if has_obj: try: - if scip_model.getNSols() > 0 and scip_model.getObjVal() < scip_model.infinity(): + if ( + scip_model.getNSols() > 0 + and scip_model.getObjVal() < scip_model.infinity() + ): results.incumbent_objective = scip_model.getObjVal() else: results.incumbent_objective = None @@ -831,7 +829,7 @@ def _postsolve( return results def _mipstart(self): - # TODO: it is also possible to specify continuous variables, but + # TODO: it is also possible to specify continuous variables, but # I think we should have a differnt option for that sol = self._solver_model.createPartialSol() for vid, scip_var in self._pyomo_var_to_solver_var_map.items(): @@ -1009,7 +1007,7 @@ def _add_sos_constraints(self, cons: List[SOSConstraintData]): self._check_reopt() self._invalidate_last_results() return super()._add_sos_constraints(cons) - + def _add_objectives(self, objs: List[ObjectiveData]): self._check_reopt() if len(objs) > 1: diff --git a/pyomo/contrib/solver/tests/solvers/test_solvers.py b/pyomo/contrib/solver/tests/solvers/test_solvers.py index 3665de4521a..e6686266028 100644 --- a/pyomo/contrib/solver/tests/solvers/test_solvers.py +++ b/pyomo/contrib/solver/tests/solvers/test_solvers.py @@ -1152,7 +1152,8 @@ def test_results_infeasible( ): res.solution_loader.get_duals() with self.assertRaisesRegex( - NoReducedCostsError, '.*does not currently have valid reduced costs.*' + NoReducedCostsError, + '.*does not currently have valid reduced costs.*', ): res.solution_loader.get_reduced_costs() From 960c531bba4c9bc9ea65cb652c699c5a131a8382 Mon Sep 17 00:00:00 2001 From: Michael Bynum Date: Mon, 6 Oct 2025 17:06:23 -0600 Subject: [PATCH 104/104] typos --- pyomo/contrib/solver/common/solution_loader.py | 2 +- pyomo/contrib/solver/solvers/scip/scip_direct.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyomo/contrib/solver/common/solution_loader.py b/pyomo/contrib/solver/common/solution_loader.py index f8723b6e0f4..6be23b63c77 100644 --- a/pyomo/contrib/solver/common/solution_loader.py +++ b/pyomo/contrib/solver/common/solution_loader.py @@ -63,7 +63,7 @@ def get_solution_ids(self) -> List[Any]: """ If there are multiple solutions available, this will return a list of the solution ids which can then be used with other - methods like `load_soltuion`. If only one solution is + methods like `load_solution`. If only one solution is available, this will return [None]. If no solutions are available, this will return None diff --git a/pyomo/contrib/solver/solvers/scip/scip_direct.py b/pyomo/contrib/solver/solvers/scip/scip_direct.py index 7e39d6e8595..05f39b0cb16 100644 --- a/pyomo/contrib/solver/solvers/scip/scip_direct.py +++ b/pyomo/contrib/solver/solvers/scip/scip_direct.py @@ -830,7 +830,7 @@ def _postsolve( def _mipstart(self): # TODO: it is also possible to specify continuous variables, but - # I think we should have a differnt option for that + # I think we should have a different option for that sol = self._solver_model.createPartialSol() for vid, scip_var in self._pyomo_var_to_solver_var_map.items(): pyomo_var = self._vars[vid]