Merge branch 'main' into sens

Author: blnicho

pyomo/contrib/parmest/examples/semibatch/semibatch.py

@@ -284,6 +284,20 @@ def label_model(self):
 
         m = self.model
 
+        m.experiment_outputs = Suffix(direction=Suffix.LOCAL)
+        m.experiment_outputs.update(
+            (m.Ca[t], self.data["Ca_meas"][f"{t}"]) for t in m.measT
+        )
+        m.experiment_outputs.update(
+            (m.Cb[t], self.data["Cb_meas"][f"{t}"]) for t in m.measT
+        )
+        m.experiment_outputs.update(
+            (m.Cc[t], self.data["Cc_meas"][f"{t}"]) for t in m.measT
+        )
+        m.experiment_outputs.update(
+            (m.Tr[t], self.data["Tr_meas"][f"{t}"]) for t in m.measT
+        )
+
         m.unknown_parameters = Suffix(direction=Suffix.LOCAL)
         m.unknown_parameters.update(
             (k, ComponentUID(k)) for k in [m.k1, m.k2, m.E1, m.E2]

pyomo/contrib/parmest/parmest.py

@@ -284,13 +284,13 @@ def __init__(
         try:
             outputs = [k.name for k, v in model.experiment_outputs.items()]
         except:
-            RuntimeError(
+            raise RuntimeError(
                 'Experiment list model does not have suffix ' + '"experiment_outputs".'
             )
         try:
             params = [k.name for k, v in model.unknown_parameters.items()]
         except:
-            RuntimeError(
+            raise RuntimeError(
                 'Experiment list model does not have suffix ' + '"unknown_parameters".'
             )
 

pyomo/contrib/parmest/tests/test_parmest.py

@@ -80,6 +80,39 @@ def SSE(model):
             exp_list, obj_function=SSE, solver_options=solver_options, tee=True
         )
 
+    def test_parmest_exception(self):
+        """
+        Test the exception raised by parmest when the "experiment_outputs"
+        attribute is not defined in the model
+        """
+        from pyomo.contrib.parmest.examples.rooney_biegler.rooney_biegler import (
+            RooneyBieglerExperiment,
+        )
+
+        # create an instance of the RooneyBieglerExperiment class
+        # without the "experiment_outputs" attribute
+        class RooneyBieglerExperimentException(RooneyBieglerExperiment):
+            def label_model(self):
+                m = self.model
+
+                # add the unknown parameters
+                m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL)
+                m.unknown_parameters.update(
+                    (k, pyo.ComponentUID(k)) for k in [m.asymptote, m.rate_constant]
+                )
+
+        # create an experiment list
+        exp_list = []
+        for i in range(self.data.shape[0]):
+            exp_list.append(RooneyBieglerExperimentException(self.data.loc[i, :]))
+
+        # check the exception raised by parmest due to not defining
+        # the "experiment_outputs"
+        with self.assertRaises(RuntimeError) as context:
+            parmest.Estimator(exp_list, obj_function="SSE", tee=True)
+
+        self.assertIn("experiment_outputs", str(context.exception))
+
     def test_theta_est(self):
         objval, thetavals = self.pest.theta_est()
 
@@ -816,6 +849,22 @@ def label_model(self):
 
                 m = self.model
 
+                if isinstance(self.data, pd.DataFrame):
+                    meas_time_points = self.data.index
+                else:
+                    meas_time_points = list(self.data["ca"].keys())
+
+                m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL)
+                m.experiment_outputs.update(
+                    (m.ca[t], self.data["ca"][t]) for t in meas_time_points
+                )
+                m.experiment_outputs.update(
+                    (m.cb[t], self.data["cb"][t]) for t in meas_time_points
+                )
+                m.experiment_outputs.update(
+                    (m.cc[t], self.data["cc"][t]) for t in meas_time_points
+                )
+
                 m.unknown_parameters = pyo.Suffix(direction=pyo.Suffix.LOCAL)
                 m.unknown_parameters.update(
                     (k, pyo.ComponentUID(k)) for k in [m.k1, m.k2]
@@ -885,6 +934,51 @@ def get_labeled_model(self):
         self.m_df = ABC_model(data_df)
         self.m_dict = ABC_model(data_dict)
 
+        # create an instance of the ReactorDesignExperimentDAE class
+        # without the "unknown_parameters" attribute
+        class ReactorDesignExperimentException(ReactorDesignExperimentDAE):
+            def label_model(self):
+
+                m = self.model
+
+                if isinstance(self.data, pd.DataFrame):
+                    meas_time_points = self.data.index
+                else:
+                    meas_time_points = list(self.data["ca"].keys())
+
+                m.experiment_outputs = pyo.Suffix(direction=pyo.Suffix.LOCAL)
+                m.experiment_outputs.update(
+                    (m.ca[t], self.data["ca"][t]) for t in meas_time_points
+                )
+                m.experiment_outputs.update(
+                    (m.cb[t], self.data["cb"][t]) for t in meas_time_points
+                )
+                m.experiment_outputs.update(
+                    (m.cc[t], self.data["cc"][t]) for t in meas_time_points
+                )
+
+        # create an experiment list without the "unknown_parameters" attribute
+        exp_list_df_no_params = [ReactorDesignExperimentException(data_df)]
+        exp_list_dict_no_params = [ReactorDesignExperimentException(data_dict)]
+
+        self.exp_list_df_no_params = exp_list_df_no_params
+        self.exp_list_dict_no_params = exp_list_dict_no_params
+
+    def test_parmest_exception(self):
+        """
+        Test the exception raised by parmest when the "unknown_parameters"
+        attribute is not defined in the model
+        """
+        with self.assertRaises(RuntimeError) as context:
+            parmest.Estimator(self.exp_list_df_no_params, obj_function="SSE")
+
+        self.assertIn("unknown_parameters", str(context.exception))
+
+        with self.assertRaises(RuntimeError) as context:
+            parmest.Estimator(self.exp_list_dict_no_params, obj_function="SSE")
+
+        self.assertIn("unknown_parameters", str(context.exception))
+
     def test_dataformats(self):
         obj1, theta1 = self.pest_df.theta_est()
         obj2, theta2 = self.pest_dict.theta_est()

pyomo/neos/kestrel.py

@@ -94,8 +94,13 @@ def __del__(self):
         # Note that this is only to suppress warnings, as __del__ is not
         # guaranteed to be called (especially for any objects that still
         # exist when the Python process terminates)
-        if self.neos is not None:
-            self.transport.close()
+        if getattr(self, 'neos', None) is not None:
+            try:
+                if hasattr(self, 'transport') and hasattr(self.transport, 'close'):
+                    self.transport.close()
+            except Exception:
+                # never raise from a destructor
+                pass
 
     def setup_connection(self):
         import http.client
@@ -127,14 +132,16 @@ def setup_connection(self):
             xmlrpclib.ProtocolError,
             http.client.BadStatusLine,
             NotImplementedError,
-        ):
-            e = sys.exc_info()[1]
+            Exception,
+        ) as e:
+            self.connect_error = e
             self.neos = None
             logger.info("Fail: %s" % (e,))
             logger.warning("NEOS is temporarily unavailable:\n\t(%s)" % (e,))
 
     def tempfile(self):
-        return os.path.join(tempfile.gettempdir(), 'at%s.jobs' % os.getenv('ampl_id'))
+        ampl_id = os.getenv('ampl_id', 'unknown')
+        return os.path.join(tempfile.gettempdir(), f'at{ampl_id}.jobs')
 
     def kill(self, jobNumber, password):
         response = self.neos.killJob(jobNumber, password)
@@ -148,7 +155,7 @@ def solvers(self):
             while attempt < 3:
                 try:
                     return self.neos.listSolversInCategory("kestrel")
-                except socket.timeout:
+                except (socket.timeout, Exception):
                     attempt += 1
             return []
 
@@ -157,13 +164,14 @@ def retrieve(self, stub, jobNumber, password):
         results = self.neos.getFinalResults(jobNumber, password)
         if isinstance(results, xmlrpclib.Binary):
             results = results.data
+        if isinstance(results, str):
+            results = results.encode()
         # decode results to kestrel.sol; well try to anyway, any errors
         # will result in error strings in .sol file instead of solution.
         if stub[-4:] == '.sol':
             stub = stub[:-4]
-        solfile = open(stub + ".sol", "wb")
-        solfile.write(results)
-        solfile.close()
+        with open(stub + ".sol", "wb") as solfile:
+            solfile.write(results)
 
     def submit(self, xml):
         # LOGNAME and USER should map to the effective user (i.e., the
@@ -214,7 +222,13 @@ def getJobAndPassword(self):
 
     def getAvailableSolvers(self):
         """Return a list of all NEOS solvers that this interface supports"""
-        allKestrelSolvers = self.neos.listSolversInCategory("kestrel")
+        if self.neos is None:
+            return []
+        try:
+            allKestrelSolvers = self.neos.listSolversInCategory("kestrel")
+        except Exception as e:
+            logger.warning("Failed to retrieve solver list from NEOS: %s", e)
+            return []
         _ampl = ':AMPL'
         return sorted(s[: -len(_ampl)] for s in allKestrelSolvers if s.endswith(_ampl))
 
@@ -226,34 +240,36 @@ def getSolverName(self):
 
           - we don't want to be case sensitive, but NEOS is.
           - we need to read in options variable
-
         """
         # Get a list of available kestrel solvers from NEOS
         kestrelAmplSolvers = self.getAvailableSolvers()
-        self.options = None
+
+        NEOS_solver_name = None
+        m = None
+
         # Read kestrel_options to get solver name
         if "kestrel_options" in os.environ:
             self.options = os.getenv("kestrel_options")
         elif "KESTREL_OPTIONS" in os.environ:
             self.options = os.getenv("KESTREL_OPTIONS")
-        #
+        else:
+            self.options = None
+
         if self.options is not None:
             m = re.search(r'solver\s*=*\s*(\S+)', self.options, re.IGNORECASE)
-            NEOS_solver_name = None
             if m:
                 solver_name = m.groups()[0]
                 for s in kestrelAmplSolvers:
                     if s.upper() == solver_name.upper():
                         NEOS_solver_name = s
                         break
-                #
                 if not NEOS_solver_name:
                     raise RuntimeError(
                         "%s is not available on NEOS.  Choose from:\n\t%s"
                         % (solver_name, "\n\t".join(kestrelAmplSolvers))
                     )
-        #
-        if self.options is None or m is None:
+
+        if NEOS_solver_name is None:
             raise RuntimeError(
                 "%s is not available on NEOS.  Choose from:\n\t%s"
                 % (solver_name, "\n\t".join(kestrelAmplSolvers))
@@ -274,19 +290,16 @@ def formXML(self, stub):
         ampl_files = {}
         for key in ['adj', 'col', 'env', 'fix', 'spc', 'row', 'slc', 'unv']:
             if os.access(stub + "." + key, os.R_OK):
-                f = open(stub + "." + key, "r")
-                val = ""
-                buf = f.read()
-                while buf:
-                    val += buf
-                    buf = f.read()
-                f.close()
-                ampl_files[key] = val
+                with open(stub + "." + key, "r") as f:
+                    ampl_files[key] = f.read()
+
         # Get priority
         priority = ""
-        m = re.search(r'priority[\s=]+(\S+)', self.options)
-        if m:
-            priority = "<priority>%s</priority>\n" % (m.groups()[0])
+        if self.options:
+            m = re.search(r'priority[\s=]+(\S+)', self.options)
+            if m:
+                priority = "<priority>%s</priority>\n" % (m.groups()[0])
+
         # Add any AMPL-created environment variables to dictionary
         solver_options = "kestrel_options:solver=%s\n" % solver.lower()
         solver_options_key = "%s_options" % solver
@@ -298,7 +311,7 @@ def formXML(self, stub):
             solver_options_value = os.getenv(solver_options_key.lower())
         elif solver_options_key.upper() in os.environ:
             solver_options_value = os.getenv(solver_options_key.upper())
-        if not solver_options_value == "":
+        if solver_options_value:
             solver_options += "%s_options:%s\n" % (solver.lower(), solver_options_value)
         #
         nl_string = (base64.encodebytes(zipped_nl_file.getvalue())).decode('utf-8')