diff --git a/doc/OnlineDocs/explanation/developer_utils/config.rst b/doc/OnlineDocs/explanation/developer_utils/config.rst index a6b959b0165..5a224fd3f08 100644 --- a/doc/OnlineDocs/explanation/developer_utils/config.rst +++ b/doc/OnlineDocs/explanation/developer_utils/config.rst @@ -10,9 +10,9 @@ The Pyomo configuration system provides a set of three classes configuration information and user input. The system is based around the :class:`ConfigValue` class, which provides storage for a single configuration entry. :class:`ConfigValue` objects can be grouped using -two containers (:class:`ConfigDict` and :class:`ConfigList`), which -provide functionality analogous to Python's dict and list classes, -respectively. +two containers (:class:`ConfigDict` and :class:`ConfigList`) that +provide functionality analogous to Python's :class:`dict` and +:class:`list` classes, respectively. At its simplest, the configuration system allows for developers to specify a dictionary of documented configuration entries: @@ -71,7 +71,8 @@ Domain validation All Config objects support a ``domain`` keyword that accepts a callable object (type, function, or callable instance). The domain callable -should take data and map it onto the desired domain, optionally +should take a single argument (the incoming data value) and map it onto +the desired domain, optionally performing domain validation (see :py:class:`ConfigValue`, :py:class:`ConfigDict`, and :py:class:`ConfigList` for more information). This allows client code to accept a very flexible set of @@ -266,7 +267,7 @@ In addition to basic storage and retrieval, the configuration system provides hooks to the argparse command-line argument parsing system. Individual configuration entries can be declared as :mod:`argparse` arguments using the :py:meth:`~ConfigBase.declare_as_argument` method. To make declaration -simpler, the :py:meth:`declare` method returns the declared configuration +simpler, the :py:meth:`~ConfigDict.declare` method returns the declared configuration object so that the argument declaration can be done inline: .. testcode:: @@ -373,18 +374,17 @@ were set but never retrieved (:py:meth:`unused_user_values`): >>> print([val.name() for val in config.unused_user_values()]) ['lbfgs', 'absolute tolerance'] -Generating output & documentation -================================= +Outputting the current state +============================ -Configuration objects support three methods for generating output and -documentation: :py:meth:`display()`, -:py:meth:`generate_yaml_template()`, and -:py:meth:`generate_documentation()`. The simplest is -:py:meth:`display()`, which prints out the current values of the -configuration object (and if it is a container type, all of its -children). :py:meth:`generate_yaml_template` is similar to -:py:meth:`display`, but also includes the description fields as -formatted comments. +Configuration objects support two methods for generating output: +:py:meth:`~ConfigBase.display` and +:py:meth:`~ConfigBase.generate_yaml_template`. The simpler is +:py:meth:`~ConfigBase.display`, which prints out the current values of +the configuration object (and if it is a container type, all of its +children). :py:meth:`~ConfigBase.generate_yaml_template` is similar to +:py:meth:`~ConfigBase.display`, but also includes the description fields +as formatted comments. .. testcode:: @@ -452,14 +452,31 @@ output: absolute tolerance: 0.2 # absolute convergence tolerance -The third method (:py:meth:`generate_documentation`) behaves -differently. This method is designed to generate reference -documentation. For each configuration item, the ``doc`` field is output. -If the item has no ``doc``, then the ``description`` field is used. +Generating documentation +======================== + +One of the most useful features of the Configuration system is the +ability to automatically generate documentation. To accomplish this, we +rely on a series of formatters derived from :class:`ConfigFormatter` +that implement a visitor pattern for walking the hierarchy of +configuration containers (:class:`ConfigDict` and :class:`ConfigList`) +and documenting the members. As the :class:`ConfigFormatter` was +designed to generate reference documentation, it behaves differently +from :meth:`~ConfigBase.display` or +:meth:`~ConfigBase.generate_yaml_template`): + + - For each configuration item, the ``doc`` field is output. If the + item has no ``doc``, then the ``description`` field is used. + + - List containers have their *domain* documented and not their + current values. -List containers have their *domain* documented and not their current -values. The documentation can be configured through optional arguments. -The defaults generate LaTeX documentation: +The simplest interface for generating documentation is to call the +:py:meth:`~ConfigBase.generate_documentation` method. This method +retrieves the specified formatter, instantiates it, and returns the +result from walking the configuration object. The documentation format +can be configured through optional arguments. The defaults generate +LaTeX documentation: .. doctest:: @@ -486,3 +503,15 @@ The defaults generate LaTeX documentation: \end{description} \end{description} + +More useful is actually documenting the source code itself. To this +end, the Configuration system provides three decorators that append +documentation of the referenced :class:`ConfigDict` (in +`numpydoc format `_) for the most +common situations: + +.. autosummary:: + + document_configdict + document_class_CONFIG + document_kwargs_from_configdict diff --git a/pyomo/common/config.py b/pyomo/common/config.py index f8e43135a8e..3b010ea3699 100644 --- a/pyomo/common/config.py +++ b/pyomo/common/config.py @@ -44,7 +44,7 @@ from operator import attrgetter -from pyomo.common.collections import Sequence, Mapping +from pyomo.common.collections import Sequence, MutableMapping from pyomo.common.deprecation import ( deprecated, deprecation_warning, @@ -53,6 +53,7 @@ from pyomo.common.fileutils import import_file from pyomo.common.flags import building_documentation, NOTSET from pyomo.common.formatting import wrap_reStructuredText +from pyomo.common.sorting import sorted_robust logger = logging.getLogger(__name__) @@ -714,6 +715,13 @@ def dump(x, **args): return str(x).lower() if type(x) is type: return str(type(x)) + if isinstance(x, str): + # If the str is a number, then we need to quote it. + try: + float(x) + return repr(x) + except: + return str(x) return str(x) assert '_dump' in globals() @@ -853,7 +861,7 @@ def _picklable(field, obj): # either: exceeding recursion depth raises a RuntimeError # through 3.4, then switches to a RecursionError (a derivative # of RuntimeError). - if isinstance(sys.exc_info()[0], RuntimeError): + if issubclass(sys.exc_info()[0], RuntimeError): raise if ftype not in _picklable.unknowable_types: _picklable.known[ftype] = False @@ -873,7 +881,7 @@ def _build_lexer(literals=''): # Ignore whitespace (space, tab, linefeed, and comma) t_ignore = " \t\r," - tokens = ["STRING", "WORD"] # quoted string # unquoted string + tokens = ["STRING", "WORD"] # [quoted string, unquoted string] # A "string" is a proper quoted string _quoted_str = r"'(?:[^'\\]|\\.)*'" @@ -1175,7 +1183,8 @@ def add_docstring_list(docstring, configdict, indent_by=4): class document_kwargs_from_configdict: - """Decorator to append the documentation of a ConfigDict to the docstring + """Decorator to append the documentation of a ConfigDict to a class, + method, or function docstring. This adds the documentation of the specified :py:class:`ConfigDict` (using the :py:class:`numpydoc_ConfigFormatter` formatter) to the @@ -1223,6 +1232,7 @@ class document_kwargs_from_configdict: ... ... @document_kwargs_from_configdict(CONFIG) ... def solve(self, **kwargs): + ... "Solve a model." ... config = self.CONFIG(kwargs) ... # ... ... @@ -1230,12 +1240,16 @@ class document_kwargs_from_configdict: Help on function solve: solve(self, **kwargs) + Solve a model. + Keyword Arguments ----------------- iterlim: int, default=3000 + Iteration limit. Specify None for no limit tee: bool, optional + If True, stream the solver output to the console """ @@ -1277,11 +1291,11 @@ def __call__(self, fcn): fcn.__doc__ = ( doc + f'{section}' - + config.generate_documentation( + + numpydoc_ConfigFormatter().generate( + config=config, indent_spacing=self.indent_spacing, width=self.width, visibility=self.visibility, - format='numpydoc', ) ) return fcn @@ -1315,8 +1329,69 @@ def method(self, *args, **kwargs): class document_configdict(document_kwargs_from_configdict): - """A simplified wrapper around :class:`document_kwargs_from_configdict` - for documenting classes derived from ConfigDict. + """Class decorator for documenting classes derived from :class:`ConfigDict`. + + This is a wrapper around :class:`document_kwargs_from_configdict` + for documenting classes derived from :class:`ConfigDict` with + pre-declared members. See :class:`document_kwargs_from_configdict` + for a description of the decorator arguments. + + Example + ------- + + .. testcode:: + + @document_configdict() + class MyConfig(ConfigDict): + 'Custom configuration object' + + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + super().__init__( + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + + self.iterlim = self.declare('iterlim', ConfigValue( + domain=int, doc='Solver iteration limit' + )) + + self.timeout = self.declare('timeout', ConfigValue( + domain=float, doc='Solver (wall clock) time limit' + )) + + Will result in + + .. doctest:: + + >>> help(MyConfig) + Help on class MyConfig ... + + class MyConfig(pyomo.common.config.ConfigDict) + | MyConfig(...) + | + | Custom configuration object + | + | Options + | ------- + | iterlim: int, optional + | + | Solver iteration limit + | + | timeout: float, optional + | + | Solver (wall clock) time limit + | + | ... """ @@ -1345,6 +1420,89 @@ def __call__(self, fcn): class document_class_CONFIG(document_kwargs_from_configdict): + """Class decorator for documenting ``CONFIG`` class attributes. + + This wrapper around the :class:`document_kwargs_from_configdict` + decorator will add the documentation generated from the target's + ``CONFIG`` class attribute to the main class docstring. + + In addition to the standard options accepted by + :class:`document_kwargs_from_configdict`, this decorator + also accepts ``methods``, an iterable of strings specifying methods + on the target class to also document as accepting the ``CONFIG`` + entries as keyword arguments. + + Example + ------- + + .. testcode:: + + @document_class_CONFIG(methods=['solve']) + class MyClass: + '''A class with a CONFIG class attribute.''' + + CONFIG = ConfigDict() + CONFIG.declare('iterlim', ConfigValue( + domain=int, doc='Solver iteration limit' + )) + CONFIG.declare('timeout', ConfigValue( + domain=float, doc='Solver (wall clock) time limit' + )) + + def solve(model, **kwargs): + "Solve the specified model" + config = self.CONFIG(kwargs) + + Will result in + + .. doctest:: + + >>> help(MyClass) + Help on class MyClass ... + + class MyClass(object) + | A class with a CONFIG class attribute. + | + | **Class configuration** + | + | This class leverages the Pyomo Configuration System for managing + | configuration options. See the discussion on :ref:`configuring class + | hierarchies ` for more information on how configuration + | class attributes, instance attributes, and method keyword arguments + | interact. + | + | .. _MyClass::CONFIG: + | + | CONFIG + | ------ + | iterlim: int, optional + | + | Solver iteration limit + | + | timeout: float, optional + | + | Solver (wall clock) time limit + | + | ... + + >>> help(MyClass.solve) + Help on function solve: + + solve(model, **kwargs) + Solve the specified model + + Keyword Arguments + ----------------- + iterlim: int, optional + + Solver iteration limit + + timeout: float, optional + + Solver (wall clock) time limit + + """ + def __init__( self, section='CONFIG', @@ -1387,19 +1545,20 @@ class attributes, instance attributes, and method keyword arguments self.preamble += f"\n\n.. _{ref}:\n" if self.methods: + method_documenter = document_kwargs_from_configdict( + self.config, + indent_spacing=self.indent_spacing, + width=self.width, + visibility=self.visibility, + doc=self.doc, + ) for method in self.methods: if method not in vars(cls): # If this method is inherited, we need to make a # "local" version of it so we don't change the # docstring on the base class. setattr(cls, method, _method_wrapper(getattr(cls, method))) - document_kwargs_from_configdict( - self.config, - indent_spacing=self.indent_spacing, - width=self.width, - visibility=self.visibility, - doc=self.doc, - )(getattr(cls, method)) + method_documenter(getattr(cls, method)) return super().__call__(cls) @@ -1549,8 +1708,6 @@ def __call__( description=NOTSET, doc=NOTSET, visibility=NOTSET, - implicit=NOTSET, - implicit_domain=NOTSET, preserve_implicit=False, ): # We will pass through overriding arguments to the constructor. @@ -1559,45 +1716,17 @@ def __call__( # that code here. Unfortunately, it means we need to do a bit # of logic to be sure we only pass through appropriate # arguments. - kwds = {} - fields = ('description', 'doc', 'visibility') - if isinstance(self, ConfigDict): - fields += (('implicit', '_implicit_declaration'), 'implicit_domain') - assert domain is NOTSET - assert default is NOTSET - else: - fields += ('domain',) - if default is NOTSET: - default = self.value() - if default is NOTSET: - default = None - kwds['default'] = default - assert implicit is NOTSET - assert implicit_domain is NOTSET - for field in fields: - if type(field) is tuple: - field, attr = field - else: - attr = '_' + field - if locals()[field] is NOTSET: - kwds[field] = getattr(self, attr, NOTSET) - else: - kwds[field] = locals()[field] + kwds = { + 'default': self.value() if default is NOTSET else default, + 'domain': self._domain if domain is NOTSET else domain, + 'description': self._description if description is NOTSET else description, + 'doc': self._doc if doc is NOTSET else doc, + 'visibility': self._visibility if visibility is NOTSET else visibility, + } # Initialize the new config object ans = self.__class__(**kwds) - if isinstance(self, ConfigDict): - # Copy over any Dict definitions - ans._domain = self._domain - for k, v in self._data.items(): - if preserve_implicit or k in self._declared: - ans._data[k] = _tmp = v(preserve_implicit=preserve_implicit) - if k in self._declared: - ans._declared.add(k) - _tmp._parent = ans - _tmp._name = v._name - # ... and set the value, if appropriate if value is not NOTSET: # Note that because we are *creating* a new Config object, @@ -1673,11 +1802,17 @@ def reset(self): def declare_as_argument(self, *args, **kwds): """Map this Config item to an argparse argument. - Valid arguments include all valid arguments to argparse's - ArgumentParser.add_argument() with the exception of 'default'. - In addition, you may provide a group keyword argument to either - pass in a pre-defined option group or subparser, or else pass in - the string name of a group, subparser, or (subparser, group). + Valid arguments include all valid arguments to + :meth:`argparse.ArgumentParser.add_argument()` with the exception of + ``default``. + + In addition, you may provide a `group` keyword argument that can be: + + - an argument group returned from + `~argparse.ArgumentParser.add_argument_group` + - a subparser returned from `~argparse.ArgumentParser.add_subparsers` + - a string specifying the name of a subparser or argument group + - a tuple of strings specifying a (subparser, group) """ @@ -1708,6 +1843,14 @@ def declare_as_argument(self, *args, **kwds): return self def initialize_argparse(self, parser): + """Initialize an :class:`~argparse.ArgumentParser` with arguments from + this Config object. + + Translate items from this Config object that have been marked + with :meth:`declare_as_argument` into :mod:`argparse` arguments. + + """ + def _get_subparser_or_group(_parser, name): # Note: strings also have a 'title()' method. We are # looking for things that look like argparse @@ -1772,6 +1915,7 @@ def _process_argparse_def(obj, _args, _kwds): _process_argparse_def(obj, _args, _kwds) def import_argparse(self, parsed_args): + """Import parsed arguments back into this Config object""" for level, prefix, value, obj in self._data_collector(None, ""): if obj._argparse is None: continue @@ -1790,10 +1934,19 @@ def import_argparse(self, parsed_args): def display( self, content_filter=None, indent_spacing=2, ostream=None, visibility=None ): + """Print the current Config value, in YAML format. + + The current values stored in this Config object are output to + ``ostream`` (or :attr:`sys.stdout` if ``ostream`` is ``None``). + If ``visibility`` is not ``None``, then only items with + visibility less than or equal to ``visibility`` will be output. + Output can be further filtered by providing a ``content_filter``. + + """ if content_filter not in ConfigDict.content_filters: raise ValueError( "unknown content filter '%s'; valid values are %s" - % (content_filter, ConfigDict.content_filters) + % (content_filter, sorted_robust(ConfigDict.content_filters)) ) _blocks = [] if ostream is None: @@ -1810,6 +1963,19 @@ def display( _blocks[i] = None def generate_yaml_template(self, indent_spacing=2, width=78, visibility=0): + """Document Config object, in YAML format. + + Output a deccription of this Config object. While similar to + :meth:`display`, this routine has two key differences: + + - The ``description`` for each item is output as a comment. + - The result is returned as a string instead of being sent + directly to an output stream + + If ``visibility`` is not ``None``, then only items with + visibility less than or equal to ``visibility`` will be output. + + """ minDocWidth = 20 comment = " # " data = list(self._data_collector(0, "", visibility)) @@ -1892,11 +2058,41 @@ def generate_documentation( item_start=None, item_body=None, item_end=None, - indent_spacing=2, - width=78, - visibility=None, - format='latex', + indent_spacing: int = 2, + width: int = 78, + visibility: int | None = None, + format: ConfigFormatter | str = 'latex', ): + """Document the this Config object. + + Generate documentation for this config object in the specified + format. While it can be called on any class derived from + :class:`ConfigBase`, it is typically used for documenting + :class:`ConfigDict` instances. + + Note that unlike :meth:`display` and + :meth:`generate_yaml_template`, :meth:`generate_documentation` + does not document the current value of any `ConfigList` + containers. Instead, it generates the documentation for the + :attr:`ConfigList` domain. + + If the ``format`` argument is a string, this method is equivalent to: + + .. code:: + + ConfigFormatter.formats[format]().generate( + self, indent_spacing, width, visibility + ) + + Otherwise, if ``format`` is a :class:`ConfigFormatter` instance, + then this is simply: + + .. code:: + + format.generate(self, indent_spacing, width, visibility) + + """ + if isinstance(format, str): formatter = ConfigFormatter.formats.get(format, None) if formatter is None: @@ -2313,7 +2509,7 @@ def _data_collector(self, level, prefix, visibility=None, docMode=False): ) -class ConfigDict(ConfigBase, Mapping): +class ConfigDict(ConfigBase, MutableMapping): """Store and manipulate a dictionary of configuration values. Parameters @@ -2394,6 +2590,55 @@ def __setstate__(self, state): for x in self._data.values(): x._parent = self + def __call__( + self, + value=NOTSET, + description=NOTSET, + doc=NOTSET, + visibility=NOTSET, + implicit=NOTSET, + implicit_domain=NOTSET, + preserve_implicit=False, + ): + # We will pass through overriding arguments to the constructor. + # This way if the constructor does special processing of any of + # the arguments (like implicit_domain), we don't have to repeat + # that code here. Unfortunately, it means we need to do a bit + # of logic to be sure we only pass through appropriate + # arguments. + kwds = { + 'description': self._description if description is NOTSET else description, + 'doc': self._doc if doc is NOTSET else doc, + 'visibility': self._visibility if visibility is NOTSET else visibility, + 'implicit': self._implicit_declaration if implicit is NOTSET else implicit, + 'implicit_domain': ( + self._implicit_domain if implicit_domain is NOTSET else implicit_domain + ), + } + + # Initialize the new config object + ans = self.__class__(**kwds) + + # Copy over any Dict definitions + ans._domain = self._domain + for k, v in self._data.items(): + if preserve_implicit or k in self._declared: + ans._data[k] = _tmp = v(preserve_implicit=preserve_implicit) + if k in self._declared: + ans._declared.add(k) + _tmp._parent = ans + _tmp._name = v._name + + # ... and set the value, if appropriate + if value is not NOTSET: + # Note that because we are *creating* a new Config object, + # we do not want set_value() to change the current (default) + # userSet flag for this object/container (see #3721). + tmp = ans._userSet + ans.set_value(value) + ans._userSet = tmp + return ans + def __dir__(self): # Note that dir() returns the *normalized* names (i.e., no spaces) return sorted(super(ConfigDict, self).__dir__() + list(self._data)) @@ -2537,10 +2782,11 @@ def _add(self, name, config): return config def declare(self, name, config): + """Declare a new configuration item in the :class:`ConfigDict`""" _name = str(name).replace(' ', '_') - ans = self._add(name, config) + self._add(name, config) self._declared.add(_name) - return ans + return config def declare_from(self, other, skip=None): if not isinstance(other, ConfigDict): @@ -2557,7 +2803,7 @@ def declare_from(self, other, skip=None): ) self.declare(key, other.get(key)()) - def add(self, name, config): + def add(self, name, config, **kwargs): if not self._implicit_declaration: raise ValueError( "Key '%s' not defined in ConfigDict '%s'" @@ -2568,11 +2814,18 @@ def add(self, name, config): if isinstance(config, ConfigBase): ans = self._add(name, config) else: - ans = self._add(name, ConfigValue(config)) + ans = self._add(name, ConfigValue(config, **kwargs)) + kwargs = None elif type(self._implicit_domain) is DynamicImplicitDomain: ans = self._add(name, self._implicit_domain(name, config)) else: ans = self._add(name, self._implicit_domain(config)) + if kwargs: + if self._implicit_domain is None: + why = f'user-provided {config.__class__.__name__}' + else: + why = 'implicit domain' + logger.warning(f"user-defined Config attributes {kwargs} ignored by {why}") ans._userSet = True # Adding something to the container should not change the # userSet on the container (see Pyomo/pyomo#352; now diff --git a/pyomo/common/enums.py b/pyomo/common/enums.py index d85adb774e0..bc6bef42d24 100644 --- a/pyomo/common/enums.py +++ b/pyomo/common/enums.py @@ -33,14 +33,14 @@ import re import sys -# A local alias for enum.Enum so that clients can use +# Aliases for enum members so that clients can use # :py:mod:`pyomo.common.enums` like they would :py:mod:`enum`. -Enum = enum.Enum +from enum import Enum, Flag if sys.version_info[:2] < (3, 11): - _EnumType = enum.EnumMeta + EnumType = enum.EnumMeta else: - _EnumType = enum.EnumType + EnumType = enum.EnumType if sys.version_info[:2] < (3, 13): import inspect @@ -53,40 +53,52 @@ def _rewrite(func): return _rewrite - class IntEnum(enum.IntEnum): - __doc__ = ( + class _int_bytes_doc_wrapper: + """Class to wrap to_bytes/from_bytes and rewrite the docstring""" + + @_fix_doc(int.to_bytes) + def to_bytes(self, /, length=1, byteorder='big', *, signed=False): + return super().to_bytes(length=length, byteorder=byteorder, signed=signed) + + # Note: we need to use a decorator to set the __doc__ *before* + # the @classmethod (which makes __doc__ read-only). + @classmethod + @_fix_doc(int.from_bytes) + def from_bytes(cls, bytes, byteorder='big', *, signed=False): + return super(_int_bytes_doc_wrapper, cls).from_bytes( + bytes, byteorder=byteorder, signed=signed + ) + + def _doc_updater(base): + return ( inspect.cleandoc( - """A compatibility wrapper around :class:`enum.IntEnum` + f"""A compatibility wrapper around :class:`enum.{base.__name__}` - This wrapper class updates the :meth:`to_bytes` and - :meth:`from_bytes` docstrings in Python <= 3.12 to suppress - warnings generated by Sphinx. + This wrapper class updates the :meth:`to_bytes` and + :meth:`from_bytes` docstrings in Python <= 3.12 to + suppress warnings generated by Sphinx. - .. rubric:: IntEnum + .. rubric:: {base.__name__} - """ + """ ) + "\n\n" # There are environments where IntEnum.__doc__ is None (see #3710) - + inspect.cleandoc(getattr(enum.IntEnum, "__doc__", "") or "") + + inspect.cleandoc(getattr(base, "__doc__", "") or "") ) - @_fix_doc(enum.IntEnum.to_bytes) - def to_bytes(self, /, length=1, byteorder='big', *, signed=False): - return super().to_bytes(length=length, byteorder=byteorder, signed=signed) + class IntEnum(_int_bytes_doc_wrapper, enum.IntEnum): + __doc__ = _doc_updater(enum.IntEnum) - # Note: we need to use a decorator to set the __doc__ *before* - # the @classmethod (which makes __doc__ read-only). - @classmethod - @_fix_doc(enum.IntEnum.from_bytes) - def from_bytes(cls, bytes, byteorder='big', *, signed=False): - return super()(bytes, byteorder=byteorder, signed=signed) + class IntFlag(_int_bytes_doc_wrapper, enum.IntFlag): + __doc__ = _doc_updater(enum.IntFlag) else: IntEnum = enum.IntEnum + IntFlag = enum.IntFlag -class ExtendedEnumType(_EnumType): +class ExtendedEnumType(EnumType): """Metaclass for creating an :py:class:`enum.Enum` that extends another Enum In general, :py:class:`enum.Enum` classes are not extensible: that is, diff --git a/pyomo/common/tests/test_config.py b/pyomo/common/tests/test_config.py index 193f8ab340d..1650c5ee724 100644 --- a/pyomo/common/tests/test_config.py +++ b/pyomo/common/tests/test_config.py @@ -37,6 +37,7 @@ from io import StringIO from pyomo.common.dependencies import yaml, yaml_available, yaml_load_args +from pyomo.common.tee import capture_output def yaml_load(arg): @@ -71,6 +72,7 @@ def yaml_load(arg): String_ConfigFormatter, document_kwargs_from_configdict, document_class_CONFIG, + document_configdict, add_docstring_list, USER_OPTION, DEVELOPER_OPTION, @@ -87,6 +89,14 @@ def _display(obj, *args): return test.getvalue() +class _Unpicklable: + def __getstate__(self): + raise RuntimeError("Pickling this should fail") + + def __call__(self, val): + return val + + class GlobalClass: "test class for test_known_types" @@ -1022,6 +1032,16 @@ def test_immutable_config_value(self): with self.assertRaisesRegex(RuntimeError, 'is currently immutable'): config.reset() + def test_lock_uninitialized(self): + cfg = ConfigDict() + arg = cfg.declare('arg', ConfigValue(default=5)) + self.assertIs(arg.__class__, ConfigValue._UninitializedClass) + + with MarkImmutable(arg): + self.assertEqual(5, cfg.arg) + self.assertIs(arg.__class__, ImmutableConfigValue) + self.assertIs(arg.__class__, ConfigValue) + class TestConfig(unittest.TestCase): def setUp(self): @@ -1030,6 +1050,7 @@ def setUp(self): self.original_environ, os.environ = os.environ, os.environ.copy() os.environ["COLUMNS"] = "80" + # This config was based on the WST flushing model configuration self.config = config = ConfigDict( "Basic configuration for Flushing models", implicit=True ) @@ -1310,6 +1331,53 @@ def test_template_3space_narrow(self): self.config, reference_template, indent_spacing=3, width=72 ) + def test_template_10space_narrow(self): + reference_template = """# Basic configuration for Flushing models +network: + epanet file: Net3.inp # EPANET network inp file +scenario: # Single scenario block + scenario file: Net3.tsg # Scenario generation file, see + # the TEVASIM documentation + merlion: false # Water quality model + detection: [1, 2, 3] # Sensor placement list, + # epanetID +scenarios: [] # List of scenario blocks +nodes: [] # List of node IDs +impact: + metric: MC # Population or network based + # impact metric +flushing: + flush nodes: + feasible nodes: ALL # ALL, NZD, NONE, list + # or filename + infeasible nodes: NONE # ALL, NZD, NONE, list + # or filename + max nodes: 2 # Maximum number of + # nodes to flush + rate: 600.0 # Flushing rate + # [gallons/min] + response time: 60.0 # Time [min] between + # detection and + # flushing + duration: 600.0 # Time [min] for + # flushing + close valves: + feasible pipes: ALL # ALL, DIAM min max + # [inch], NONE, list + # or filename + infeasible pipes: NONE # ALL, DIAM min max + # [inch], NONE, list + # or filename + max pipes: 2 # Maximum number of + # pipes to close + response time: 60.0 # Time [min] between + # detection and + # closing valves +""" + self._validateTemplate( + self.config, reference_template, indent_spacing=10, width=67 + ) + def test_display_default(self): reference = """network: epanet file: Net3.inp @@ -1335,8 +1403,17 @@ def test_display_default(self): max pipes: 2 response time: 60.0 """ - test = _display(self.config) - self.assertEqual(test, reference) + # test that output goes to stdout: + with capture_output() as OUT: + self.config.display() + self.assertEqual(OUT.getvalue(), reference) + + # test that we can directly capture the output + test = StringIO() + with capture_output() as OUT: + self.config.display(ostream=test) + self.assertEqual("", OUT.getvalue()) + self.assertEqual(test.getvalue(), reference) def test_display_list(self): reference = """network: @@ -1451,6 +1528,14 @@ def test_display_userdata_declare_block_nonDefault(self): test = _display(self.config, 'userdata') self.assertEqual(test, "bar:\n baz:\n") + def test_display_error(self): + with self.assertRaisesRegex( + ValueError, + "unknown content filter 'badfilter'; valid values are " + r"\[None, 'all', 'userdata'\]", + ): + self.config.display(content_filter='badfilter') + def test_unusedUserValues_default(self): test = '\n'.join(x.name(True) for x in self.config.unused_user_values()) self.assertEqual(test, "") @@ -1762,6 +1847,12 @@ def test_setValue_scalarList_badSubDomain(self): self.assertIs(type(val), list) self.assertEqual(val, [1, 2, 3]) + def test_setValue_list_scalardomain_str_parser(self): + self.config['nodes'] = "10, 5" + val = self.config['nodes'].value() + self.assertIs(type(val), list) + self.assertEqual(val, [10, 5]) + def test_setValue_list_scalardomain_list(self): self.config['nodes'] = [5, 10] val = self.config['nodes'].value() @@ -2565,6 +2656,33 @@ def test_argparse_help(self): help, ) + def test_argparse_multiple_args(self): + parser = argparse.ArgumentParser(prog='tester') + cfg = ConfigDict() + arg = cfg.declare('arg', ConfigValue(domain=bool, default=False)) + arg.declare_as_argument() + arg.declare_as_argument('--no-arg', action='store_false') + cfg.initialize_argparse(parser) + + self.assertEqual( + arg._argparse, + ( + (('--arg',), {'action': 'store_true', 'help': None}), + (('--no-arg',), {'action': 'store_false', 'help': None}), + ), + ) + help = parser.format_help() + self.assertEqual( + """usage: tester [-h] [--arg] [--no-arg] + +options: + -h, --help show this help message and exit + --arg + --no-arg +""", + help, + ) + def test_argparse_help_implicit_disable(self): self.config['scenario'].declare( 'epanet', @@ -2742,6 +2860,53 @@ def test_argparse_lists(self): ): leftovers = c.import_argparse(args) + def test_argparse_errors(self): + parser = argparse.ArgumentParser(prog='tester') + + # Cannot specify 'default' + config = ConfigDict() + with self.assertRaisesRegex( + TypeError, + "You cannot specify an argparse default value with " + "ConfigBase.declare_as_argument", + ): + config.declare('arg', ConfigValue()).declare_as_argument(default=5) + + # specify a bad group type + config = ConfigDict() + config.declare('arg', ConfigValue()).declare_as_argument(group=5) + with self.assertRaisesRegex( + RuntimeError, + r"Unknown datatype \(int\) for argparse group on configuration " + "definition arg", + ): + config.initialize_argparse(parser) + + # specify an undefined subparser + config = ConfigDict() + config.declare('arg1', ConfigValue()).declare_as_argument( + group=("missing", "arg group") + ) + with self.assertRaisesRegex( + RuntimeError, + "Could not find argparse subparser 'missing' for Config item arg1", + ): + config.initialize_argparse(parser) + + subp = parser.add_subparsers(title="missing").add_parser('missing') + config.initialize_argparse(parser) + + # specify an undefined sub-subparser + config = ConfigDict() + config.declare('arg2', ConfigValue()).declare_as_argument( + group=("missing", "subparser", "arg group") + ) + with self.assertRaisesRegex( + RuntimeError, + "Could not find argparse subparser 'subparser' for Config item arg", + ): + config.initialize_argparse(parser) + def test_getattr_setattr(self): config = ConfigDict() foo = config.declare('foo', ConfigDict(implicit=True, implicit_domain=int)) @@ -3001,6 +3166,14 @@ def cast(x): self.assertIn('dill', sys.modules) self.assertEqual(cfg2['lambda'], 6) + def test_pickle_error(self): + cfg = ConfigDict() + cfg.declare('fail', ConfigValue(domain=_Unpicklable(), default=5)) + + self.assertEqual(cfg.fail, 5) + with self.assertRaisesRegex(RuntimeError, "Pickling this should fail"): + pickle.dumps(cfg) + def test_unknowable_types(self): obj = ConfigValue() @@ -3380,12 +3553,14 @@ class _base: def fcn1(self, **kwargs): "Base class docstring 1" + return len(kwargs) def fcn2(self, **kwargs): "Base class docstring 2" + return sum(kwargs.values()) def fcn3(self, **kwargs): - pass + return ','.join(kwargs) @document_class_CONFIG(methods=['fcn1', 'fcn2', 'fcn3']) class _derived(_base): @@ -3393,6 +3568,7 @@ class _derived(_base): def fcn1(self, **kwargs): "Derived docstring 1" + return 10 * len(kwargs) self.assertEqual(_base.__doc__, None) self.assertEqual( @@ -3466,6 +3642,200 @@ class option 1 class option 2""", ) + # Verify that the overloaded / documented functions are callable + b = _base() + self.assertEqual(2, b.fcn1(arg1=5, arg2=10)) + self.assertEqual(15, b.fcn2(arg1=5, arg2=10)) + self.assertEqual('arg1,arg2', b.fcn3(arg1=5, arg2=10)) + d = _derived() + self.assertEqual(20, d.fcn1(arg1=5, arg2=10)) + self.assertEqual(15, d.fcn2(arg1=5, arg2=10)) + self.assertEqual('arg1,arg2', d.fcn3(arg1=5, arg2=10)) + + def test_domcument_configdict(self): + @document_configdict() + class CustomConfig(ConfigDict): + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + super().__init__( + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + + self.bool_option = self.declare( + 'bool_option', ConfigValue(domain=bool, default=False) + ) + + self.assertEqual( + """Options +------- +bool_option: bool, default=False""", + CustomConfig.__doc__, + ) + + @document_configdict() + class NestedConfig(ConfigDict): + def __init__( + self, + description=None, + doc=None, + implicit=False, + implicit_domain=None, + visibility=0, + ): + super().__init__( + description=description, + doc=doc, + implicit=implicit, + implicit_domain=implicit_domain, + visibility=visibility, + ) + + self.str_option = self.declare('str_option', ConfigValue(domain=str)) + + self.nested = self.declare('nested', CustomConfig()) + + self.assertEqual( + """Options +------- +str_option: str, optional + +nested: CustomConfig, optional""", + NestedConfig.__doc__, + ) + + def test_copy_configdict_default(self): + cfg = ConfigDict() + cfg.declare("arg_default", ConfigValue(domain=int, default=5)) + cfg.declare("arg_default_value", ConfigValue(domain=int, default=5)) + cfg.declare("arg_nodefault", ConfigValue(domain=int)) + cfg.declare("arg_nodefault_value", ConfigValue(domain=int)) + + newcfg = cfg({'arg_default_value': 10, 'arg_nodefault_value': 20}) + self.assertEqual(newcfg.get('arg_default')._default, 5) + self.assertEqual(newcfg.get('arg_default_value')._default, 5) + self.assertEqual(newcfg.get('arg_nodefault')._default, None) + self.assertEqual(newcfg.get('arg_nodefault_value')._default, None) + self.assertEqual(newcfg.get('arg_default')._data, 5) + self.assertEqual(newcfg.get('arg_default_value')._data, 10) + self.assertEqual(newcfg.get('arg_nodefault')._data, None) + self.assertEqual(newcfg.get('arg_nodefault_value')._data, 20) + + cfg.arg_default_value = 10 + cfg.arg_nodefault_value = 20 + newcfg = cfg() + self.assertEqual(newcfg.get('arg_default')._default, 5) + self.assertEqual(newcfg.get('arg_default_value')._default, 10) + self.assertEqual(newcfg.get('arg_nodefault')._default, None) + self.assertEqual(newcfg.get('arg_nodefault_value')._default, 20) + self.assertEqual(newcfg.get('arg_default')._data, 5) + self.assertEqual(newcfg.get('arg_default_value')._data, 10) + self.assertEqual(newcfg.get('arg_nodefault')._data, None) + self.assertEqual(newcfg.get('arg_nodefault_value')._data, 20) + + def test_configdict_add(self): + cfg = ConfigDict() + with self.assertRaisesRegex(ValueError, "Key 'arg' not defined"): + cfg.add('arg', 5) + + cfg = ConfigDict(implicit=True) + with LoggingIntercept() as LOG: + cfg.add('arg1', 5) + self.assertEqual(cfg.arg1, 5) + self.assertEqual("", LOG.getvalue()) + self.assertIs(cfg._data['arg1'].__class__, ConfigValue) + self.assertIs(cfg._data['arg1']._visibility, 0) + + with LoggingIntercept() as LOG: + cfg.add('arg2', 15, visibility=10) + self.assertEqual(cfg.arg2, 15) + self.assertEqual("", LOG.getvalue()) + self.assertIs(cfg._data['arg2'].__class__, ConfigValue) + self.assertIs(cfg._data['arg2']._visibility, 10) + + with LoggingIntercept() as LOG: + cfg.add('arg3', ConfigValue(default=25), visibility=10) + self.assertEqual(cfg.arg3, 25) + self.assertEqual( + "user-defined Config attributes {'visibility': 10} ignored by " + "user-provided UninitializedConfigValue\n", + LOG.getvalue(), + ) + self.assertIs(cfg._data['arg3'].__class__, ConfigValue) + self.assertIs(cfg._data['arg3']._visibility, 0) + + cfg = ConfigDict(implicit=True, implicit_domain=ConfigValue(domain=str)) + with LoggingIntercept() as LOG: + cfg.add('arg4', 35, visibility=10) + self.assertEqual(cfg.arg4, '35') + self.assertEqual( + "user-defined Config attributes {'visibility': 10} ignored by " + "implicit domain\n", + LOG.getvalue(), + ) + self.assertIs(cfg._data['arg4'].__class__, ConfigValue) + self.assertIs(cfg._data['arg4']._visibility, 0) + + def test_display_visibility(self): + cfg = ConfigDict() + cfg.declare('arg1', ConfigValue(default=1, visibility=0)) + cfg.declare('arg2', ConfigValue(default=2, visibility=10)) + cfg.declare('list1', ConfigList(default=3, domain=str, visibility=0)) + cfg.declare('list2', ConfigList(default=4, domain=str, visibility=10)) + d = cfg.declare('dict1', ConfigDict(visibility=0)) + d.declare('arg3', ConfigValue(default=5, visibility=0)) + d.declare('arg4', ConfigValue(default=6, visibility=10)) + d = cfg.declare('dict2', ConfigDict(visibility=10)) + d.declare('arg5', ConfigValue(default=7, visibility=0)) + d.declare('arg6', ConfigValue(default=8, visibility=10)) + + OUT = StringIO() + cfg.display(ostream=OUT) + self.assertEqual( + """arg1: 1 +arg2: 2 +list1: + - '3' +list2: + - '4' +dict1: + arg3: 5 + arg4: 6 +dict2: + arg5: 7 + arg6: 8 +""", + OUT.getvalue(), + ) + + OUT = StringIO() + cfg.display(ostream=OUT, visibility=0) + self.assertEqual( + """arg1: 1 +list1: + - '3' +dict1: + arg3: 5 +""", + OUT.getvalue(), + ) + + def test_ensure_blank_line(self): + dkfc = document_kwargs_from_configdict(None) + self.assertEqual(dkfc._ensure_blank_line(None), None) + self.assertEqual(dkfc._ensure_blank_line(""), "") + self.assertEqual(dkfc._ensure_blank_line("a"), "a\n\n") + self.assertEqual(dkfc._ensure_blank_line("b\n"), "b\n\n") + if __name__ == "__main__": unittest.main() diff --git a/pyomo/common/tests/test_enums.py b/pyomo/common/tests/test_enums.py index 844e7539685..37cd1ae9dc9 100644 --- a/pyomo/common/tests/test_enums.py +++ b/pyomo/common/tests/test_enums.py @@ -13,10 +13,15 @@ import pyomo.common.unittest as unittest -from pyomo.common.enums import ExtendedEnumType, ObjectiveSense, SolverAPIVersion +from pyomo.common.enums import ( + ExtendedEnumType, + ObjectiveSense, + SolverAPIVersion, + IntEnum, +) -class ProblemSense(enum.IntEnum, metaclass=ExtendedEnumType): +class ProblemSense(IntEnum, metaclass=ExtendedEnumType): __base_enum__ = ObjectiveSense unknown = 0 @@ -117,3 +122,14 @@ def test_call(self): ValueError, "'foo' is not a valid SolverAPIVersion" ): SolverAPIVersion('foo') + + +class TestEnumBackport(unittest.TestCase): + def test_bytes(self): + # Test that the Int portability wrappers (if present) define + # functional to_bytes / from_bytes + class TestEnum(IntEnum): + field = 100 + + self.assertEqual(TestEnum.field.to_bytes(), b'd') + self.assertIs(TestEnum.from_bytes(b'd'), TestEnum.field)