From d0eb9e9545bff03fa29beb5819bcf97294f57e1e Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Fri, 20 Sep 2024 13:53:15 -0400 Subject: [PATCH 01/75] refactor(createpackages): use jinja for mf6 module code generation --- autotest/test_codegen.py | 73 ++ docs/mf6_dev_guide.md | 52 +- flopy/mf6/data/dfn/utl-tas.dfn | 6 + flopy/mf6/data/dfn/utl-ts.dfn | 8 + flopy/mf6/data/mfdatastorage.py | 2 +- flopy/mf6/utils/codegen/context.py | 542 +++++++++ flopy/mf6/utils/codegen/dfn.py | 111 ++ flopy/mf6/utils/codegen/make.py | 95 ++ flopy/mf6/utils/codegen/ref.py | 97 ++ flopy/mf6/utils/codegen/render.py | 166 +++ flopy/mf6/utils/codegen/shim.py | 690 +++++++++++ flopy/mf6/utils/codegen/templates/attrs.jinja | 9 + .../utils/codegen/templates/context.py.jinja | 28 + .../utils/codegen/templates/docstring.jinja | 12 + .../codegen/templates/docstring_methods.jinja | 13 + .../codegen/templates/docstring_params.jinja | 9 + flopy/mf6/utils/codegen/templates/init.jinja | 18 + flopy/mf6/utils/codegen/templates/load.jinja | 58 + .../codegen/templates/package_container.jinja | 64 + flopy/mf6/utils/codegen/var.py | 59 + flopy/mf6/utils/createpackages.py | 1054 +---------------- flopy/mf6/utils/generate_classes.py | 9 +- pyproject.toml | 3 +- 23 files changed, 2103 insertions(+), 1075 deletions(-) create mode 100644 autotest/test_codegen.py create mode 100644 flopy/mf6/utils/codegen/context.py create mode 100644 flopy/mf6/utils/codegen/dfn.py create mode 100644 flopy/mf6/utils/codegen/make.py create mode 100644 flopy/mf6/utils/codegen/ref.py create mode 100644 flopy/mf6/utils/codegen/render.py create mode 100644 flopy/mf6/utils/codegen/shim.py create mode 100644 flopy/mf6/utils/codegen/templates/attrs.jinja create mode 100644 flopy/mf6/utils/codegen/templates/context.py.jinja create mode 100644 flopy/mf6/utils/codegen/templates/docstring.jinja create mode 100644 flopy/mf6/utils/codegen/templates/docstring_methods.jinja create mode 100644 flopy/mf6/utils/codegen/templates/docstring_params.jinja create mode 100644 flopy/mf6/utils/codegen/templates/init.jinja create mode 100644 flopy/mf6/utils/codegen/templates/load.jinja create mode 100644 flopy/mf6/utils/codegen/templates/package_container.jinja create mode 100644 flopy/mf6/utils/codegen/var.py diff --git a/autotest/test_codegen.py b/autotest/test_codegen.py new file mode 100644 index 0000000000..0378b7cede --- /dev/null +++ b/autotest/test_codegen.py @@ -0,0 +1,73 @@ +import pytest + +from autotest.conftest import get_project_root_path +from flopy.mf6.utils.codegen.context import get_context_names +from flopy.mf6.utils.codegen.dfn import Dfn +from flopy.mf6.utils.codegen.make import ( + DfnName, + make_all, + make_context, + make_targets, +) + +PROJ_ROOT = get_project_root_path() +MF6_PATH = PROJ_ROOT / "flopy" / "mf6" +TGT_PATH = MF6_PATH / "modflow" +DFN_PATH = MF6_PATH / "data" / "dfn" +DFN_NAMES = [ + dfn.stem + for dfn in DFN_PATH.glob("*.dfn") + if dfn.stem not in ["common", "flopy"] +] + + +@pytest.mark.parametrize("dfn_name", DFN_NAMES) +def test_dfn_load(dfn_name): + dfn_path = DFN_PATH / f"{dfn_name}.dfn" + with open(dfn_path, "r") as f: + dfn = Dfn.load(f, name=DfnName(*dfn_name.split("-"))) + if dfn_name in ["sln-ems", "exg-gwfprt", "exg-gwfgwe", "exg-gwfgwt"]: + assert not any(dfn) + else: + assert any(dfn) + + +@pytest.mark.parametrize( + "dfn, n_vars, n_flat, n_meta", + [("gwf-ic", 2, 2, 0), ("prt-prp", 18, 40, 1)], +) +def test_make_context(dfn, n_vars, n_flat, n_meta): + with open(DFN_PATH / "common.dfn") as f: + commonvars = Dfn.load(f) + + with open(DFN_PATH / f"{dfn}.dfn") as f: + dfn = DfnName(*dfn.split("-")) + definition = Dfn.load(f, name=dfn) + + context_names = get_context_names(dfn) + context_name = context_names[0] + context = make_context(context_name, definition, commonvars) + assert len(context_names) == 1 + assert len(context.variables) == n_vars + assert len(context.definition) == n_flat + assert len(context.definition.metadata) == n_meta + + +@pytest.mark.parametrize("dfn_name", DFN_NAMES) +def test_make_targets(dfn_name, function_tmpdir): + with open(DFN_PATH / "common.dfn") as f: + common = Dfn.load(f) + + with open(DFN_PATH / f"{dfn_name}.dfn", "r") as f: + dfn_name = DfnName(*dfn_name.split("-")) + dfn = Dfn.load(f, name=dfn_name) + + make_targets(dfn, function_tmpdir, commonvars=common) + for ctx_name in get_context_names(dfn_name): + source_path = function_tmpdir / ctx_name.target + assert source_path.is_file() + + +def test_make_all(function_tmpdir): + make_all(DFN_PATH, function_tmpdir, verbose=True) + assert any(function_tmpdir.glob("*.py")) diff --git a/docs/mf6_dev_guide.md b/docs/mf6_dev_guide.md index 61c364d3de..8f95ee9bbb 100644 --- a/docs/mf6_dev_guide.md +++ b/docs/mf6_dev_guide.md @@ -10,12 +10,17 @@ FPMF6 uses meta-data files located in flopy/mf6/data/dfn to define the model and All meta-data can be accessed from the flopy.mf6.data.mfstructure.MFStructure class. This is a singleton class, meaning only one instance of this class can be created. The class contains a sim_struct attribute (which is a flopy.mf6.data.mfstructure.MFSimulationStructure object) which contains all of the meta-data for all package files. Meta-data is stored in a structured format. MFSimulationStructure contains MFModelStructure and MFInputFileStructure objects, which contain the meta-data for each model type and each "simulation-level" package (tdis, ims, ...). MFModelStructure contains model specific meta-data and a MFInputFileStructure object for each package in that model. MFInputFileStructure contains package specific meta-data and a MFBlockStructure object for each block contained in the package file. MFBlockStructure contains block specific meta-data and a MFDataStructure object for each data structure defined in the block, and MFDataStructure contains data structure specific meta-data and a MFDataItemStructure object for each data item contained in the data structure. Data structures define the structure of data that is naturally grouped together, for example, the data in a numpy recarray. Data item structures define the structure of specific pieces of data, for example, a single column of a numpy recarray. The meta-data defined in these classes provides all the information FloPy needs to read and write MODFLOW 6 package and name files, create the Flopy interface, and check the data for various constraints. - -*** -MFStructure --+ MFSimulationStructure --+ MFModelStructure --+ MFInputFileStructure --+ MFBlockStructure --+ MFDataStructure --+ MFDataItemStructure - -Figure 1: FPMF6 generic data structure classes. Lines connecting classes show a relationship defined between the two connected classes. A "*" next to the class means that the class is a sub-class of the connected class. A "+" next to the class means that the class is contained within the connected class. -*** +```mermaid +classDiagram + MFStructure --* "1" MFSimulationStructure : has + MFSimulationStructure --* "1+" MFModelStructure : has + MFModelStructure --* "1" MFInputFileStructure : has + MFInputFileStructure --* "1+" MFBlockStructure : has + MFBlockStructure --* "1+" MFDataStructure : has + MFDataStructure --* "1+" MFDataItemStructure : has +``` + +Figure 1: Generic data structure hierarchy. Connections show composition relationships. Package and Data Base Classes ----------------------------------------------- @@ -23,25 +28,26 @@ Package and Data Base Classes The package and data classes are related as shown below in figure 2. On the top of the figure 2 is the MFPackage class, which is the base class for all packages. MFPackage contains generic methods for building data objects and reading and writing the package to a file. MFPackage contains a MFInputFileStructure object that defines how the data is structured in the package file. MFPackage also contains a dictionary of blocks (MFBlock). The MFBlock class is a generic class used to represent a block within a package. MFBlock contains a MFBlockStructure object that defines how the data in the block is structured. MFBlock also contains a dictionary of data objects (subclasses of MFData) contained in the block and a list of block headers (MFBlockHeader) for that block. Block headers contain the block's name and optionally data items (eg. iprn). -*** -MFPackage --+ MFBlock --+ MFData - -MFPackage --+ MFInputFileStructure - -MFBlock --+ MFBlockStructure - -MFData --+ MFDataStructure - -MFData --* MFArray --* MFTransientArray - -MFData --* MFList --* MFTransientList - -MFData --* MFScalar --* MFTransientScalar - -MFTransientData --* MFTransientArray, MFTransientList, MFTransientScalar +```mermaid +classDiagram + +MFPackage --* "1+" MFBlock : has +MFBlock --* "1+" MFData : has +MFPackage --* "1" MFInputFileStructure : has +MFBlock --* "1" MFBlockStructure : has +MFData --* "1" MFDataStructure : has +MFData --|> MFArray +MFArray --|> MFTransientArray +MFData --|> MFList +MFList --|> MFTransientList +MFData --|> MFScalar +MFScalar --|> MFTransientScalar +MFTransientData --|> MFTransientArray +MFTransientData --|> MFTransientList +MFTransientData --|> MFTransientScalar +``` Figure 2: FPMF6 package and data classes. Lines connecting classes show a relationship defined between the two connected classes. A "*" next to the class means that the class is a sub-class of the connected class. A "+" next to the class means that the class is contained within the connected class. -*** There are three main types of data, MFList, MFArray, and MFScalar data. All three of these data types are derived from the MFData abstract base class. MFList data is the type of data stored in a spreadsheet with different column headings. For example, the data describing a flow barrier are of type MFList. MFList data is stored in numpy recarrays. MFArray data is data of a single type (eg. all integer values). For example, the model's HK values are of type MFArray. MFArrays are stored in numpy ndarrays. MFScalar data is a single data item. Most MFScalar data are options. All MFData subclasses contain an MFDataStructure object that defines the expected structure and types of the data. diff --git a/flopy/mf6/data/dfn/utl-tas.dfn b/flopy/mf6/data/dfn/utl-tas.dfn index 6316beba5c..81c6fd25bc 100644 --- a/flopy/mf6/data/dfn/utl-tas.dfn +++ b/flopy/mf6/data/dfn/utl-tas.dfn @@ -19,6 +19,7 @@ type keyword shape reader urword optional false +in_record true longname description xxx @@ -29,6 +30,7 @@ shape any1d tagged false reader urword optional false +in_record true longname description Name by which a package references a particular time-array series. The name must be unique among all time-array series used in a package. @@ -48,6 +50,7 @@ type keyword shape reader urword optional false +in_record true longname description xxx @@ -59,6 +62,7 @@ shape tagged false reader urword optional false +in_record true longname description Interpolation method, which is either STEPWISE or LINEAR. @@ -78,6 +82,7 @@ type keyword shape reader urword optional false +in_record true longname description xxx @@ -88,6 +93,7 @@ shape time_series_name tagged false reader urword optional false +in_record true longname description Scale factor, which will multiply all array values in time series. SFAC is an optional attribute; if omitted, SFAC = 1.0. diff --git a/flopy/mf6/data/dfn/utl-ts.dfn b/flopy/mf6/data/dfn/utl-ts.dfn index a7165ea382..cb641256f2 100644 --- a/flopy/mf6/data/dfn/utl-ts.dfn +++ b/flopy/mf6/data/dfn/utl-ts.dfn @@ -20,6 +20,7 @@ type keyword shape reader urword optional false +in_record true longname description xxx @@ -30,6 +31,7 @@ shape any1d tagged false reader urword optional false +in_record true longname description Name by which a package references a particular time-array series. The name must be unique among all time-array series used in a package. @@ -49,6 +51,7 @@ type keyword shape reader urword optional false +in_record true longname description xxx @@ -59,6 +62,7 @@ valid stepwise linear linearend shape time_series_names tagged false reader urword +in_record true optional false longname description Interpolation method, which is either STEPWISE or LINEAR. @@ -108,6 +112,7 @@ name sfacs type keyword shape reader urword +in_record true optional false longname description xxx @@ -119,6 +124,7 @@ shape 1 and package_dim.model_dim[0].model_name is not None and package_dim.model_dim[0].model_name.lower() diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py new file mode 100644 index 0000000000..bbd95b457a --- /dev/null +++ b/flopy/mf6/utils/codegen/context.py @@ -0,0 +1,542 @@ +from ast import literal_eval +from dataclasses import dataclass +from keyword import kwlist +from os import PathLike +from typing import ( + Any, + Dict, + Iterator, + List, + NamedTuple, + Optional, + Union, +) + +from flopy.mf6.utils.codegen.dfn import Dfn, DfnName +from flopy.mf6.utils.codegen.ref import Ref, Refs +from flopy.mf6.utils.codegen.render import renderable +from flopy.mf6.utils.codegen.shim import SHIM +from flopy.mf6.utils.codegen.var import Var, VarKind, Vars + +_SCALAR_TYPES = { + "keyword", + "integer", + "double precision", + "string", +} + + +class ContextName(NamedTuple): + """ + Uniquely identifies an input context by its name, which + consists of a <= 3-letter left term and optional right + term also of <= 3 letters. + + Notes + ----- + A single `DefinitionName` may be associated with one or + more `ContextName`s. For instance, a model DFN file will + produce both a NAM package class and also a model class. + + From the `ContextName` several other things are derived, + including: + + - the input context class' name + - a description of the context class + - the name of the source file to write + - the base class the context inherits from + + """ + + l: str + r: Optional[str] + + @property + def title(self) -> str: + """ + The input context's unique title. This is not + identical to `f"{l}{r}` in some cases, but it + remains unique. The title is substituted into + the file name and class name. + """ + + l, r = self + if self == ("sim", "nam"): + return "simulation" + if l is None: + return r + if r is None: + return l + if l == "sim": + return r + if l in ["sln", "exg"]: + return r + return f"{l}{r}" + + @property + def base(self) -> str: + """Base class from which the input context should inherit.""" + _, r = self + if self == ("sim", "nam"): + return "MFSimulationBase" + if r is None: + return "MFModel" + return "MFPackage" + + @property + def target(self) -> str: + """The source file name to generate.""" + return f"mf{self.title}.py" + + @property + def description(self) -> str: + """A description of the input context.""" + l, r = self + title = self.title.title() + if self.base == "MFPackage": + return f"Modflow{title} defines a {r.upper()} package." + elif self.base == "MFModel": + return f"Modflow{title} defines a {l.upper()} model." + elif self.base == "MFSimulationBase": + return """ + MFSimulation is used to load, build, and/or save a MODFLOW 6 simulation. + A MFSimulation object must be created before creating any of the MODFLOW 6 + model objects.""" + + +def get_context_names(dfn_name: DfnName) -> List[ContextName]: + """ + Returns a list of contexts this definition produces. + + Notes + ----- + An input definition may produce one or more input contexts. + + Model definition files produce both a model class context and + a model namefile package context. The same goes for simulation + definition files. All other definition files produce a single + context. + """ + if dfn_name.r == "nam": + if dfn_name.l == "sim": + return [ + ContextName(None, dfn_name.r), # nam pkg + ContextName(*dfn_name), # simulation + ] + else: + return [ + ContextName(*dfn_name), # nam pkg + ContextName(dfn_name.l, None), # model + ] + elif (dfn_name.l, dfn_name.r) in [ + ("gwf", "mvr"), + ("gwf", "gnc"), + ("gwt", "mvt"), + ]: + return [ContextName(*dfn_name), ContextName(None, dfn_name.r)] + return [ContextName(*dfn_name)] + + +@renderable( + # shim for implementation details in the + # generated context classes which aren't + # really concerns of the core framework, + # and may eventually go away + **SHIM +) +@dataclass +class Context: + """ + An input context. Each of these is specified by a definition file + and becomes a generated class. A definition file may specify more + than one input context (e.g. model DFNs yield a model class and a + package class). + + Notes + ----- + A context class minimally consists of a name, a definition, and a + map of variables. The definition and variables are redundant (the + latter are generated from the former) but for now, the definition + is needed. When generated classes no longer reproduce definitions + verbatim, it can be removed. + + The context class may inherit from a base class, and may specify + a parent context within which it can be created (the parent then + becomes the first `__init__` method parameter). + + The context class may reference other contexts via foreign key + relations held by its variables, and may itself be referenced + by other contexts if desired. + + """ + + name: ContextName + definition: Dfn + variables: Vars + base: Optional[type] = None + parent: Optional[str] = None + description: Optional[str] = None + reference: Optional[Ref] = None + references: Optional[Refs] = None + + +def make_context( + name: ContextName, + definition: Dfn, + commonvars: Optional[Dfn] = None, + references: Optional[Refs] = None, +) -> Context: + """ + Extract from an input definition a context descriptor: + a structured representation of an input context class. + + Each input definition yields one or more input contexts. + The `name` parameter selects which context to make. + + A map of common variables may be provided, which can be + referenced in the given context's variable descriptions. + + A map of other definitions may be provided, in which case a + parameter in this context may act as kind of "foreign key", + identifying another context as a subpackage which this one + is related to. + + Notes + ----- + This function does most of the work in the whole module. + A bit of a beast, but convenient to use the outer scope + (including the input definition, etc) in inner functions + without sending around a lot of parameters. And it's not + complicated; we just map a variable specification from a + definition file to a corresponding Python representation. + """ + + _definition = dict(definition) + _commonvars = dict(commonvars or dict()) + _references = dict(references or dict()) + + # is this context referenceable? + reference = Ref.from_dfn(definition) + + # contexts referenced by this one + referenced = dict() + + def _parent() -> Optional[str]: + """ + Get a string parameter name for the context's parent(s), + i.e. context(s) which can own an instance of this one. + + If this context is a subpackage with multiple possible + parent types "x" and "y, this will be of form "x_or_y". + + """ + l, r = definition.name + if (l, r) == ("sim", "nam") and name == ("sim", "nam"): + return None + if l in ["sim", "exg", "sln"] or name.r is None: + return "simulation" + if reference: + if len(reference.parents) > 1: + return "_or_".join(reference.parents) + return reference.parents[0] + return "model" + + def _convert(var: Dict[str, Any], wrap: bool = False) -> Var: + """ + Transform a variable from its original representation in + an input definition to a Python specification appropriate + for generating an input context class. + + Notes + ----- + This involves expanding nested type hierarchies, mapping + types to roughly equivalent Python primitives/composites, + and other shaping. + + The rules for optional variable defaults are as follows: + If a `default_value` is not provided, keywords are `False` + by default, everything else is `None`. + + If `wrap` is true, scalars will be wrapped as records. + This is useful to distinguish among choices in unions. + + Any filepath variable whose name functions as a foreign key + for another context will be given a pointer to the context. + + """ + + _name = var["name"] + _type = var.get("type", None) + block = var.get("block", None) + shape = var.get("shape", None) + shape = None if shape == "" else shape + default = var.get("default", None) + descr = var.get("description", "") + + # if the var is a foreign key, register the referenced context + ref = _references.get(_name, None) + if ref: + referenced[_name] = ref + + def _description(descr: str) -> str: + """ + Make substitutions from common variable definitions, + remove backslashes, TODO: generate/insert citations. + """ + descr = descr.replace("\\", "") + _, replace, tail = descr.strip().partition("REPLACE") + if replace: + key, _, subs = tail.strip().partition(" ") + subs = literal_eval(subs) + cmn_var = _commonvars.get(key, None) + if cmn_var is None: + raise ValueError(f"Common variable not found: {key}") + descr = cmn_var.get("description", "") + if any(subs): + return descr.replace("\\", "").replace( + "{#1}", subs["{#1}"] + ) + return descr + return descr + + def _default(value: str) -> Any: + """ + Try to parse a default value as a literal. + """ + if _type != "string": + try: + return literal_eval(value) + except: + return value + + def _fields(record_name: str) -> Vars: + """Recursively load/convert a record's fields.""" + record = _definition[record_name] + field_names = record["type"].split()[1:] + fields: Dict[str, Var] = { + n: _convert(field, wrap=False) + for n, field in _definition.items() + if n in field_names + } + field_names = list(fields.keys()) + + # if the record represents a file... + if "file" in record_name: + # remove filein/fileout + for term in ["filein", "fileout"]: + if term in field_names: + fields.pop(term) + + # remove leading keyword + keyword = next(iter(fields), None) + if keyword: + fields.pop(keyword) + + # set the type + n = list(fields.keys())[0] + path_field = fields[n] + path_field._type = Union[str, PathLike] + fields[n] = path_field + + # if tagged, remove the leading keyword + elif record.get("tagged", False): + keyword = next(iter(fields), None) + if keyword: + fields.pop(keyword) + + return fields + + def _var() -> Var: + """ + Create the variable. + + Notes + ----- + Goes through all the possible input kinds + from top (composites) to bottom (scalars): + + - list + - union + - record + - array + - scalar + + Creates and returs a variable of the proper + kind. This may be a composite variable, in + which case nested variables are recursively + created as needed to produce the composite. + """ + + children = dict() + + # list input, child is the item type + if _type.startswith("recarray"): + # make sure columns are defined + names = _type.split()[1:] + n_names = len(names) + if n_names < 1: + raise ValueError(f"Missing recarray definition: {_type}") + + # list input can have records or unions as rows. + # lists which have a consistent record type are + # regular, inconsistent record types irregular. + + # regular tabular/columnar data (1 record type) can be + # defined with a nested record (i.e. explicit) or with + # fields directly inside the recarray (implicit). list + # data for unions/keystrings necessarily comes nested. + + is_explicit_record = len(names) == 1 and _definition[names[0]][ + "type" + ].startswith("record") + + def _is_implicit_scalar_record(): + # if the record is defined implicitly and it has + # only scalar fields + types = [ + v["type"] for n, v in _definition.items() if n in names + ] + return all(t in _SCALAR_TYPES for t in types) + + if is_explicit_record: + record_name = names[0] + record_spec = _definition[record_name] + record = _convert(record_spec, wrap=False) + children = {record_name: record} + kind = VarKind.List + elif _is_implicit_scalar_record(): + record_name = _name + fields = _fields(record_name) + record = Var( + name=record_name, + _type=_type.split()[0], + kind=VarKind.Record, + block=block, + children=fields, + description=descr, + ) + children = {record_name: record} + kind = VarKind.List + else: + # implicit complex record (i.e. some fields are records or unions) + fields = { + n: _convert(_definition[n], wrap=False) for n in names + } + first = list(fields.values())[0] + single = len(fields) == 1 + record_name = first.name if single else _name + record = Var( + name=record_name, + _type=_type.split()[0], + kind=VarKind.Record, + block=block, + children=first.children if single else fields, + description=descr, + ) + children = {record_name: record} + kind = VarKind.List + + # union (product), children are record choices + elif _type.startswith("keystring"): + names = _type.split()[1:] + children = { + n: _convert(_definition[n], wrap=True) for n in names + } + kind = VarKind.Union + + # record (sum), children are fields + elif _type.startswith("record"): + children = _fields(_name) + kind = VarKind.Record + + # are we wrapping a var into a record + # as a choice in a union? + elif wrap: + field_name = _name + field = _convert(var, wrap=False) + children = {field_name: field} + kind = VarKind.Record + + # at this point, if it has a shape, it's an array + elif shape is not None: + if _type not in _SCALAR_TYPES: + raise TypeError(f"Unsupported array type: {_type}") + elif _type == "string": + kind = VarKind.List + else: + kind = VarKind.Array + + # finally scalars + else: + kind = VarKind.Scalar + + # create var + return Var( + # if name is a reserved keyword, add a trailing underscore to it. + # convert dashes to underscores since it may become a class attr. + name=(f"{_name}_" if _name in kwlist else _name).replace( + "-", "_" + ), + _type=_type, + kind=kind, + block=block, + description=_description(descr), + default=_default(default), + children=children, + reference=ref, + ) + + return _var() + + def _variables() -> Vars: + """ + Return all input variables for an input context class. + + Notes + ----- + Not all variables become parameters; nested variables + will become components of composite parameters, e.g., + record fields, keystring (union) choices, list items. + + Variables may be added, depending on the context type. + """ + + vars_ = _definition.copy() + vars_ = { + name: _convert(var, wrap=False) + for name, var in vars_.items() + # skip composites, we already inflated + # their parents in the var hierarchy + if not var.get("in_record", False) + } + + # reset var name since we may have altered + # it when creating the variable e.g. to + # avoid a reserved keyword collision + return {v.name: v for v in vars_.values()} + + return Context( + name=name, + definition=definition, + variables=_variables(), + base=name.base, + parent=_parent(), + description=name.description, + reference=reference, + references=referenced, + ) + + +def make_contexts( + definition: Dfn, + commonvars: Optional[Dfn] = None, + references: Optional[Refs] = None, +) -> Iterator[Context]: + """Generate input contexts from the given input definition.""" + for name in get_context_names(definition.name): + yield make_context( + name=name, + definition=definition, + commonvars=commonvars, + references=references, + ) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py new file mode 100644 index 0000000000..6391a1f27c --- /dev/null +++ b/flopy/mf6/utils/codegen/dfn.py @@ -0,0 +1,111 @@ +from collections import UserList +from dataclasses import dataclass +from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Tuple + + +class DfnName(NamedTuple): + """ + Uniquely identifies an input definition by its name, which + consists of a <= 3-letter left term and an optional right + term, also <= 3 letters. + """ + + l: str + r: str + + +Metadata = List[str] + + +class Dfn(UserList): + """ + An MF6 input definition. + + Notes + ----- + This class is a list rather than a dictionary to + accommodate duplicate variable names. Dictionary + would be nicer; this constraint goes away if the + DFN specifications become nested instead of flat. + + With conversion to a standard format we get this + for free, and we could then drop the custom load. + """ + + name: Optional[DfnName] + metadata: Optional[Metadata] + + def __init__( + self, + variables: Optional[Iterable[Tuple[str, Dict[str, Any]]]] = None, + name: Optional[DfnName] = None, + metadata: Optional[Metadata] = None, + ): + super().__init__(variables) + self.name = name + self.metadata = metadata or [] + + @classmethod + def load(cls, f, name: Optional[DfnName] = None) -> "Dfn": + """ + Load an input definition from a definition file. + """ + + meta = None + vars_ = list() + var = dict() + + for line in f: + # remove whitespace/etc from the line + line = line.strip() + + # record context name and flopy metadata + # attributes, skip all other comment lines + if line.startswith("#"): + _, sep, tail = line.partition("flopy") + if sep == "flopy": + if meta is None: + meta = list() + tail = tail.strip() + if "solution_package" in tail: + tail = tail.split() + tail.pop(1) + meta.append(tail) + continue + _, sep, tail = line.partition("package-type") + if sep == "package-type": + if meta is None: + meta = list + meta.append(f"{sep} {tail.strip()}") + continue + _, sep, tail = line.partition("solution_package") + continue + + # if we hit a newline and the parameter dict + # is nonempty, we've reached the end of its + # block of attributes + if not any(line): + if any(var): + n = var["name"] + vars_.append((n, var)) + var = dict() + continue + + # split the attribute's key and value and + # store it in the parameter dictionary + key, _, value = line.partition(" ") + if key == "default_value": + key = "default" + if value in ["true", "false"]: + value = value == "true" + var[key] = value + + # add the final parameter + if any(var): + n = var["name"] + vars_.append((n, var)) + + return cls(variables=vars_, name=name, metadata=meta) + + +Dfns = Dict[str, Dfn] diff --git a/flopy/mf6/utils/codegen/make.py b/flopy/mf6/utils/codegen/make.py new file mode 100644 index 0000000000..298f68fbe4 --- /dev/null +++ b/flopy/mf6/utils/codegen/make.py @@ -0,0 +1,95 @@ +from pathlib import Path +from typing import ( + Optional, +) +from warnings import warn + +from jinja2 import Environment, FileSystemLoader + +# noqa: F401 +from flopy.mf6.utils.codegen.context import ( + get_context_names, + make_context, + make_contexts, +) +from flopy.mf6.utils.codegen.dfn import Dfn, DfnName, Dfns +from flopy.mf6.utils.codegen.ref import Ref, Refs + +_TEMPLATE_LOADER = FileSystemLoader(Path(__file__).parent / "templates") +_TEMPLATE_ENV = Environment(loader=_TEMPLATE_LOADER) +_TEMPLATE_NAME = "context.py.jinja" +_TEMPLATE = _TEMPLATE_ENV.get_template(_TEMPLATE_NAME) + + +def make_targets( + definition: Dfn, + outdir: Path, + commonvars: Optional[Dfn] = None, + references: Optional[Refs] = None, + verbose: bool = False, +): + """Generate Python source file(s) from the given input definition.""" + + for context in make_contexts( + definition=definition, commonvars=commonvars, references=references + ): + target = outdir / context.name.target + with open(target, "w") as f: + source = _TEMPLATE.render(**context.render()) + f.write(source) + if verbose: + print(f"Wrote {target}") + + +def make_all(dfndir: Path, outdir: Path, verbose: bool = False): + """Generate Python source files from the DFN files in the given location.""" + + # find definition files + paths = [ + p for p in dfndir.glob("*.dfn") if p.stem not in ["common", "flopy"] + ] + + # try to load common variables + common_path = dfndir / "common.dfn" + if not common_path.is_file: + warn("No common input definition file...") + common = None + else: + with open(common_path, "r") as f: + common = Dfn.load(f) + + # load all the input definitions before we generate input + # contexts so we can create foreign key refs between them. + dfns: Dfns = {} + refs: Refs = {} + for p in paths: + name = DfnName(*p.stem.split("-")) + with open(p) as f: + dfn = Dfn.load(f, name=name) + dfns[name] = dfn + ref = Ref.from_dfn(dfn) + if ref: + # key is the name of the file record + # that's the reference's foreign key + refs[ref.key] = ref + + # generate target files + for dfn in dfns.values(): + with open(p) as f: + make_targets( + definition=dfn, + outdir=outdir, + references=refs, + commonvars=common, + verbose=verbose, + ) + + # generate __init__.py file + init_path = outdir / "__init__.py" + with open(init_path, "w") as f: + for dfn in dfns.values(): + for ctx in get_context_names(dfn.name): + prefix = "MF" if ctx.base == "MFSimulationBase" else "Modflow" + f.write( + f"from .mf{ctx.title} import {prefix}{ctx.title.title()}\n" + ) diff --git a/flopy/mf6/utils/codegen/ref.py b/flopy/mf6/utils/codegen/ref.py new file mode 100644 index 0000000000..97a79f75e0 --- /dev/null +++ b/flopy/mf6/utils/codegen/ref.py @@ -0,0 +1,97 @@ +from dataclasses import dataclass +from typing import Dict, List, Optional +from warnings import warn + +from flopy.mf6.utils.codegen.dfn import Dfn + + +@dataclass +class Ref: + """ + A foreign-key-like reference between a file input variable + and another input definition. This allows an input context + to refer to another input context, by including a filepath + variable whose name acts as a foreign key for a different + input context. Extra parameters are added to the referring + context's `__init__` method so a selected "value" variable + defined in the referenced context can be provided directly + instead of the file path (foreign key) variable. + + Parameters + ---------- + key : str + The name of the foreign key file input variable. + val : str + The name of the selected variable in the referenced context. + abbr : str + An abbreviation of the referenced context's name. + param : str + The subpackage parameter name. TODO: explain + parents : List[Union[str, type]] + The subpackage's supported parent types. + """ + + key: str + val: str + abbr: str + param: str + parents: List[str] + description: Optional[str] + + @classmethod + def from_dfn(cls, dfn: Dfn) -> Optional["Ref"]: + if not dfn.metadata: + return None + + lines = { + "subpkg": next( + iter( + m + for m in dfn.metadata + if isinstance(m, str) and m.startswith("subpac") + ), + None, + ), + "parent": next( + iter( + m + for m in dfn.metadata + if isinstance(m, str) and m.startswith("parent") + ), + None, + ), + } + + def _subpkg(): + line = lines["subpkg"] + _, key, abbr, param, val = line.split() + matches = [v for _, v in dfn if v["name"] == val] + if not any(matches): + descr = None + else: + if len(matches) > 1: + warn(f"Multiple matches for referenced variable {val}") + match = matches[0] + descr = match.get("description", None) + + return { + "key": key, + "val": val, + "abbr": abbr, + "param": param, + "description": descr, + } + + def _parents(): + line = lines["parent"] + _, _, _type = line.split() + return [t.lower().replace("mf", "") for t in _type.split("/")] + + return ( + cls(**_subpkg(), parents=_parents()) + if all(v for v in lines.values()) + else None + ) + + +Refs = Dict[str, Ref] diff --git a/flopy/mf6/utils/codegen/render.py b/flopy/mf6/utils/codegen/render.py new file mode 100644 index 0000000000..e61d839b5f --- /dev/null +++ b/flopy/mf6/utils/codegen/render.py @@ -0,0 +1,166 @@ +from dataclasses import asdict +from enum import Enum +from typing import Any, Callable, Dict, Iterable, Optional, Tuple, Union + +Predicate = Callable[[Any], bool] +Transform = Callable[[Any], Dict[str, str]] +Pair = Tuple[str, Any] +Pairs = Iterable[Pair] + + +def _try_get_enum_value(v: Any) -> Any: + return v.value if isinstance(v, Enum) else v + + +def renderable( + maybe_cls=None, + *, + keep_none: Optional[Iterable[str]] = None, + quote_str: Optional[Iterable[str]] = None, + set_pairs: Optional[Iterable[Tuple[Predicate, Pairs]]] = None, + transform: Optional[Iterable[Tuple[Predicate, Transform]]] = None, +): + """ + Decorator for dataclasses which are meant + to be passed into a Jinja template. The + decorator adds a `.render()` method to + the decorated class, which recursively + converts the instance to a dictionary + with (by default) the `asdict()` builtin + `dataclasses` module function, plus a + few modifications to make the instance + easier to work with from the template. + + By default, attributes with value `None` + are dropped before conversion to a `dict`. + To specify that a given attribute should + remain even with a `None` value, use the + `keep_none` parameter. + + When a string value is to become the RHS + of an assignment or an argument-passing + expression, it needs to be wrapped with + quotation marks before insertion into + the template. To indicate an attribute's + value should be wrapped with quotation + marks, use the `quote_str` parameter. + + Arbitrary transformations of the instance + to which the decorator is applied can be + specified with the `transform` parameter, + which accepts a set of predicate/function + pairs; see below for more information on + how to use the transformation mechanism. + + Notes + ----- + This decorator is intended as a convenient + way to modify dataclass instances to make + them more palatable for templates. It also + aims to keep keep edge cases incidental to + the current design of MF6 input framework + cleanly isolated from the reimplementation + of which this code is a part. + + The basic idea behind this decorator is for + the developer to specify conditions in which + a given dataclass instance should be altered, + and a function to make the alteration. These + are provided as a collection of `Predicate`/ + `Transform` pairs. + + Transformations might be for convenience, or + to handle special cases where an object has + some other need for modification. + + Edge cases in the MF6 classes, e.g. the logic + determining the members of generated classes, + can be isolated as rendering transformations. + This allows keeping more general templating + infrastructure free of incidental complexity + while we move toward a leaner core framework. + + Jinja supports attribute- and dictionary- + based access on arbitrary objects but does + not support arbitrary expressions, and has + only a limited set of custom filters; this + can make it awkward to express some things, + which transformations can also remedy. + + Because a transformation function accepts an + instance of a dataclass and converts it to a + dictionary, only one transformation function + (the first predicate to match) is applied. + """ + + quote_str = quote_str or list() + keep_none = keep_none or list() + set_pairs = set_pairs or list() + transform = transform or list() + + def __renderable(cls): + def _render(d: dict) -> dict: + """ + Render the dictionary recursively, + with requested value modifications. + """ + + def _render_val(k, v): + v = _try_get_enum_value(v) + if ( + k in quote_str + and isinstance(v, str) + and v[0] not in ["'", '"'] + ): + v = f"'{v}'" + elif isinstance(v, dict): + v = _render(v) + return v + + return { + k: _render_val(k, v) + for k, v in d.items() + # drop nones except where requested to keep them + if (k in keep_none or v is not None) + } + + def _dict(o): + """ + Convert the dataclass instance to a dictionary, + applying a transformation if applicable and any + extra key/value pairs if provided. + """ + d = dict(o) + for p, t in transform: + if p(o): + d = t(o) + break + + for p, e in set_pairs: + if not p(d): + continue + if e is None: + raise ValueError(f"No value for key: {k}") + for k, v in e: + if callable(v): + v = v(d) + d[k] = v + + return d + + def _dict_factory(o): + return _render(_dict(o)) + + def render(self) -> dict: + """ + Recursively render the dataclass instance. + """ + return _render(asdict(self, dict_factory=_dict_factory)) + + setattr(cls, "render", render) + return cls + + # first arg value depends on the decorator usage: + # class if `@renderable`, `None` if `@renderable()`. + # referenced from https://github.com/python-attrs/attrs/blob/a59c5d7292228dfec5480388b5f6a14ecdf0626c/src/attr/_next_gen.py#L405C4-L406C65 + return __renderable if maybe_cls is None else __renderable(maybe_cls) diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py new file mode 100644 index 0000000000..e9840cd701 --- /dev/null +++ b/flopy/mf6/utils/codegen/shim.py @@ -0,0 +1,690 @@ +""" +The purpose of this module is to keep special handling +necessary to support the current `flopy.mf6` generated +classes separate from more general templating and code +generation infrastructure. +""" + +import os +from keyword import kwlist +from typing import List, Optional + +from flopy.mf6.utils.codegen.dfn import Metadata +from flopy.mf6.utils.codegen.var import VarKind + + +def _is_ctx(o) -> bool: + """Whether the object is an input context.""" + d = dict(o) + return "name" in d and "base" in d + + +def _is_var(o) -> bool: + """Whether the object is a input context variable.""" + d = dict(o) + return "name" in d and "_type" in d + + +def _is_init_param(o) -> bool: + """Whether the object is an `__init__` method parameter.""" + d = dict(o) + return not d.get("ref", None) + + +def _is_container_init_param(o) -> bool: + """ + Whether the object is a parameter of the corresponding + package container class. This is only relevant for some + subpackage contexts. + """ + return True + + +def _add_exg_params(ctx: dict) -> dict: + """ + Add initializer parameters for an exchange input context. + Exchanges need different parameters than a typical package. + """ + vars_ = ctx["variables"].copy() + vars_ = { + "loading_package": { + "name": "loading_package", + "_type": "bool", + "description": ( + "Do not set this parameter. It is intended for " + "debugging and internal processing purposes only." + ), + "default": False, + "init_param": True, + }, + "exgtype": { + "name": "exgtype", + "_type": "str", + "default": f"{ctx['name'].r[:3].upper()}6-{ctx['name'].r[:3].upper()}6", + "description": "The exchange type.", + "init_param": True, + }, + "exgmnamea": { + "name": "exgmnamea", + "_type": "str", + "description": "The name of the first model in the exchange.", + "default": None, + "init_param": True, + }, + "exgmnameb": { + "name": "exgmnameb", + "_type": "str", + "description": "The name of the second model in the exchange.", + "default": None, + "init_param": True, + }, + **vars_, + "filename": { + "name": "filename", + "_type": "pathlike", + "description": "File name for this package.", + "default": None, + "init_param": True, + }, + "pname": { + "name": "pname", + "_type": "str", + "description": "Package name for this package.", + "default": None, + "init_param": True, + }, + } + + if ctx["references"]: + for key, ref in ctx["references"].items(): + if key not in vars_: + continue + vars_[ref["val"]] = { + "name": ref["val"], + "description": ref.get("description", None), + "reference": ref, + "init_param": True, + "default": None, + "construct_package": ref["abbr"], + "construct_data": ref["val"], + "parameter_name": ref["param"], + } + + ctx["variables"] = vars_ + return ctx + + +def _add_pkg_params(ctx: dict) -> dict: + """Add variables for a package context.""" + vars_ = ctx["variables"].copy() + + if ctx["name"].r == "nam": + init_skip = ["export_netcdf", "nc_filerecord"] + elif ctx["name"] == ("utl", "ts"): + init_skip = ["method", "interpolation_method_single", "sfac"] + else: + init_skip = [] + for k in init_skip: + var = vars_.get(k, None) + if var: + var["init_param"] = False + var["init_skip"] = True + vars_[k] = var + + vars_ = { + "loading_package": { + "name": "loading_package", + "_type": "bool", + "description": ( + "Do not set this variable. It is intended for debugging " + "and internal processing purposes only." + ), + "default": False, + "init_param": True, + }, + **vars_, + "filename": { + "name": "filename", + "_type": "str", + "description": "File name for this package.", + "default": None, + "init_param": True, + }, + "pname": { + "name": "pname", + "_type": "str", + "description": "Package name for this package.", + "default": None, + "init_param": True, + }, + } + + if ctx["name"].l == "utl": + vars_["parent_file"] = { + "name": "parent_file", + "_type": "pathlike", + "description": ( + "Parent package file that references this package. Only needed " + "for utility packages (mfutl*). For example, mfutllaktab package " + "must have a mfgwflak package parent_file." + ), + } + + if ctx["references"]: + for key, ref in ctx["references"].items(): + if key not in vars_: + continue + vars_[key] = { + "name": ref["val"], + "description": ref.get("description", None), + "reference": ref, + "init_param": ctx["name"].r != "nam", + "default": None, + "construct_package": ref["abbr"], + "construct_data": ref["val"], + "parameter_name": ref["param"], + } + + ctx["variables"] = vars_ + return ctx + + +def _add_mdl_params(ctx: dict) -> dict: + """Add variables for a model context.""" + vars_ = ctx["variables"].copy() + init_skip = ["packages", "export_netcdf", "nc_filerecord"] + for k in init_skip: + var = vars_.get(k, None) + if var: + var["init_param"] = False + var["init_skip"] = True + vars_[k] = var + vars_ = { + "modelname": { + "name": "modelname", + "_type": "str", + "description": "The name of the model.", + "default": "model", + "init_param": True, + }, + "model_nam_file": { + "name": "model_nam_file", + "_type": "pathlike", + "default": None, + "description": ( + "The relative path to the model name file from model working folder." + ), + "init_param": True, + }, + "version": { + "name": "version", + "_type": "str", + "description": "The version of modflow", + "default": "mf6", + "init_param": True, + }, + "exe_name": { + "name": "exe_name", + "_type": "str", + "description": "The executable name.", + "default": "mf6", + "init_param": True, + }, + "model_rel_path": { + "name": "model_rel_path", + "_type": "pathlike", + "description": "The model working folder path.", + "default": os.curdir, + "init_param": True, + }, + **vars_, + } + + if ctx["references"]: + for key, ref in ctx["references"].items(): + if key not in vars_: + continue + vars_[key] = { + "name": ref["val"], + "description": ref.get("description", None), + "reference": ref, + "construct_package": ref["abbr"], + "construct_data": ref["val"], + "parameter_name": ref["param"], + } + + ctx["variables"] = vars_ + return ctx + + +def _add_sim_params(ctx: dict) -> dict: + """Add variables for a simulation context.""" + vars_ = ctx["variables"].copy() + init_skip = [ + "tdis6", + "models", + "exchanges", + "mxiter", + "solutiongroup", + ] + for k in init_skip: + var = vars_.get(k, None) + if var: + var["init_param"] = False + var["init_skip"] = True + vars_[k] = var + vars_ = { + "sim_name": { + "name": "sim_name", + "_type": "str", + "default": "sim", + "description": "Name of the simulation.", + "init_param": True, + }, + "version": { + "name": "version", + "_type": "str", + "default": "mf6", + "init_param": True, + }, + "exe_name": { + "name": "exe_name", + "_type": "pathlike", + "default": "mf6", + "init_param": True, + }, + "sim_ws": { + "name": "sim_ws", + "_type": "pathlike", + "default": ".", + "init_param": True, + }, + "verbosity_level": { + "name": "verbosity_level", + "_type": "int", + "default": 1, + "init_param": True, + }, + "write_headers": { + "name": "write_headers", + "_type": "bool", + "default": True, + "init_param": True, + }, + "use_pandas": { + "name": "use_pandas", + "_type": "bool", + "default": True, + "init_param": True, + }, + "lazy_io": { + "name": "lazy_io", + "_type": "bool", + "default": False, + "init_param": True, + }, + **vars_, + } + + if ctx["references"] and ctx["name"] != (None, "nam"): + for key, ref in ctx["references"].items(): + if key not in vars_: + continue + vars_[key] = { + "name": ref["param"], + "description": ref.get("description", None), + "reference": ref, + "init_param": True, + "default": None, + } + + ctx["variables"] = vars_ + return ctx + + +def _add_parent_param(ctx: dict) -> dict: + vars_ = ctx["variables"] + parent = ctx["parent"] + if ctx.get("reference"): + parent = f"parent_{parent}" + ctx["variables"] = { + parent: { + "name": parent, + "_type": str(ctx["parent"]), + "description": f"Parent {parent} that this package is part of.", + "init_param": True, + }, + **vars_, + } + return ctx + + +def _add_init_params(o): + """Add context-specific `__init__()` method parameters.""" + ctx = dict(o) + if ctx["name"].base == "MFSimulationBase": + ctx = _add_sim_params(ctx) + elif ctx["name"].base == "MFModel": + ctx = _add_mdl_params(ctx) + ctx = _add_parent_param(ctx) + elif ctx["name"].base == "MFPackage": + if ctx["name"].l == "exg": + ctx = _add_exg_params(ctx) + else: + ctx = _add_pkg_params(ctx) + ctx = _add_parent_param(ctx) + return ctx + + +def _transform_context(o): + # add vars depending on the + # specific type of context. + # do this as a transform so + # we can control the order + # they appear in `__init__` + # or other method signatures. + return _add_init_params(o) + + +def _var_attrs(ctx: dict) -> str: + """ + Get class attributes for the context. + """ + ctx_name = ctx["name"] + + def _attr(var: dict) -> Optional[str]: + var_name = var["name"] + var_kind = var.get("kind", None) + var_block = var.get("block", None) + + if var_kind is None or var_kind == VarKind.Scalar.value: + return None + + if var_name in ["cvoptions", "output"]: + return None + + if ( + ctx_name.l is not None and ctx_name.r == "nam" + ) and var_name != "packages": + return None + + if var_kind in [ + VarKind.List.value, + VarKind.Record.value, + VarKind.Union.value, + ]: + if not var_block: + raise ValueError("Need block") + args = [f"'{ctx_name.r}'", f"'{var_block}'", f"'{var_name}'"] + if ctx_name.l is not None and ctx_name.l not in [ + "sim", + "sln", + "utl", + "exg", + ]: + args.insert(0, f"'{ctx_name.l}6'") + return f"{var_name} = ListTemplateGenerator(({', '.join(args)}))" + + if var_kind == VarKind.Array.value: + if not var_block: + raise ValueError("Need block") + args = [f"'{ctx_name.r}'", f"'{var_block}'", f"'{var_name}'"] + if ctx_name.l is not None and ctx_name.l not in [ + "sim", + "sln", + "utl", + "exg", + ]: + args.insert(0, f"'{ctx_name.l}6'") + return f"{var_name} = ArrayTemplateGenerator(({', '.join(args)}))" + + return None + + attrs = [_attr(var) for var in ctx["variables"].values()] + return "\n ".join([a for a in attrs if a]) + + +def _init_body(ctx: dict) -> str: + """ + Get the `__init__` method body for the context. + """ + + def _super_call() -> Optional[str]: + """ + Whether to pass the variable to `super().__init__()` + by name in the `__init__` method. + """ + + if ctx["base"] == "MFPackage": + parent = ctx["parent"] + if ctx["reference"]: + parent = f"parent_{parent}" + pkgtyp = ctx["name"].r + args = [ + parent, + f"'{pkgtyp}'", + "filename", + "pname", + "loading_package", + "**kwargs", + ] + elif ctx["base"] == "MFModel": + parent = ctx["parent"] + mdltyp = ctx["name"].l + args = [ + parent, + f"'{mdltyp}6'", + "modelname=modelname", + "model_nam_file=model_nam_file", + "version=version", + "exe_name=exe_name", + "model_rel_path=model_rel_path", + "**kwargs", + ] + elif ctx["base"] == "MFSimulationBase": + args = [ + "sim_name=sim_name", + "version=version", + "exe_name=exe_name", + "sim_ws=sim_ws", + "verbosity_level=verbosity_level", + "write_headers=write_headers", + "lazy_io=lazy_io", + "use_pandas=use_pandas", + ] + + return f"super().__init__({', '.join(args)})" + + def _should_assign(var: dict) -> bool: + """ + Whether to assign arguments to self in the + `__init__` method. if this is false, assume + the template has conditionals for any more + involved initialization needs. + """ + return var["name"] in ["exgtype", "exgmnamea", "exgmnameb"] + + def _should_build(var: dict) -> bool: + """ + Whether to call `build_mfdata()` on the variable. + in the `__init__` method. + """ + if var.get("reference", None): + return False + name = var["name"] + if name in [ + "simulation", + "model", + "package", + "parent_model", + "parent_package", + "loading_package", + "parent_model_or_package", + "exgtype", + "exgmnamea", + "exgmnameb", + "filename", + "pname", + "parent_file", + "modelname", + "model_nam_file", + "version", + "exe_name", + "model_rel_path", + "sim_name", + "sim_ws", + "verbosity_level", + "write_headers", + "use_pandas", + "lazy_io", + "export_netcdf", + "nc_filerecord", + "method", + "interpolation_method_single", + "sfac", + "output", + ]: + return False + return True + + def _body() -> Optional[str]: + if ctx["base"] in ["MFSimulationBase", "MFModel"]: + statements = [] + references = {} + for var in ctx["variables"].values(): + if not var.get("kind", None) or var.get("init_skip", False): + continue + name = var["name"] + if name in kwlist: + name = f"{name}_" + ref = var.get("reference", None) + statements.append(f"self.name_file.{name}.set_data({name})") + statements.append(f"self.{name} = self.name_file.{name}") + if ref and ref["key"] not in references: + references[ref["key"]] = ref + statements.append( + f"self._{ref['param']} = self._create_package('{ref['abbr']}', {ref['param']})" + ) + else: + statements = [] + references = {} + for var in ctx["variables"].values(): + name = var["name"] + ref = var.get("reference", None) + if name in kwlist: + name = f"{name}_" + + if _should_assign(var): + statements.append(f"self.{name} = {name}") + if name == "exgmnameb": + statements.append( + "simulation.register_exchange_file(self)" + ) + elif _should_build(var): + lname = name[:-1] if name.endswith("_") else name + statements.append( + f"self.{'_' if ref else ''}{name} = self.build_mfdata('{lname}', {name if var.get('init_param', True) else 'None'})" + ) + + if ( + ref + and ref["key"] not in references + and ctx["name"].r != "nam" + ): + references[ref["key"]] = ref + statements.append( + f"self._{ref['key']} = self.build_mfdata('{ref['key']}', None)" + ) + statements.append( + f"self._{ref['abbr']}_package = self.build_child_package('{ref['abbr']}', {ref['val']}, '{ref['param']}', self._{ref['key']})" + ) + + return ( + None + if not any(statements) + else "\n".join([" " + s for s in statements]) + ) + + sections = [_super_call(), _body()] + sections = [s for s in sections if s] + return "\n".join(sections) + + +def _dfn(o) -> List[Metadata]: + """ + Get a list of the class' original definition attributes + as a partial, internal reproduction of the DFN contents. + + Notes + ----- + Currently, generated classes have a `.dfn` property that + reproduces the corresponding DFN sans a few attributes. + This represents the DFN in raw form, before adapting to + Python, consolidating nested types, etc. + """ + + ctx = dict(o) + dfn = ctx["definition"] + + def _fmt_var(var: dict) -> List[str]: + exclude = ["longname", "description"] + + def _fmt_name(k, v): + return v.replace("-", "_") if k == "name" else v + + return [ + " ".join([k, str(_fmt_name(k, v))]).strip() + for k, v in var.items() + if k not in exclude + ] + + meta = dfn.metadata or list() + _dfn = [] + for name, var in dfn: + var_ = ctx["variables"].get(name, None) + if var_ and "construct_package" in var_: + var["construct_package"] = var_["construct_package"] + var["construct_data"] = var_["construct_data"] + var["parameter_name"] = var_["parameter_name"] + _dfn.append((name, var)) + return [["header"] + [m for m in meta]] + [_fmt_var(v) for k, v in _dfn] + + +def _qual_base(ctx: dict): + base = ctx["base"] + if base == "MFSimulationBase": + module = "mfsimbase" + elif base == "MFModel": + module = "mfmodel" + else: + module = "mfpackage" + return f"{module}.{base}" + + +SHIM = { + "keep_none": ["default", "block", "metadata"], + "quote_str": ["default"], + "set_pairs": [ + ( + _is_ctx, + [ + ("dfn", _dfn), + ("qual_base", _qual_base), + ("var_attrs", _var_attrs), + ("init_body", _init_body), + ], + ), + ( + _is_var, + [ + ("init_param", _is_init_param), + ("container_init_param", _is_container_init_param), + ], + ), + ], + "transform": [(_is_ctx, _transform_context)], +} +""" +Arguments for `renderable` as applied to `Context` +to support the current `flopy.mf6` input framework. +""" diff --git a/flopy/mf6/utils/codegen/templates/attrs.jinja b/flopy/mf6/utils/codegen/templates/attrs.jinja new file mode 100644 index 0000000000..86d8170006 --- /dev/null +++ b/flopy/mf6/utils/codegen/templates/attrs.jinja @@ -0,0 +1,9 @@ + {%- if base == "MFModel" %} + model_type = "{{ name.title }}" + {%- elif base == "MFPackage" %} + {{ var_attrs }} + package_abbr = "{% if name.l != "sln" and name.l != "sim" and name.l != "exg" and name.l is not none %}{{ name.l }}{% endif %}{{ name.r }}" + _package_type = "{{ name.r }}" + dfn_file_name = "{% if name.l is not none %}{{ name.l }}-{% elif name.l is none %}sim-{% endif %}{{ name.r }}.dfn" + dfn = {{ dfn|pprint|indent(10) }} + {% endif -%} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/context.py.jinja b/flopy/mf6/utils/codegen/templates/context.py.jinja new file mode 100644 index 0000000000..89fc1140eb --- /dev/null +++ b/flopy/mf6/utils/codegen/templates/context.py.jinja @@ -0,0 +1,28 @@ +# autogenerated file, do not modify +from os import PathLike, curdir +import typing +import numpy as np +from typing import Any, Optional, Tuple, List, Dict, Union, Literal, Iterable +from numpy.typing import NDArray + +from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator +from flopy.mf6 import mfpackage +from flopy.mf6 import mfmodel +{# avoid circular import; some pkgs (e.g. mfnam) are used by mfsimbase.py #} +{% if base == "MFSimulationBase" %} +from flopy.mf6 import mfsimbase +{% endif %} + +class {% if base == "MFSimulationBase" %}MF{% else %}Modflow{% endif %}{{ name.title.title() }}({{ qual_base }}): + {% include "docstring.jinja" %} + + {% include "attrs.jinja" %} + + {% include "init.jinja" %} + + {% include "load.jinja" %} + +{# TODO: cleaner way to filter out hpc subpkgs? #} +{% if reference is defined and name.r != "hpc" %} +{% include "package_container.jinja" %} +{% endif %} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/docstring.jinja b/flopy/mf6/utils/codegen/templates/docstring.jinja new file mode 100644 index 0000000000..488b567d45 --- /dev/null +++ b/flopy/mf6/utils/codegen/templates/docstring.jinja @@ -0,0 +1,12 @@ +""" + {{ description }} + + Parameters + ---------- + {% include "docstring_params.jinja" %} + + Methods + ------- + {% include "docstring_methods.jinja" %} + """ + diff --git a/flopy/mf6/utils/codegen/templates/docstring_methods.jinja b/flopy/mf6/utils/codegen/templates/docstring_methods.jinja new file mode 100644 index 0000000000..41daf5715d --- /dev/null +++ b/flopy/mf6/utils/codegen/templates/docstring_methods.jinja @@ -0,0 +1,13 @@ +{% if base == "MFSimulationBase" %} + load : (sim_name : str, version : string, + exe_name : str or PathLike, sim_ws : str or PathLike, strict : bool, + verbosity_level : int, load_only : list, verify_data : bool, + write_headers : bool, lazy_io : bool, use_pandas : bool, + ) : MFSimulation + a class method that loads a simulation from files +{% elif base == "MFModel" %} + load : (simulation : MFSimulationData, model_name : string, + namfile : string, version : string, exe_name : string, + model_ws : string, strict : boolean) : MFSimulation + a class method that loads a model from files +{% endif %} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/docstring_params.jinja b/flopy/mf6/utils/codegen/templates/docstring_params.jinja new file mode 100644 index 0000000000..3afb3be69f --- /dev/null +++ b/flopy/mf6/utils/codegen/templates/docstring_params.jinja @@ -0,0 +1,9 @@ +{%- for v in variables.values() recursive %} + {% if loop.depth > 1 %}* {% endif %}{{ v.name }}{% if v._type is defined and v._type is not none %} : {{ v._type }}{% endif %} +{%- if v.description is defined and v.description is not none %} +{{ v.description|wordwrap|indent(4 + (loop.depth * 4), first=True) }} +{%- endif %} +{%- if v.children is defined and v.children is not none -%} +{{ loop(v.children.values())|indent(4) }} +{%- endif %} +{% endfor -%} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/init.jinja b/flopy/mf6/utils/codegen/templates/init.jinja new file mode 100644 index 0000000000..f51d0fad7c --- /dev/null +++ b/flopy/mf6/utils/codegen/templates/init.jinja @@ -0,0 +1,18 @@ +def __init__( + self, + {%- for var in variables.values() if var.init_param %} + {%- if var.default is defined %} + {{ var.name }}={{ var.default }}, + {%- else %} + {{ var.name }}, + {%- endif -%} + {%- endfor %} + **kwargs, + ): + {{ init_body }} + {% if name.l == "exg" and n == "exgmnameb" -%} + parent.register_exchange_file(self) + {% endif -%} + {% if base == "MFPackage" %} + self._init_complete = True + {% endif %} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/load.jinja b/flopy/mf6/utils/codegen/templates/load.jinja new file mode 100644 index 0000000000..e36e13c64e --- /dev/null +++ b/flopy/mf6/utils/codegen/templates/load.jinja @@ -0,0 +1,58 @@ +{% if base == "MFSimulationBase" %} + @classmethod + def load( + cls, + sim_name="modflowsim", + version="mf6", + exe_name: Union[str, PathLike] = "mf6", + sim_ws: Union[str, PathLike] = curdir, + strict=True, + verbosity_level=1, + load_only=None, + verify_data=False, + write_headers=True, + lazy_io=False, + use_pandas=True, + ): + return mfsimbase.MFSimulationBase.load( + cls, + sim_name, + version, + exe_name, + sim_ws, + strict, + verbosity_level, + load_only, + verify_data, + write_headers, + lazy_io, + use_pandas, + ) +{% elif base == "MFModel" %} + @classmethod + def load( + cls, + simulation, + structure, + modelname="NewModel", + model_nam_file="modflowtest.nam", + version="mf6", + exe_name="mf6", + strict=True, + model_rel_path=curdir, + load_only=None, + ): + return mfmodel.MFModel.load_base( + cls, + simulation, + structure, + modelname, + model_nam_file, + "{{ name.title }}6", + version, + exe_name, + strict, + model_rel_path, + load_only, + ) +{% endif %} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/package_container.jinja b/flopy/mf6/utils/codegen/templates/package_container.jinja new file mode 100644 index 0000000000..a8060b1770 --- /dev/null +++ b/flopy/mf6/utils/codegen/templates/package_container.jinja @@ -0,0 +1,64 @@ +class {{ name.title.title() }}Packages(mfpackage.MFChildPackages): + """ + {{ name.title.title() }}Packages is a container class for the Modflow{{ name.title.title() }} class. + + Methods + ------- + initialize + Initializes a new Modflow{{ name.title.title() }} package removing any sibling child + packages attached to the same parent package. See Modflow{{ name.title.title() }} init + documentation for definition of parameters. + append_package + Adds a new Modflow{{ name.title.title() }} package to the container. See Modflow{{ name.title.title() }} + init documentation for definition of parameters. + """ + + package_abbr = "{{ name.title.lower() }}packages" + + def initialize( + self, + {%- for n, var in variables.items() if var.container_init_param and not var.init_skip %} + {%- if var.default is defined %} + {{ n }}={{ var.default }}, + {%- else -%} + {{ n }}, + {% endif -%} + {%- endfor %} + filename=None, + pname=None, + ): + new_package = Modflow{{ name.title.title() }}( + self._cpparent, + {%- for n, var in variables.items() if var.container_init_param and not var.init_skip %} + {{ n }}={{ n }}, + {%- endfor %} + filename=filename, + pname=pname, + child_builder_call=True, + ) + self.init_package(new_package, filename) + +{% if name.r != "obs" %} + def append_package( + self, + {%- for n, var in variables.items() if var.container_init_param and not var.init_skip %} + {%- if var.default is defined %} + {{ n }}={{ var.default }}, + {%- else -%} + {{ n }}, + {% endif -%} + {%- endfor %} + filename=None, + pname=None, + ): + new_package = Modflow{{ name.title.title() }}( + self._cpparent, + {%- for n, var in variables.items() if var.container_init_param and not var.init_skip %} + {{ n }}={{ n }}, + {%- endfor %} + filename=filename, + pname=pname, + child_builder_call=True, + ) + self._append_package(new_package, filename) +{% endif %} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/var.py b/flopy/mf6/utils/codegen/var.py new file mode 100644 index 0000000000..d90bb57ae6 --- /dev/null +++ b/flopy/mf6/utils/codegen/var.py @@ -0,0 +1,59 @@ +from dataclasses import dataclass +from enum import Enum +from typing import Any, Dict, Optional + +from flopy.mf6.utils.codegen.dfn import Metadata +from flopy.mf6.utils.codegen.ref import Ref + + +class VarKind(Enum): + """ + An input variable's kind. This is an enumeration + of the general shapes of data MODFLOW 6 accepts. + """ + + Array = "array" + Scalar = "scalar" + Record = "record" + Union = "union" + List = "list" + + +@dataclass +class Var: + """An input variable specification.""" + + name: str + _type: str + kind: VarKind + block: Optional[str] + description: Optional[str] + default: Optional[Any] + children: Optional[Dict[str, "Var"]] + metadata: Optional[Metadata] + reference: Optional[Ref] + + def __init__( + self, + name: str, + _type: str, + kind: VarKind, + block: Optional[str] = None, + description: Optional[str] = None, + default: Optional[Any] = None, + children: Optional["Vars"] = None, + metadata: Optional[Metadata] = None, + reference: Optional[Ref] = None, + ): + self.name = name + self._type = _type + self.kind = kind + self.block = block + self.description = description + self.default = default + self.children = children + self.metadata = metadata + self.reference = reference + + +Vars = Dict[str, Var] diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index e1a57fb094..6d76aa8cf6 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -81,1056 +81,14 @@ """ -import datetime -import os -import textwrap -from enum import Enum +from pathlib import Path -# keep below as absolute imports -from flopy.mf6.data import mfdatautil, mfstructure -from flopy.utils import datautil +from flopy.mf6.utils.codegen.make import make_all - -class PackageLevel(Enum): - sim_level = 0 - model_level = 1 - - -def build_doc_string(param_name, param_type, param_desc, indent): - return f"{indent}{param_name} : {param_type}\n{indent * 2}* {param_desc}" - - -def generator_type(data_type): - if ( - data_type == mfstructure.DataType.scalar_keyword - or data_type == mfstructure.DataType.scalar - ): - # regular scalar - return "ScalarTemplateGenerator" - elif ( - data_type == mfstructure.DataType.scalar_keyword_transient - or data_type == mfstructure.DataType.scalar_transient - ): - # transient scalar - return "ScalarTemplateGenerator" - elif data_type == mfstructure.DataType.array: - # array - return "ArrayTemplateGenerator" - elif data_type == mfstructure.DataType.array_transient: - # transient array - return "ArrayTemplateGenerator" - elif data_type == mfstructure.DataType.list: - # list - return "ListTemplateGenerator" - elif ( - data_type == mfstructure.DataType.list_transient - or data_type == mfstructure.DataType.list_multiple - ): - # transient or multiple list - return "ListTemplateGenerator" - - -def clean_class_string(name): - if len(name) > 0: - clean_string = name.replace(" ", "_") - clean_string = clean_string.replace("-", "_") - version = mfstructure.MFStructure().get_version_string() - # FIX: remove all numbers - if clean_string[-1] == version: - clean_string = clean_string[:-1] - return clean_string - return name - - -def build_dfn_string(dfn_list, header, package_abbr, flopy_dict): - dfn_string = " dfn = [" - line_length = len(dfn_string) - leading_spaces = " " * line_length - first_di = True - - # process header - dfn_string = f'{dfn_string}\n{leading_spaces}["header", ' - for key, value in header.items(): - if key == "multi-package": - dfn_string = f'{dfn_string}\n{leading_spaces} "multi-package", ' - if key == "package-type": - dfn_string = ( - f'{dfn_string}\n{leading_spaces} "package-type ' f'{value}"' - ) - - # process solution packages - if package_abbr in flopy_dict["solution_packages"]: - model_types = '", "'.join( - flopy_dict["solution_packages"][package_abbr] - ) - dfn_string = ( - f"{dfn_string}\n{leading_spaces} " - f'["solution_package", "{model_types}"], ' - ) - dfn_string = f"{dfn_string}],\n{leading_spaces}" - - # process all data items - for data_item in dfn_list: - line_length += 1 - if not first_di: - dfn_string = f"{dfn_string},\n{leading_spaces}" - line_length = len(leading_spaces) - else: - first_di = False - dfn_string = f"{dfn_string}[" - first_line = True - # process each line in a data item - for line in data_item: - line = line.strip() - # do not include the description of longname - if not line.lower().startswith( - "description" - ) and not line.lower().startswith("longname"): - line = line.replace('"', "'") - line_length += len(line) + 4 - if not first_line: - dfn_string = f"{dfn_string}," - if line_length < 77: - # added text fits on the current line - if first_line: - dfn_string = f'{dfn_string}"{line}"' - else: - dfn_string = f'{dfn_string} "{line}"' - else: - # added text does not fit on the current line - line_length = len(line) + len(leading_spaces) + 2 - if line_length > 79: - # added text too long to fit on a single line, wrap - # text as needed - line = f'"{line}"' - lines = textwrap.wrap( - line, - 75 - len(leading_spaces), - drop_whitespace=True, - ) - lines[0] = f"{leading_spaces} {lines[0]}" - line_join = f' "\n{leading_spaces} "' - dfn_string = f"{dfn_string}\n{line_join.join(lines)}" - else: - dfn_string = f'{dfn_string}\n{leading_spaces} "{line}"' - first_line = False - - dfn_string = f"{dfn_string}]" - dfn_string = f"{dfn_string}]" - return dfn_string - - -def create_init_var(clean_ds_name, data_structure_name, init_val=None): - if init_val is None: - init_val = clean_ds_name - - init_var = f" self.{clean_ds_name} = self.build_mfdata(" - leading_spaces = " " * len(init_var) - if len(init_var) + len(data_structure_name) + 2 > 79: - second_line = f'\n "{data_structure_name}",' - if len(second_line) + len(clean_ds_name) + 2 > 79: - init_var = f"{init_var}{second_line}\n {init_val})" - else: - init_var = f"{init_var}{second_line} {init_val})" - else: - init_var = f'{init_var}"{data_structure_name}",' - if len(init_var) + len(clean_ds_name) + 2 > 79: - init_var = f"{init_var}\n{leading_spaces}{init_val})" - else: - init_var = f"{init_var} {init_val})" - return init_var - - -def create_basic_init(clean_ds_name): - return f" self.{clean_ds_name} = {clean_ds_name}\n" - - -def create_property(clean_ds_name): - return f" {clean_ds_name} = property(get_{clean_ds_name}, set_{clean_ds_name})" - - -def format_var_list(base_string, var_list, is_tuple=False): - if is_tuple: - base_string = f"{base_string}(" - extra_chars = 4 - else: - extra_chars = 2 - line_length = len(base_string) - leading_spaces = " " * line_length - # determine if any variable name is too long to fit - for item in var_list: - if line_length + len(item) + extra_chars > 80: - leading_spaces = " " - base_string = f"{base_string}\n{leading_spaces}" - line_length = len(leading_spaces) - break - - for index, item in enumerate(var_list): - if is_tuple: - item = f"'{item}'" - if index == len(var_list) - 1: - next_var_str = item - else: - next_var_str = f"{item}," - line_length += len(item) + extra_chars - if line_length > 80: - base_string = f"{base_string}\n{leading_spaces}{next_var_str}" - else: - if base_string[-1] == ",": - base_string = f"{base_string} " - base_string = f"{base_string}{next_var_str}" - if is_tuple: - return f"{base_string}))" - else: - return f"{base_string})" - - -def create_package_init_var( - parameter_name, package_abbr, data_name, clean_ds_name -): - one_line = ( - f" self._{package_abbr}_package = self.build_child_package(" - ) - one_line_b = f'"{package_abbr}", {parameter_name},' - leading_spaces = " " * len(one_line) - two_line = f'\n{leading_spaces}"{data_name}",' - three_line = f"\n{leading_spaces}self._{clean_ds_name})" - return f"{one_line}{one_line_b}{two_line}{three_line}" - - -def add_var( - init_vars, - class_vars, - options_param_list, - init_param_list, - package_properties, - doc_string, - data_structure_dict, - default_value, - name, - python_name, - description, - path, - data_type, - basic_init=False, - construct_package=None, - construct_data=None, - parameter_name=None, - set_param_list=None, - mf_nam=False, -): - if set_param_list is None: - set_param_list = [] - clean_ds_name = datautil.clean_name(python_name) - if construct_package is None: - # add variable initialization lines - if basic_init: - init_vars.append(create_basic_init(clean_ds_name)) - else: - init_vars.append(create_init_var(clean_ds_name, name)) - # add to parameter list - if default_value is None: - default_value = "None" - init_param_list.append(f"{clean_ds_name}={default_value}") - if path is not None and "options" in path: - options_param_list.append(f"{clean_ds_name}={default_value}") - # add to set parameter list - set_param_list.append(f"{clean_ds_name}={clean_ds_name}") - else: - clean_parameter_name = datautil.clean_name(parameter_name) - # init hidden variable - init_vars.append(create_init_var(f"_{clean_ds_name}", name, "None")) - if mf_nam: - options_param_list.append( - [f"{parameter_name}_data=None", parameter_name] - ) - else: - # init child package - init_vars.append( - create_package_init_var( - clean_parameter_name, - construct_package, - construct_data, - clean_ds_name, - ) - ) - # add to parameter list - init_param_list.append(f"{clean_parameter_name}=None") - # add to set parameter list - set_param_list.append( - f"{clean_parameter_name}={clean_parameter_name}" - ) - - package_properties.append(create_property(clean_ds_name)) - doc_string.add_parameter(description, model_parameter=True) - data_structure_dict[python_name] = 0 - if class_vars is not None: - gen_type = generator_type(data_type) - if gen_type != "ScalarTemplateGenerator": - new_class_var = f" {clean_ds_name} = {gen_type}(" - class_vars.append(format_var_list(new_class_var, path, True)) - return gen_type - return None - - -def build_init_string( - init_string, init_param_list, whitespace=" " -): - line_chars = len(init_string) - for index, param in enumerate(init_param_list): - if isinstance(param, list): - param = param[0] - if index + 1 < len(init_param_list): - line_chars += len(param) + 2 - else: - line_chars += len(param) + 3 - if line_chars > 79: - if len(param) + len(whitespace) + 1 > 79: - # try to break apart at = sign - param_list = param.split("=") - if len(param_list) == 2: - init_string = "{},\n{}{}=\n{}{}".format( - init_string, - whitespace, - param_list[0], - whitespace, - param_list[1], - ) - line_chars = len(param_list[1]) + len(whitespace) + 1 - continue - init_string = f"{init_string},\n{whitespace}{param}" - line_chars = len(param) + len(whitespace) + 1 - else: - init_string = f"{init_string}, {param}" - return f"{init_string}):\n" - - -def build_model_load(model_type): - model_load_c = ( - " Methods\n -------\n" - " load : (simulation : MFSimulationData, model_name : " - "string,\n namfile : string, " - "version : string, exe_name : string,\n model_ws : " - "string, strict : boolean) : MFSimulation\n" - " a class method that loads a model from files" - '\n """' - ) - - model_load = ( - " @classmethod\n def load(cls, simulation, structure, " - "modelname='NewModel',\n " - "model_nam_file='modflowtest.nam', version='mf6',\n" - " exe_name='mf6', strict=True, " - "model_rel_path='.',\n" - " load_only=None):\n " - "return mfmodel.MFModel.load_base(cls, simulation, structure, " - "modelname,\n " - "model_nam_file, '{}6', version,\n" - " exe_name, strict, " - "model_rel_path,\n" - " load_only)" - "\n".format(model_type) - ) - return model_load, model_load_c - - -def build_sim_load(): - sim_load_c = ( - " Methods\n -------\n" - " load : (sim_name : str, version : " - "string,\n exe_name : str or PathLike, " - "sim_ws : str or PathLike, strict : bool,\n verbosity_level : " - "int, load_only : list, verify_data : bool,\n " - "write_headers : bool, lazy_io : bool, use_pandas : bool,\n " - ") : MFSimulation\n" - " a class method that loads a simulation from files" - '\n """' - ) - - sim_load = ( - " @classmethod\n def load(cls, sim_name='modflowsim', " - "version='mf6',\n " - "exe_name: Union[str, os.PathLike] = 'mf6',\n " - "sim_ws: Union[str, os.PathLike] = os.curdir,\n " - "strict=True, verbosity_level=1, load_only=None,\n " - "verify_data=False, write_headers=True,\n " - "lazy_io=False, use_pandas=True):\n " - "return mfsimbase.MFSimulationBase.load(cls, sim_name, version, " - "\n " - "exe_name, sim_ws, strict,\n" - " verbosity_level, " - "load_only,\n " - "verify_data, write_headers, " - "\n lazy_io, use_pandas)" - "\n" - ) - return sim_load, sim_load_c - - -def build_model_init_vars(param_list): - init_var_list = [] - # build set data calls - for param in param_list: - if not isinstance(param, list): - param_parts = param.split("=") - init_var_list.append( - f" self.name_file.{param_parts[0]}.set_data({param_parts[0]})" - ) - init_var_list.append("") - # build attributes - for param in param_list: - if isinstance(param, list): - pkg_name = param[1] - param_parts = param[0].split("=") - init_var_list.append( - f" self.{param_parts[0]} = " - f"self._create_package('{pkg_name}', {param_parts[0]})" - ) - else: - param_parts = param.split("=") - init_var_list.append( - f" self.{param_parts[0]} = self.name_file.{param_parts[0]}" - ) - - return "\n".join(init_var_list) - - -def create_packages(): - indent = " " - init_string_def = " def __init__(self" - - # load JSON file - file_structure = mfstructure.MFStructure(load_from_dfn_files=True) - sim_struct = file_structure.sim_struct - - # assemble package list of buildable packages - package_list = [] - for package in sim_struct.utl_struct_objs.values(): - # add utility packages to list - package_list.append( - ( - package, - PackageLevel.model_level, - "utl", - package.dfn_list, - package.file_type, - package.header, - ) - ) - package_list.append( - ( - sim_struct.name_file_struct_obj, - PackageLevel.sim_level, - "", - sim_struct.name_file_struct_obj.dfn_list, - sim_struct.name_file_struct_obj.file_type, - sim_struct.name_file_struct_obj.header, - ) - ) - for package in sim_struct.package_struct_objs.values(): - # add simulation level package to list - package_list.append( - ( - package, - PackageLevel.sim_level, - "", - package.dfn_list, - package.file_type, - package.header, - ) - ) - for model_key, model in sim_struct.model_struct_objs.items(): - package_list.append( - ( - model.name_file_struct_obj, - PackageLevel.model_level, - model_key, - model.name_file_struct_obj.dfn_list, - model.name_file_struct_obj.file_type, - model.name_file_struct_obj.header, - ) - ) - for package in model.package_struct_objs.values(): - package_list.append( - ( - package, - PackageLevel.model_level, - model_key, - package.dfn_list, - package.file_type, - package.header, - ) - ) - - util_path, tail = os.path.split(os.path.realpath(__file__)) - init_file = open( - os.path.join(util_path, "..", "modflow", "__init__.py"), - "w", - newline="\n", - ) - init_file.write("from .mfsimulation import MFSimulation # isort:skip\n") - - nam_import_string = ( - "from .. import mfmodel\nfrom ..data.mfdatautil " - "import ArrayTemplateGenerator, ListTemplateGenerator" - ) - - # loop through packages list - init_file_imports = [] - flopy_dict = file_structure.flopy_dict - for package in package_list: - data_structure_dict = {} - package_properties = [] - init_vars = [] - init_param_list = [] - options_param_list = [] - set_param_list = [] - class_vars = [] - template_gens = [] - - package_abbr = clean_class_string( - f"{clean_class_string(package[2])}{package[0].file_type}" - ).lower() - dfn_string = build_dfn_string( - package[3], package[5], package_abbr, flopy_dict - ) - package_name = clean_class_string( - "{}{}{}".format( - clean_class_string(package[2]), - package[0].file_prefix, - package[0].file_type, - ) - ).lower() - if package[0].description: - doc_string = mfdatautil.MFDocString(package[0].description) - else: - if package[2]: - package_container_text = f" within a {package[2]} model" - else: - package_container_text = "" - ds = "Modflow{} defines a {} package{}.".format( - package_name.title(), - package[0].file_type, - package_container_text, - ) - if package[0].file_type == "mvr": - # mvr package warning - if package[2]: - ds = ( - "{} This package\n can only be used to move " - "water between packages within a single model." - "\n To move water between models use ModflowMvr" - ".".format(ds) - ) - else: - ds = ( - "{} This package can only be used to move\n " - "water between two different models. To move " - "water between two packages\n in the same " - 'model use the "model level" mover package (ex. ' - "ModflowGwfmvr).".format(ds) - ) - - doc_string = mfdatautil.MFDocString(ds) - - if package[0].dfn_type == mfstructure.DfnType.exch_file: - exgtype = ( - f'"{package_abbr[0:3].upper()}6-{package_abbr[3:].upper()}6"' - ) - - add_var( - init_vars, - None, - options_param_list, - init_param_list, - package_properties, - doc_string, - data_structure_dict, - exgtype, - "exgtype", - "exgtype", - build_doc_string( - "exgtype", - "", - "is the exchange type (GWF-GWF or GWF-GWT).", - indent, - ), - None, - None, - True, - ) - add_var( - init_vars, - None, - options_param_list, - init_param_list, - package_properties, - doc_string, - data_structure_dict, - None, - "exgmnamea", - "exgmnamea", - build_doc_string( - "exgmnamea", - "", - "is the name of the first model that is " - "part of this exchange.", - indent, - ), - None, - None, - True, - ) - add_var( - init_vars, - None, - options_param_list, - init_param_list, - package_properties, - doc_string, - data_structure_dict, - None, - "exgmnameb", - "exgmnameb", - build_doc_string( - "exgmnameb", - "", - "is the name of the second model that is " - "part of this exchange.", - indent, - ), - None, - None, - True, - ) - init_vars.append( - " simulation.register_exchange_file(self)\n" - ) - - # loop through all blocks - for block in package[0].blocks.values(): - for data_structure in block.data_structures.values(): - # only create one property for each unique data structure name - if data_structure.name not in data_structure_dict: - mf_sim = ( - "parent_name_type" in package[0].header - and package[0].header["parent_name_type"][1] - == "MFSimulation" - ) - mf_nam = package[0].file_type == "nam" - if ( - data_structure.construct_package is not None - and not mf_sim - and not mf_nam - ): - c_pkg = data_structure.construct_package - else: - c_pkg = None - tg = add_var( - init_vars, - class_vars, - options_param_list, - init_param_list, - package_properties, - doc_string, - data_structure_dict, - data_structure.default_value, - data_structure.name, - data_structure.python_name, - data_structure.get_doc_string(79, indent, indent), - data_structure.path, - data_structure.get_datatype(), - False, - # c_pkg, - data_structure.construct_package, - data_structure.construct_data, - data_structure.parameter_name, - set_param_list, - mf_nam, - ) - if tg is not None and tg not in template_gens: - template_gens.append(tg) - - import_string = "from .. import mfpackage" - if template_gens: - import_string += "\nfrom ..data.mfdatautil import " - import_string += ", ".join(sorted(template_gens)) - # add extra docstrings for additional variables - doc_string.add_parameter( - " filename : String\n File name for this package." - ) - doc_string.add_parameter( - " pname : String\n Package name for this package." - ) - doc_string.add_parameter( - " parent_file : MFPackage\n " - "Parent package file that references this " - "package. Only needed for\n utility " - "packages (mfutl*). For example, mfutllaktab " - "package must have \n a mfgwflak " - "package parent_file." - ) - - # build package builder class string - init_vars.append(" self._init_complete = True") - init_vars = "\n".join(init_vars) - package_short_name = clean_class_string(package[0].file_type).lower() - class_def_string = "class Modflow{}(mfpackage.MFPackage):\n".format( - package_name.title() - ) - class_def_string = class_def_string.replace("-", "_") - class_var_string = ( - '{}\n package_abbr = "{}"\n _package_type = ' - '"{}"\n dfn_file_name = "{}"' - "\n".format( - "\n".join(class_vars), - package_abbr, - package[4], - package[0].dfn_file_name, - ) - ) - init_string_full = init_string_def - init_string_sim = f"{init_string_def}, simulation" - # add variables to init string - doc_string.add_parameter( - " loading_package : bool\n " - "Do not set this parameter. It is intended " - "for debugging and internal\n " - "processing purposes only.", - beginning_of_list=True, - ) - if "parent_name_type" in package[0].header: - init_var = package[0].header["parent_name_type"][0] - parent_type = package[0].header["parent_name_type"][1] - elif package[1] == PackageLevel.sim_level: - init_var = "simulation" - parent_type = "MFSimulation" - else: - init_var = "model" - parent_type = "MFModel" - doc_string.add_parameter( - f" {init_var} : {parent_type}\n " - f"{init_var.capitalize()} that this package is a part " - "of. Package is automatically\n " - f"added to {init_var} when it is " - "initialized.", - beginning_of_list=True, - ) - init_string_full = ( - f"{init_string_full}, {init_var}, loading_package=False" - ) - init_param_list.append("filename=None") - init_param_list.append("pname=None") - init_param_list.append("**kwargs") - init_string_full = build_init_string(init_string_full, init_param_list) - - # build init code - parent_init_string = " super().__init__(" - spaces = " " * len(parent_init_string) - parent_init_string = ( - '{}{}, "{}", filename, pname,\n{}' - "loading_package, **kwargs)\n\n" - " # set up variables".format( - parent_init_string, init_var, package_short_name, spaces - ) - ) - local_datetime = datetime.datetime.now(datetime.timezone.utc) - comment_string = ( - "# DO NOT MODIFY THIS FILE DIRECTLY. THIS FILE " - "MUST BE CREATED BY\n# mf6/utils/createpackages.py\n" - "# FILE created on {} UTC".format( - local_datetime.strftime("%B %d, %Y %H:%M:%S") - ) - ) - # assemble full package string - package_string = "{}\n{}\n\n\n{}{}\n{}\n{}\n\n{}{}\n{}\n".format( - comment_string, - import_string, - class_def_string, - doc_string.get_doc_string(), - class_var_string, - dfn_string, - init_string_full, - parent_init_string, - init_vars, - ) - - # open new Packages file - pb_file = open( - os.path.join(util_path, "..", "modflow", f"mf{package_name}.py"), - "w", - newline="\n", - ) - pb_file.write(package_string) - if ( - package[0].sub_package - and package_abbr != "utltab" - and ( - "parent_name_type" not in package[0].header - or package[0].header["parent_name_type"][1] != "MFSimulation" - ) - ): - set_param_list.append("filename=filename") - set_param_list.append("pname=pname") - set_param_list.append("child_builder_call=True") - whsp_1 = " " - whsp_2 = " " - - file_prefix = package[0].dfn_file_name[0:3] - chld_doc_string = ( - ' """\n {}Packages is a container ' - "class for the Modflow{} class.\n\n " - "Methods\n ----------" - "\n".format(package_name.title(), package_name.title()) - ) - - # write out child packages class - chld_cls = ( - "\n\nclass {}Packages(mfpackage.MFChildPackage" "s):\n".format( - package_name.title() - ) - ) - chld_var = ( - f" package_abbr = " - f'"{package_name.title().lower()}packages"\n\n' - ) - chld_init = " def initialize(self" - chld_init = build_init_string( - chld_init, init_param_list[:-1], whsp_1 - ) - init_pkg = "\n self.init_package(new_package, filename)" - params_init = ( - " new_package = Modflow" - f"{package_name.title()}(self._cpparent" - ) - params_init = build_init_string( - params_init, set_param_list, whsp_2 - ) - chld_doc_string = ( - "{} initialize\n Initializes a new " - "Modflow{} package removing any sibling " - "child\n packages attached to the same " - "parent package. See Modflow{} init\n " - " documentation for definition of " - "parameters.\n".format( - chld_doc_string, package_name.title(), package_name.title() - ) - ) - - chld_appn = "" - params_appn = "" - append_pkg = "" - if package_abbr != "utlobs": # Hard coded obs no multi-pkg support - chld_appn = "\n\n def append_package(self" - chld_appn = build_init_string( - chld_appn, init_param_list[:-1], whsp_1 - ) - append_pkg = ( - "\n self._append_package(new_package, filename)" - ) - params_appn = ( - " new_package = Modflow" - f"{file_prefix.capitalize()}" - f"{package_short_name}(self._cpparent" - ) - params_appn = build_init_string( - params_appn, set_param_list, whsp_2 - ) - chld_doc_string = ( - "{} append_package\n Adds a " - "new Modflow{}{} package to the container." - " See Modflow{}{}\n init " - "documentation for definition of " - "parameters.\n".format( - chld_doc_string, - file_prefix.capitalize(), - package_short_name, - file_prefix.capitalize(), - package_short_name, - ) - ) - chld_doc_string = f'{chld_doc_string} """\n' - packages_str = "{}{}{}{}{}{}{}{}{}\n".format( - chld_cls, - chld_doc_string, - chld_var, - chld_init, - params_init[:-2], - init_pkg, - chld_appn, - params_appn[:-2], - append_pkg, - ) - pb_file.write(packages_str) - pb_file.close() - - init_file_imports.append( - f"from .mf{package_name} import Modflow{package_name.title()}\n" - ) - - if package[0].dfn_type == mfstructure.DfnType.model_name_file: - # build model file - init_vars = build_model_init_vars(options_param_list) - - options_param_list.insert(0, "model_rel_path='.'") - options_param_list.insert(0, "exe_name='mf6'") - options_param_list.insert(0, "version='mf6'") - options_param_list.insert(0, "model_nam_file=None") - options_param_list.insert(0, "modelname='model'") - options_param_list.append("**kwargs,") - init_string_sim = build_init_string( - init_string_sim, options_param_list - ) - sim_name = clean_class_string(package[2]) - class_def_string = "class Modflow{}(mfmodel.MFModel):\n".format( - sim_name.capitalize() - ) - class_def_string = class_def_string.replace("-", "_") - doc_string.add_parameter( - " sim : MFSimulation\n " - "Simulation that this model is a part " - "of. Model is automatically\n " - "added to simulation when it is " - "initialized.", - beginning_of_list=True, - model_parameter=True, - ) - doc_string.description = ( - f"Modflow{sim_name} defines a {sim_name} model" - ) - class_var_string = f" model_type = '{sim_name}'\n" - mparent_init_string = " super().__init__(" - spaces = " " * len(mparent_init_string) - mparent_init_string = ( - "{}simulation, model_type='{}6',\n{}" - "modelname=modelname,\n{}" - "model_nam_file=model_nam_file,\n{}" - "version=version, exe_name=exe_name,\n{}" - "model_rel_path=model_rel_path,\n{}" - "**kwargs," - ")\n".format( - mparent_init_string, - sim_name, - spaces, - spaces, - spaces, - spaces, - spaces, - ) - ) - load_txt, doc_text = build_model_load(sim_name) - package_string = "{}\n{}\n\n\n{}{}\n{}\n{}\n{}{}\n{}\n\n{}".format( - comment_string, - nam_import_string, - class_def_string, - doc_string.get_doc_string(True), - doc_text, - class_var_string, - init_string_sim, - mparent_init_string, - init_vars, - load_txt, - ) - md_file = open( - os.path.join(util_path, "..", "modflow", f"mf{sim_name}.py"), - "w", - newline="\n", - ) - md_file.write(package_string) - md_file.close() - init_file_imports.append( - f"from .mf{sim_name} import Modflow{sim_name.capitalize()}\n" - ) - elif package[0].dfn_type == mfstructure.DfnType.sim_name_file: - # build simulation file - init_vars = build_model_init_vars(options_param_list) - - options_param_list.insert(0, "lazy_io=False") - options_param_list.insert(0, "use_pandas=True") - options_param_list.insert(0, "write_headers=True") - options_param_list.insert(0, "verbosity_level=1") - options_param_list.insert( - 0, "sim_ws: Union[str, os.PathLike] = " "os.curdir" - ) - options_param_list.insert( - 0, "exe_name: Union[str, os.PathLike] " '= "mf6"' - ) - options_param_list.insert(0, "version='mf6'") - options_param_list.insert(0, "sim_name='sim'") - init_string_sim = " def __init__(self" - init_string_sim = build_init_string( - init_string_sim, options_param_list - ) - class_def_string = ( - "class MFSimulation(mfsimbase." "MFSimulationBase):\n" - ) - doc_string.add_parameter( - " sim_name : str\n" " Name of the simulation", - beginning_of_list=True, - model_parameter=True, - ) - doc_string.description = ( - "MFSimulation is used to load, build, and/or save a MODFLOW " - "6 simulation. \n A MFSimulation object must be created " - "before creating any of the MODFLOW 6 \n model objects." - ) - sparent_init_string = " super().__init__(" - spaces = " " * len(sparent_init_string) - sparent_init_string = ( - "{}sim_name=sim_name,\n{}" - "version=version,\n{}" - "exe_name=exe_name,\n{}" - "sim_ws=sim_ws,\n{}" - "verbosity_level=verbosity_level,\n{}" - "write_headers=write_headers,\n{}" - "lazy_io=lazy_io,\n{}" - "use_pandas=use_pandas,\n{}" - ")\n".format( - sparent_init_string, - spaces, - spaces, - spaces, - spaces, - spaces, - spaces, - spaces, - spaces, - ) - ) - sim_import_string = ( - "import os\n" - "from typing import Union\n" - "from .. import mfsimbase" - ) - - load_txt, doc_text = build_sim_load() - package_string = "{}\n{}\n\n\n{}{}\n{}\n{}{}\n{}\n\n{}".format( - comment_string, - sim_import_string, - class_def_string, - doc_string.get_doc_string(False, True), - doc_text, - init_string_sim, - sparent_init_string, - init_vars, - load_txt, - ) - sim_file = open( - os.path.join(util_path, "..", "modflow", "mfsimulation.py"), - "w", - newline="\n", - ) - sim_file.write(package_string) - sim_file.close() - init_file_imports.append( - "from .mfsimulation import MFSimulation\n" - ) - - # Sort the imports - for line in sorted(init_file_imports, key=lambda x: x.split()[3]): - init_file.write(line) - init_file.close() +_MF6_PATH = Path(__file__).parents[1] +_DFN_PATH = _MF6_PATH / "data" / "dfn" +_TGT_PATH = _MF6_PATH / "modflow" if __name__ == "__main__": - create_packages() + make_all(_DFN_PATH, _TGT_PATH) diff --git a/flopy/mf6/utils/generate_classes.py b/flopy/mf6/utils/generate_classes.py index 32c1d6978c..5f59135171 100644 --- a/flopy/mf6/utils/generate_classes.py +++ b/flopy/mf6/utils/generate_classes.py @@ -2,9 +2,10 @@ import shutil import tempfile import time +from pathlib import Path from warnings import warn -from .createpackages import create_packages +from .createpackages import make_all thisfilepath = os.path.dirname(os.path.abspath(__file__)) flopypth = os.path.join(thisfilepath, "..", "..") @@ -14,6 +15,10 @@ default_owner = "MODFLOW-USGS" default_repo = "modflow6" +_MF6_PATH = Path(__file__).parents[1] +_DFN_PATH = _MF6_PATH / "data" / "dfn" +_TGT_PATH = _MF6_PATH / "modflow" + def delete_files(files, pth, allow_failure=False, exclude=None): if exclude is None: @@ -189,7 +194,7 @@ def generate_classes( delete_mf6_classes() print(" Create mf6 classes using the downloaded definition files.") - create_packages() + make_all(_DFN_PATH, _TGT_PATH) list_files(os.path.join(flopypth, "mf6", "modflow")) diff --git a/pyproject.toml b/pyproject.toml index 9f5cf4e121..59867f5891 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,9 +29,10 @@ classifiers = [ ] requires-python = ">=3.9" dependencies = [ + "Jinja2>=3.0", "numpy>=1.20.3", "matplotlib >=1.4.0", - "pandas >=2.0.0" + "pandas >=2.0.0", ] dynamic = ["version", "readme"] From 581fe18c0fc39e538c217b217f6be1c655f4c5f1 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 10 Oct 2024 15:26:24 -0400 Subject: [PATCH 02/75] add templates to package-data, misc --- autotest/test_codegen.py | 21 --------------------- flopy/mf6/utils/codegen/make.py | 4 ++-- pyproject.toml | 1 + 3 files changed, 3 insertions(+), 23 deletions(-) diff --git a/autotest/test_codegen.py b/autotest/test_codegen.py index 0378b7cede..deff5ca242 100644 --- a/autotest/test_codegen.py +++ b/autotest/test_codegen.py @@ -32,27 +32,6 @@ def test_dfn_load(dfn_name): assert any(dfn) -@pytest.mark.parametrize( - "dfn, n_vars, n_flat, n_meta", - [("gwf-ic", 2, 2, 0), ("prt-prp", 18, 40, 1)], -) -def test_make_context(dfn, n_vars, n_flat, n_meta): - with open(DFN_PATH / "common.dfn") as f: - commonvars = Dfn.load(f) - - with open(DFN_PATH / f"{dfn}.dfn") as f: - dfn = DfnName(*dfn.split("-")) - definition = Dfn.load(f, name=dfn) - - context_names = get_context_names(dfn) - context_name = context_names[0] - context = make_context(context_name, definition, commonvars) - assert len(context_names) == 1 - assert len(context.variables) == n_vars - assert len(context.definition) == n_flat - assert len(context.definition.metadata) == n_meta - - @pytest.mark.parametrize("dfn_name", DFN_NAMES) def test_make_targets(dfn_name, function_tmpdir): with open(DFN_PATH / "common.dfn") as f: diff --git a/flopy/mf6/utils/codegen/make.py b/flopy/mf6/utils/codegen/make.py index 298f68fbe4..1233030590 100644 --- a/flopy/mf6/utils/codegen/make.py +++ b/flopy/mf6/utils/codegen/make.py @@ -4,7 +4,7 @@ ) from warnings import warn -from jinja2 import Environment, FileSystemLoader +from jinja2 import Environment, PackageLoader # noqa: F401 from flopy.mf6.utils.codegen.context import ( @@ -15,7 +15,7 @@ from flopy.mf6.utils.codegen.dfn import Dfn, DfnName, Dfns from flopy.mf6.utils.codegen.ref import Ref, Refs -_TEMPLATE_LOADER = FileSystemLoader(Path(__file__).parent / "templates") +_TEMPLATE_LOADER = PackageLoader("flopy", "mf6/utils/codegen/templates/") _TEMPLATE_ENV = Environment(loader=_TEMPLATE_LOADER) _TEMPLATE_NAME = "context.py.jinja" _TEMPLATE = _TEMPLATE_ENV.get_template(_TEMPLATE_NAME) diff --git a/pyproject.toml b/pyproject.toml index 59867f5891..3c4f370fb9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,6 +111,7 @@ include = ["flopy", "flopy.*"] [tool.setuptools.package-data] "flopy.mf6.data" = ["dfn/*.dfn"] +"flopy.mf6.utils.codegen" = ["templates/*.jinja"] "flopy.plot" = ["mplstyle/*.mplstyle"] [tool.ruff] From a5aa0ccb2efe7bcd07c6c11fdac0840643448489 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 10 Oct 2024 15:56:46 -0400 Subject: [PATCH 03/75] shim fixes --- flopy/mf6/utils/codegen/shim.py | 90 +++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 28 deletions(-) diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index e9840cd701..1c1c51163f 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -28,7 +28,11 @@ def _is_var(o) -> bool: def _is_init_param(o) -> bool: """Whether the object is an `__init__` method parameter.""" d = dict(o) - return not d.get("ref", None) + if d.get("ref", None): + return False + if d["name"] in ["output"]: + return False + return True def _is_container_init_param(o) -> bool: @@ -37,6 +41,9 @@ def _is_container_init_param(o) -> bool: package container class. This is only relevant for some subpackage contexts. """ + d = dict(o) + if d["name"] in ["output"]: + return False return True @@ -97,9 +104,11 @@ def _add_exg_params(ctx: dict) -> dict: if ctx["references"]: for key, ref in ctx["references"].items(): - if key not in vars_: + key_var = vars_.get(key, None) + if not key_var: continue - vars_[ref["val"]] = { + vars_[key] = { + **key_var, "name": ref["val"], "description": ref.get("description", None), "reference": ref, @@ -172,9 +181,11 @@ def _add_pkg_params(ctx: dict) -> dict: if ctx["references"]: for key, ref in ctx["references"].items(): - if key not in vars_: + key_var = vars_.get(key, None) + if not key_var: continue vars_[key] = { + **key_var, "name": ref["val"], "description": ref.get("description", None), "reference": ref, @@ -242,9 +253,11 @@ def _add_mdl_params(ctx: dict) -> dict: if ctx["references"]: for key, ref in ctx["references"].items(): - if key not in vars_: + key_var = vars_.get(key, None) + if not key_var: continue vars_[key] = { + **key_var, "name": ref["val"], "description": ref.get("description", None), "reference": ref, @@ -328,13 +341,16 @@ def _add_sim_params(ctx: dict) -> dict: if ctx["references"] and ctx["name"] != (None, "nam"): for key, ref in ctx["references"].items(): - if key not in vars_: + key_var = vars_.get(key, None) + if not key_var: continue vars_[key] = { + **key_var, "name": ref["param"], "description": ref.get("description", None), "reference": ref, "init_param": True, + "init_skip": True, "default": None, } @@ -407,6 +423,9 @@ def _attr(var: dict) -> Optional[str]: ctx_name.l is not None and ctx_name.r == "nam" ) and var_name != "packages": return None + + if ctx_name.r == "dis" and var_name == "packagedata": + return None if var_kind in [ VarKind.List.value, @@ -425,7 +444,7 @@ def _attr(var: dict) -> Optional[str]: args.insert(0, f"'{ctx_name.l}6'") return f"{var_name} = ListTemplateGenerator(({', '.join(args)}))" - if var_kind == VarKind.Array.value: + elif var_kind == VarKind.Array.value: if not var_block: raise ValueError("Need block") args = [f"'{ctx_name.r}'", f"'{var_block}'", f"'{var_name}'"] @@ -440,7 +459,7 @@ def _attr(var: dict) -> Optional[str]: return None - attrs = [_attr(var) for var in ctx["variables"].values()] + attrs = [_attr(v) for v in ctx["variables"].values()] return "\n ".join([a for a in attrs if a]) @@ -509,7 +528,7 @@ def _should_build(var: dict) -> bool: Whether to call `build_mfdata()` on the variable. in the `__init__` method. """ - if var.get("reference", None): + if var.get("reference", None) and ctx["name"] != (None, "nam"): return False name = var["name"] if name in [ @@ -552,18 +571,21 @@ def _body() -> Optional[str]: statements = [] references = {} for var in ctx["variables"].values(): - if not var.get("kind", None) or var.get("init_skip", False): + ref = var.get("reference", None) + if not var.get("kind", None): continue + name = var["name"] if name in kwlist: name = f"{name}_" - ref = var.get("reference", None) - statements.append(f"self.name_file.{name}.set_data({name})") - statements.append(f"self.{name} = self.name_file.{name}") + + if not var.get("init_skip", False): + statements.append(f"self.name_file.{name}.set_data({name})") + statements.append(f"self.{name} = self.name_file.{name}") if ref and ref["key"] not in references: references[ref["key"]] = ref statements.append( - f"self._{ref['param']} = self._create_package('{ref['abbr']}', {ref['param']})" + f"self.{ref['param']} = self._create_package('{ref['abbr']}', {ref['param']})" ) else: statements = [] @@ -582,9 +604,14 @@ def _body() -> Optional[str]: ) elif _should_build(var): lname = name[:-1] if name.endswith("_") else name - statements.append( - f"self.{'_' if ref else ''}{name} = self.build_mfdata('{lname}', {name if var.get('init_param', True) else 'None'})" - ) + if ref and ctx["name"] == (None, "nam"): + statements.append( + f"self.{'_' if ref else ''}{ref['key']} = self.build_mfdata('{ref['key']}', None)" + ) + else: + statements.append( + f"self.{'_' if ref else ''}{name} = self.build_mfdata('{lname}', {name if var.get('init_param', True) else 'None'})" + ) if ( ref @@ -626,7 +653,12 @@ def _dfn(o) -> List[Metadata]: ctx = dict(o) dfn = ctx["definition"] - def _fmt_var(var: dict) -> List[str]: + def _meta(): + meta = dfn.metadata or list() + exclude = ["subpackage", "parent_name_type"] + return [m for m in meta if not any(p in m for p in exclude)] + + def _var(var: dict) -> List[str]: exclude = ["longname", "description"] def _fmt_name(k, v): @@ -638,16 +670,18 @@ def _fmt_name(k, v): if k not in exclude ] - meta = dfn.metadata or list() - _dfn = [] - for name, var in dfn: - var_ = ctx["variables"].get(name, None) - if var_ and "construct_package" in var_: - var["construct_package"] = var_["construct_package"] - var["construct_data"] = var_["construct_data"] - var["parameter_name"] = var_["parameter_name"] - _dfn.append((name, var)) - return [["header"] + [m for m in meta]] + [_fmt_var(v) for k, v in _dfn] + def _dfn(): + dfn_ = [] + for name, var in dfn: + var_ = ctx["variables"].get(name, None) + if var_ and "construct_package" in var_: + var["construct_package"] = var_["construct_package"] + var["construct_data"] = var_["construct_data"] + var["parameter_name"] = var_["parameter_name"] + dfn_.append((name, var)) + return [_var(v) for _, v in dfn_] + + return [["header"] + _meta()] + _dfn() def _qual_base(ctx: dict): From 82c806b14ec2f3dbe1e560c96fa59bcc182f9b60 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 10 Oct 2024 22:18:10 -0400 Subject: [PATCH 04/75] cleanup --- flopy/mf6/utils/codegen/ref.py | 25 +++++++++++++++++++++---- flopy/mf6/utils/codegen/shim.py | 6 ++++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/flopy/mf6/utils/codegen/ref.py b/flopy/mf6/utils/codegen/ref.py index 97a79f75e0..1ee3c32973 100644 --- a/flopy/mf6/utils/codegen/ref.py +++ b/flopy/mf6/utils/codegen/ref.py @@ -17,18 +17,24 @@ class Ref: defined in the referenced context can be provided directly instead of the file path (foreign key) variable. + Notes + ----- + This class is used to represent subpackage references. + Parameters ---------- key : str The name of the foreign key file input variable. val : str - The name of the selected variable in the referenced context. + The name of the data variable in the referenced context. abbr : str An abbreviation of the referenced context's name. param : str - The subpackage parameter name. TODO: explain - parents : List[Union[str, type]] - The subpackage's supported parent types. + The referenced parameter name. + parents : List[str] + The referenced context's supported parents. + description : Optional[str] + The reference's description. """ key: str @@ -40,6 +46,17 @@ class Ref: @classmethod def from_dfn(cls, dfn: Dfn) -> Optional["Ref"]: + """ + Try to load a reference from the definition. + Returns `None` if the definition cannot be + referenced by other contexts. + + Notes + ----- + Reference info is located in the definition's + metadata in an unstructured form. It would be + easier if we had a structured representation. + """ if not dfn.metadata: return None diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index 1c1c51163f..13c00e8c38 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -423,7 +423,7 @@ def _attr(var: dict) -> Optional[str]: ctx_name.l is not None and ctx_name.r == "nam" ) and var_name != "packages": return None - + if ctx_name.r == "dis" and var_name == "packagedata": return None @@ -580,7 +580,9 @@ def _body() -> Optional[str]: name = f"{name}_" if not var.get("init_skip", False): - statements.append(f"self.name_file.{name}.set_data({name})") + statements.append( + f"self.name_file.{name}.set_data({name})" + ) statements.append(f"self.{name} = self.name_file.{name}") if ref and ref["key"] not in references: references[ref["key"]] = ref From 3f62859c69b6579696ceabf8045b02d9661f2aa9 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 10 Oct 2024 22:25:30 -0400 Subject: [PATCH 05/75] appease codacy --- flopy/mf6/utils/codegen/context.py | 2 +- flopy/mf6/utils/codegen/dfn.py | 1 - flopy/mf6/utils/codegen/make.py | 1 - flopy/mf6/utils/codegen/render.py | 6 ++---- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py index bbd95b457a..67896415f9 100644 --- a/flopy/mf6/utils/codegen/context.py +++ b/flopy/mf6/utils/codegen/context.py @@ -306,7 +306,7 @@ def _default(value: str) -> Any: if _type != "string": try: return literal_eval(value) - except: + except SyntaxError: return value def _fields(record_name: str) -> Vars: diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index 6391a1f27c..d0d8f6474a 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -1,5 +1,4 @@ from collections import UserList -from dataclasses import dataclass from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Tuple diff --git a/flopy/mf6/utils/codegen/make.py b/flopy/mf6/utils/codegen/make.py index 1233030590..1d81aceb93 100644 --- a/flopy/mf6/utils/codegen/make.py +++ b/flopy/mf6/utils/codegen/make.py @@ -9,7 +9,6 @@ # noqa: F401 from flopy.mf6.utils.codegen.context import ( get_context_names, - make_context, make_contexts, ) from flopy.mf6.utils.codegen.dfn import Dfn, DfnName, Dfns diff --git a/flopy/mf6/utils/codegen/render.py b/flopy/mf6/utils/codegen/render.py index e61d839b5f..3ac2bdc690 100644 --- a/flopy/mf6/utils/codegen/render.py +++ b/flopy/mf6/utils/codegen/render.py @@ -1,6 +1,6 @@ from dataclasses import asdict from enum import Enum -from typing import Any, Callable, Dict, Iterable, Optional, Tuple, Union +from typing import Any, Callable, Dict, Iterable, Optional, Tuple Predicate = Callable[[Any], bool] Transform = Callable[[Any], Dict[str, str]] @@ -137,10 +137,8 @@ def _dict(o): break for p, e in set_pairs: - if not p(d): + if not (p(d) and e): continue - if e is None: - raise ValueError(f"No value for key: {k}") for k, v in e: if callable(v): v = v(d) From 4985ab884e304435c2255473475df13237a2438d Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 10 Oct 2024 22:27:25 -0400 Subject: [PATCH 06/75] remove unneeded imports from context template --- flopy/mf6/utils/codegen/templates/context.py.jinja | 4 ---- 1 file changed, 4 deletions(-) diff --git a/flopy/mf6/utils/codegen/templates/context.py.jinja b/flopy/mf6/utils/codegen/templates/context.py.jinja index 89fc1140eb..c7b87680a9 100644 --- a/flopy/mf6/utils/codegen/templates/context.py.jinja +++ b/flopy/mf6/utils/codegen/templates/context.py.jinja @@ -1,9 +1,5 @@ # autogenerated file, do not modify from os import PathLike, curdir -import typing -import numpy as np -from typing import Any, Optional, Tuple, List, Dict, Union, Literal, Iterable -from numpy.typing import NDArray from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator from flopy.mf6 import mfpackage From cc79c7d56520ad468dbfa225ed85850dcc2bfa69 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 10 Oct 2024 22:28:26 -0400 Subject: [PATCH 07/75] remove unneeded conditional from init template --- flopy/mf6/utils/codegen/templates/init.jinja | 3 --- 1 file changed, 3 deletions(-) diff --git a/flopy/mf6/utils/codegen/templates/init.jinja b/flopy/mf6/utils/codegen/templates/init.jinja index f51d0fad7c..72b98aa522 100644 --- a/flopy/mf6/utils/codegen/templates/init.jinja +++ b/flopy/mf6/utils/codegen/templates/init.jinja @@ -10,9 +10,6 @@ def __init__( **kwargs, ): {{ init_body }} - {% if name.l == "exg" and n == "exgmnameb" -%} - parent.register_exchange_file(self) - {% endif -%} {% if base == "MFPackage" %} self._init_complete = True {% endif %} \ No newline at end of file From 029458a83e1da8fdb5febf5ff1da792e3e8edb72 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 10 Oct 2024 22:39:11 -0400 Subject: [PATCH 08/75] cleanup --- flopy/mf6/utils/codegen/render.py | 2 +- flopy/mf6/utils/codegen/shim.py | 62 +++++++++++++++---------------- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/flopy/mf6/utils/codegen/render.py b/flopy/mf6/utils/codegen/render.py index 3ac2bdc690..fe228b8ff0 100644 --- a/flopy/mf6/utils/codegen/render.py +++ b/flopy/mf6/utils/codegen/render.py @@ -120,7 +120,7 @@ def _render_val(k, v): return { k: _render_val(k, v) for k, v in d.items() - # drop nones except where requested to keep them + # drop nones except where requested if (k in keep_none or v is not None) } diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index 13c00e8c38..498edf5cf3 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -20,7 +20,7 @@ def _is_ctx(o) -> bool: def _is_var(o) -> bool: - """Whether the object is a input context variable.""" + """Whether the object is an input context variable.""" d = dict(o) return "name" in d and "_type" in d @@ -47,10 +47,9 @@ def _is_container_init_param(o) -> bool: return True -def _add_exg_params(ctx: dict) -> dict: +def _set_exg_vars(ctx: dict) -> dict: """ - Add initializer parameters for an exchange input context. - Exchanges need different parameters than a typical package. + Modify variables for an exchange context. """ vars_ = ctx["variables"].copy() vars_ = { @@ -123,8 +122,8 @@ def _add_exg_params(ctx: dict) -> dict: return ctx -def _add_pkg_params(ctx: dict) -> dict: - """Add variables for a package context.""" +def _set_pkg_vars(ctx: dict) -> dict: + """Modify variables for a package context.""" vars_ = ctx["variables"].copy() if ctx["name"].r == "nam": @@ -200,8 +199,8 @@ def _add_pkg_params(ctx: dict) -> dict: return ctx -def _add_mdl_params(ctx: dict) -> dict: - """Add variables for a model context.""" +def _set_mdl_vars(ctx: dict) -> dict: + """Modify variables for a model context.""" vars_ = ctx["variables"].copy() init_skip = ["packages", "export_netcdf", "nc_filerecord"] for k in init_skip: @@ -270,8 +269,8 @@ def _add_mdl_params(ctx: dict) -> dict: return ctx -def _add_sim_params(ctx: dict) -> dict: - """Add variables for a simulation context.""" +def _set_sim_vars(ctx: dict) -> dict: + """Modify variables for a simulation context.""" vars_ = ctx["variables"].copy() init_skip = [ "tdis6", @@ -358,7 +357,7 @@ def _add_sim_params(ctx: dict) -> dict: return ctx -def _add_parent_param(ctx: dict) -> dict: +def _set_parent(ctx: dict) -> dict: vars_ = ctx["variables"] parent = ctx["parent"] if ctx.get("reference"): @@ -375,33 +374,32 @@ def _add_parent_param(ctx: dict) -> dict: return ctx -def _add_init_params(o): - """Add context-specific `__init__()` method parameters.""" +def _map_ctx(o): + """ + Transform an input context's as needed depending on its type. + + Notes + ----- + This includes adding extra variables for the `__init__` method; + This is done as a transform instead of with `set_pairs` so we + can control the order they appear in the method signature. + """ ctx = dict(o) if ctx["name"].base == "MFSimulationBase": - ctx = _add_sim_params(ctx) + ctx = _set_sim_vars(ctx) elif ctx["name"].base == "MFModel": - ctx = _add_mdl_params(ctx) - ctx = _add_parent_param(ctx) + ctx = _set_mdl_vars(ctx) + ctx = _set_parent(ctx) elif ctx["name"].base == "MFPackage": - if ctx["name"].l == "exg": - ctx = _add_exg_params(ctx) - else: - ctx = _add_pkg_params(ctx) - ctx = _add_parent_param(ctx) + ctx = ( + _set_exg_vars(ctx) + if ctx["name"].l == "exg" + else _set_pkg_vars(ctx) + ) + ctx = _set_parent(ctx) return ctx -def _transform_context(o): - # add vars depending on the - # specific type of context. - # do this as a transform so - # we can control the order - # they appear in `__init__` - # or other method signatures. - return _add_init_params(o) - - def _var_attrs(ctx: dict) -> str: """ Get class attributes for the context. @@ -718,7 +716,7 @@ def _qual_base(ctx: dict): ], ), ], - "transform": [(_is_ctx, _transform_context)], + "transform": [(_is_ctx, _map_ctx)], } """ Arguments for `renderable` as applied to `Context` From f44fa04be4f2ca77c10b987d84c267a449f4ee36 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 10 Oct 2024 22:41:48 -0400 Subject: [PATCH 09/75] fix codegen tests --- autotest/test_codegen.py | 1 - flopy/mf6/utils/codegen/context.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/autotest/test_codegen.py b/autotest/test_codegen.py index deff5ca242..736dde56df 100644 --- a/autotest/test_codegen.py +++ b/autotest/test_codegen.py @@ -6,7 +6,6 @@ from flopy.mf6.utils.codegen.make import ( DfnName, make_all, - make_context, make_targets, ) diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py index 67896415f9..6512d614c1 100644 --- a/flopy/mf6/utils/codegen/context.py +++ b/flopy/mf6/utils/codegen/context.py @@ -306,7 +306,7 @@ def _default(value: str) -> Any: if _type != "string": try: return literal_eval(value) - except SyntaxError: + except (SyntaxError, ValueError): return value def _fields(record_name: str) -> Vars: From 44dca4e1d5c514c173bc3bcc51e8a60664a5f3d1 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 10 Oct 2024 22:44:52 -0400 Subject: [PATCH 10/75] fix job name in rtd.yml --- .github/workflows/rtd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/rtd.yml b/.github/workflows/rtd.yml index b129889128..eb74792d26 100644 --- a/.github/workflows/rtd.yml +++ b/.github/workflows/rtd.yml @@ -17,7 +17,7 @@ concurrency: cancel-in-progress: true jobs: set_options: - name: Set release options + name: Set options runs-on: ubuntu-22.04 outputs: ref: ${{ steps.set_ref.outputs.ref }} From 0fcd29514820c51f9f54e1129fed0af057ad85d8 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 10 Oct 2024 22:49:07 -0400 Subject: [PATCH 11/75] import Union --- flopy/mf6/utils/codegen/templates/context.py.jinja | 1 + 1 file changed, 1 insertion(+) diff --git a/flopy/mf6/utils/codegen/templates/context.py.jinja b/flopy/mf6/utils/codegen/templates/context.py.jinja index c7b87680a9..9140664690 100644 --- a/flopy/mf6/utils/codegen/templates/context.py.jinja +++ b/flopy/mf6/utils/codegen/templates/context.py.jinja @@ -1,5 +1,6 @@ # autogenerated file, do not modify from os import PathLike, curdir +from typing import Union from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator from flopy.mf6 import mfpackage From 5aeab6cc7d1dcf519ddee86aadc18be3dfe0f5ff Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 10 Oct 2024 22:53:16 -0400 Subject: [PATCH 12/75] cleanup --- flopy/mf6/utils/codegen/shim.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index 498edf5cf3..9ecdaca1c5 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -400,7 +400,7 @@ def _map_ctx(o): return ctx -def _var_attrs(ctx: dict) -> str: +def _class_attrs(ctx: dict) -> str: """ Get class attributes for the context. """ @@ -466,7 +466,7 @@ def _init_body(ctx: dict) -> str: Get the `__init__` method body for the context. """ - def _super_call() -> Optional[str]: + def _super() -> Optional[str]: """ Whether to pass the variable to `super().__init__()` by name in the `__init__` method. @@ -512,7 +512,7 @@ def _super_call() -> Optional[str]: return f"super().__init__({', '.join(args)})" - def _should_assign(var: dict) -> bool: + def _assign(var: dict) -> bool: """ Whether to assign arguments to self in the `__init__` method. if this is false, assume @@ -521,7 +521,7 @@ def _should_assign(var: dict) -> bool: """ return var["name"] in ["exgtype", "exgmnamea", "exgmnameb"] - def _should_build(var: dict) -> bool: + def _build(var: dict) -> bool: """ Whether to call `build_mfdata()` on the variable. in the `__init__` method. @@ -596,13 +596,13 @@ def _body() -> Optional[str]: if name in kwlist: name = f"{name}_" - if _should_assign(var): + if _assign(var): statements.append(f"self.{name} = {name}") if name == "exgmnameb": statements.append( "simulation.register_exchange_file(self)" ) - elif _should_build(var): + elif _build(var): lname = name[:-1] if name.endswith("_") else name if ref and ctx["name"] == (None, "nam"): statements.append( @@ -632,7 +632,7 @@ def _body() -> Optional[str]: else "\n".join([" " + s for s in statements]) ) - sections = [_super_call(), _body()] + sections = [_super(), _body()] sections = [s for s in sections if s] return "\n".join(sections) @@ -704,7 +704,7 @@ def _qual_base(ctx: dict): [ ("dfn", _dfn), ("qual_base", _qual_base), - ("var_attrs", _var_attrs), + ("var_attrs", _class_attrs), ("init_body", _init_body), ], ), From 2e7a7fa855e213032112f887a852267b39a095f3 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Fri, 11 Oct 2024 16:23:11 -0400 Subject: [PATCH 13/75] correction --- flopy/mf6/utils/codegen/dfn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index d0d8f6474a..b5de78791f 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -6,7 +6,7 @@ class DfnName(NamedTuple): """ Uniquely identifies an input definition by its name, which consists of a <= 3-letter left term and an optional right - term, also <= 3 letters. + term. """ l: str From 0aa5263706ec724cfab5428d6d6dc0025546fc17 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Fri, 11 Oct 2024 16:52:20 -0400 Subject: [PATCH 14/75] shim fixes --- flopy/mf6/utils/codegen/shim.py | 19 +++++++++++++++++-- flopy/mf6/utils/codegen/templates/attrs.jinja | 2 +- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index 9ecdaca1c5..bbe99b2842 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -410,6 +410,7 @@ def _attr(var: dict) -> Optional[str]: var_name = var["name"] var_kind = var.get("kind", None) var_block = var.get("block", None) + var_ref = var.get("reference", None) if var_kind is None or var_kind == VarKind.Scalar.value: return None @@ -431,7 +432,21 @@ def _attr(var: dict) -> Optional[str]: VarKind.Union.value, ]: if not var_block: - raise ValueError("Need block") + raise ValueError("Need block") + + if var_ref: + # if the variable is a subpackage reference, use the original key + # (which has been replaced already with the referenced variable) + args = [f"'{ctx_name.r}'", f"'{var_block}'", f"'{var_ref['key']}'"] + if ctx_name.l is not None and ctx_name.l not in [ + "sim", + "sln", + "utl", + "exg", + ]: + args.insert(0, f"'{ctx_name.l}6'") + return f"{var_ref['key']} = ListTemplateGenerator(({', '.join(args)}))" + args = [f"'{ctx_name.r}'", f"'{var_block}'", f"'{var_name}'"] if ctx_name.l is not None and ctx_name.l not in [ "sim", @@ -704,7 +719,7 @@ def _qual_base(ctx: dict): [ ("dfn", _dfn), ("qual_base", _qual_base), - ("var_attrs", _class_attrs), + ("class_attrs", _class_attrs), ("init_body", _init_body), ], ), diff --git a/flopy/mf6/utils/codegen/templates/attrs.jinja b/flopy/mf6/utils/codegen/templates/attrs.jinja index 86d8170006..55653dc42d 100644 --- a/flopy/mf6/utils/codegen/templates/attrs.jinja +++ b/flopy/mf6/utils/codegen/templates/attrs.jinja @@ -1,7 +1,7 @@ {%- if base == "MFModel" %} model_type = "{{ name.title }}" {%- elif base == "MFPackage" %} - {{ var_attrs }} + {{ class_attrs }} package_abbr = "{% if name.l != "sln" and name.l != "sim" and name.l != "exg" and name.l is not none %}{{ name.l }}{% endif %}{{ name.r }}" _package_type = "{{ name.r }}" dfn_file_name = "{% if name.l is not none %}{{ name.l }}-{% elif name.l is none %}sim-{% endif %}{{ name.r }}.dfn" From d1e9b64750450bd8834f495c2c7a9d64423ade5a Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Fri, 11 Oct 2024 17:08:07 -0400 Subject: [PATCH 15/75] hacky shim fix --- flopy/mf6/utils/codegen/shim.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index bbe99b2842..e4f263217a 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -618,14 +618,17 @@ def _body() -> Optional[str]: "simulation.register_exchange_file(self)" ) elif _build(var): - lname = name[:-1] if name.endswith("_") else name if ref and ctx["name"] == (None, "nam"): statements.append( f"self.{'_' if ref else ''}{ref['key']} = self.build_mfdata('{ref['key']}', None)" ) else: + # hack... + _name = name[:-1] if name.endswith("_") else name + if _name == "steady_state": + _name = "steady-state" statements.append( - f"self.{'_' if ref else ''}{name} = self.build_mfdata('{lname}', {name if var.get('init_param', True) else 'None'})" + f"self.{'_' if ref else ''}{name} = self.build_mfdata('{_name}', {name if var.get('init_param', True) else 'None'})" ) if ( @@ -677,7 +680,8 @@ def _var(var: dict) -> List[str]: exclude = ["longname", "description"] def _fmt_name(k, v): - return v.replace("-", "_") if k == "name" else v + return v + # return v.replace("-", "_") if k == "name" else v return [ " ".join([k, str(_fmt_name(k, v))]).strip() From 9ea0df00eb0b2af1fc2bbfd6f3f7a0253e95de7b Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Fri, 11 Oct 2024 17:16:17 -0400 Subject: [PATCH 16/75] ruff --- flopy/mf6/utils/codegen/shim.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index e4f263217a..9ce338bc7f 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -432,12 +432,16 @@ def _attr(var: dict) -> Optional[str]: VarKind.Union.value, ]: if not var_block: - raise ValueError("Need block") + raise ValueError("Need block") - if var_ref: + if var_ref: # if the variable is a subpackage reference, use the original key # (which has been replaced already with the referenced variable) - args = [f"'{ctx_name.r}'", f"'{var_block}'", f"'{var_ref['key']}'"] + args = [ + f"'{ctx_name.r}'", + f"'{var_block}'", + f"'{var_ref['key']}'", + ] if ctx_name.l is not None and ctx_name.l not in [ "sim", "sln", @@ -446,7 +450,7 @@ def _attr(var: dict) -> Optional[str]: ]: args.insert(0, f"'{ctx_name.l}6'") return f"{var_ref['key']} = ListTemplateGenerator(({', '.join(args)}))" - + args = [f"'{ctx_name.r}'", f"'{var_block}'", f"'{var_name}'"] if ctx_name.l is not None and ctx_name.l not in [ "sim", From 0ce4f6c203723323f0e5ecd5bd7f1e07775f6d04 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Mon, 14 Oct 2024 12:43:04 -0400 Subject: [PATCH 17/75] restructuring, passing more tests * move common var replacements and composite var inflation to dfn load * misc fixes to the shim --- autotest/test_codegen.py | 30 +- flopy/mf6/utils/codegen/context.py | 610 ++++-------------- flopy/mf6/utils/codegen/dfn.py | 411 ++++++++++-- flopy/mf6/utils/codegen/make.py | 71 +- flopy/mf6/utils/codegen/ref.py | 28 +- flopy/mf6/utils/codegen/render.py | 61 +- flopy/mf6/utils/codegen/shim.py | 160 +++-- .../templates/{attrs.jinja => attrs.py.jinja} | 0 ...age_container.jinja => container.py.jinja} | 20 +- .../utils/codegen/templates/context.py.jinja | 12 +- .../codegen/templates/docstring_params.jinja | 2 +- .../templates/{init.jinja => init.py.jinja} | 2 +- .../templates/{load.jinja => load.py.jinja} | 0 flopy/mf6/utils/codegen/var.py | 59 -- pyproject.toml | 1 + 15 files changed, 675 insertions(+), 792 deletions(-) rename flopy/mf6/utils/codegen/templates/{attrs.jinja => attrs.py.jinja} (100%) rename flopy/mf6/utils/codegen/templates/{package_container.jinja => container.py.jinja} (69%) rename flopy/mf6/utils/codegen/templates/{init.jinja => init.py.jinja} (84%) rename flopy/mf6/utils/codegen/templates/{load.jinja => load.py.jinja} (100%) delete mode 100644 flopy/mf6/utils/codegen/var.py diff --git a/autotest/test_codegen.py b/autotest/test_codegen.py index 736dde56df..eaf7bb57b4 100644 --- a/autotest/test_codegen.py +++ b/autotest/test_codegen.py @@ -1,13 +1,9 @@ import pytest from autotest.conftest import get_project_root_path -from flopy.mf6.utils.codegen.context import get_context_names +from flopy.mf6.utils.codegen.context import Context from flopy.mf6.utils.codegen.dfn import Dfn -from flopy.mf6.utils.codegen.make import ( - DfnName, - make_all, - make_targets, -) +from flopy.mf6.utils.codegen.make import make_all, make_targets PROJ_ROOT = get_project_root_path() MF6_PATH = PROJ_ROOT / "flopy" / "mf6" @@ -23,8 +19,13 @@ @pytest.mark.parametrize("dfn_name", DFN_NAMES) def test_dfn_load(dfn_name): dfn_path = DFN_PATH / f"{dfn_name}.dfn" + + common_path = DFN_PATH / "common.dfn" + with open(common_path, "r") as f: + common, _ = Dfn._load(f) + with open(dfn_path, "r") as f: - dfn = Dfn.load(f, name=DfnName(*dfn_name.split("-"))) + dfn = Dfn.load(f, name=Dfn.Name(*dfn_name.split("-")), common=common) if dfn_name in ["sln-ems", "exg-gwfprt", "exg-gwfgwe", "exg-gwfgwt"]: assert not any(dfn) else: @@ -33,16 +34,17 @@ def test_dfn_load(dfn_name): @pytest.mark.parametrize("dfn_name", DFN_NAMES) def test_make_targets(dfn_name, function_tmpdir): - with open(DFN_PATH / "common.dfn") as f: - common = Dfn.load(f) + common_path = DFN_PATH / "common.dfn" + with open(common_path, "r") as f: + common, _ = Dfn._load(f) with open(DFN_PATH / f"{dfn_name}.dfn", "r") as f: - dfn_name = DfnName(*dfn_name.split("-")) - dfn = Dfn.load(f, name=dfn_name) + dfn = Dfn.load(f, name=Dfn.Name(*dfn_name.split("-")), common=common) + + make_targets(dfn, function_tmpdir, verbose=True) - make_targets(dfn, function_tmpdir, commonvars=common) - for ctx_name in get_context_names(dfn_name): - source_path = function_tmpdir / ctx_name.target + for name in Context.Name.from_dfn(dfn): + source_path = function_tmpdir / name.target assert source_path.is_file() diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py index 6512d614c1..cf51c07eaa 100644 --- a/flopy/mf6/utils/codegen/context.py +++ b/flopy/mf6/utils/codegen/context.py @@ -1,7 +1,4 @@ -from ast import literal_eval from dataclasses import dataclass -from keyword import kwlist -from os import PathLike from typing import ( Any, Dict, @@ -9,141 +6,15 @@ List, NamedTuple, Optional, - Union, ) -from flopy.mf6.utils.codegen.dfn import Dfn, DfnName -from flopy.mf6.utils.codegen.ref import Ref, Refs +from flopy.mf6.utils.codegen.dfn import Dfn, Vars +from flopy.mf6.utils.codegen.ref import Ref from flopy.mf6.utils.codegen.render import renderable from flopy.mf6.utils.codegen.shim import SHIM -from flopy.mf6.utils.codegen.var import Var, VarKind, Vars -_SCALAR_TYPES = { - "keyword", - "integer", - "double precision", - "string", -} - -class ContextName(NamedTuple): - """ - Uniquely identifies an input context by its name, which - consists of a <= 3-letter left term and optional right - term also of <= 3 letters. - - Notes - ----- - A single `DefinitionName` may be associated with one or - more `ContextName`s. For instance, a model DFN file will - produce both a NAM package class and also a model class. - - From the `ContextName` several other things are derived, - including: - - - the input context class' name - - a description of the context class - - the name of the source file to write - - the base class the context inherits from - - """ - - l: str - r: Optional[str] - - @property - def title(self) -> str: - """ - The input context's unique title. This is not - identical to `f"{l}{r}` in some cases, but it - remains unique. The title is substituted into - the file name and class name. - """ - - l, r = self - if self == ("sim", "nam"): - return "simulation" - if l is None: - return r - if r is None: - return l - if l == "sim": - return r - if l in ["sln", "exg"]: - return r - return f"{l}{r}" - - @property - def base(self) -> str: - """Base class from which the input context should inherit.""" - _, r = self - if self == ("sim", "nam"): - return "MFSimulationBase" - if r is None: - return "MFModel" - return "MFPackage" - - @property - def target(self) -> str: - """The source file name to generate.""" - return f"mf{self.title}.py" - - @property - def description(self) -> str: - """A description of the input context.""" - l, r = self - title = self.title.title() - if self.base == "MFPackage": - return f"Modflow{title} defines a {r.upper()} package." - elif self.base == "MFModel": - return f"Modflow{title} defines a {l.upper()} model." - elif self.base == "MFSimulationBase": - return """ - MFSimulation is used to load, build, and/or save a MODFLOW 6 simulation. - A MFSimulation object must be created before creating any of the MODFLOW 6 - model objects.""" - - -def get_context_names(dfn_name: DfnName) -> List[ContextName]: - """ - Returns a list of contexts this definition produces. - - Notes - ----- - An input definition may produce one or more input contexts. - - Model definition files produce both a model class context and - a model namefile package context. The same goes for simulation - definition files. All other definition files produce a single - context. - """ - if dfn_name.r == "nam": - if dfn_name.l == "sim": - return [ - ContextName(None, dfn_name.r), # nam pkg - ContextName(*dfn_name), # simulation - ] - else: - return [ - ContextName(*dfn_name), # nam pkg - ContextName(dfn_name.l, None), # model - ] - elif (dfn_name.l, dfn_name.r) in [ - ("gwf", "mvr"), - ("gwf", "gnc"), - ("gwt", "mvt"), - ]: - return [ContextName(*dfn_name), ContextName(None, dfn_name.r)] - return [ContextName(*dfn_name)] - - -@renderable( - # shim for implementation details in the - # generated context classes which aren't - # really concerns of the core framework, - # and may eventually go away - **SHIM -) +@renderable(**SHIM) @dataclass class Context: """ @@ -170,373 +41,164 @@ class Context: """ - name: ContextName - definition: Dfn - variables: Vars - base: Optional[type] = None - parent: Optional[str] = None - description: Optional[str] = None - reference: Optional[Ref] = None - references: Optional[Refs] = None - - -def make_context( - name: ContextName, - definition: Dfn, - commonvars: Optional[Dfn] = None, - references: Optional[Refs] = None, -) -> Context: - """ - Extract from an input definition a context descriptor: - a structured representation of an input context class. - - Each input definition yields one or more input contexts. - The `name` parameter selects which context to make. - - A map of common variables may be provided, which can be - referenced in the given context's variable descriptions. - - A map of other definitions may be provided, in which case a - parameter in this context may act as kind of "foreign key", - identifying another context as a subpackage which this one - is related to. - - Notes - ----- - This function does most of the work in the whole module. - A bit of a beast, but convenient to use the outer scope - (including the input definition, etc) in inner functions - without sending around a lot of parameters. And it's not - complicated; we just map a variable specification from a - definition file to a corresponding Python representation. - """ - - _definition = dict(definition) - _commonvars = dict(commonvars or dict()) - _references = dict(references or dict()) - - # is this context referenceable? - reference = Ref.from_dfn(definition) - - # contexts referenced by this one - referenced = dict() - - def _parent() -> Optional[str]: + class Name(NamedTuple): """ - Get a string parameter name for the context's parent(s), - i.e. context(s) which can own an instance of this one. - - If this context is a subpackage with multiple possible - parent types "x" and "y, this will be of form "x_or_y". - - """ - l, r = definition.name - if (l, r) == ("sim", "nam") and name == ("sim", "nam"): - return None - if l in ["sim", "exg", "sln"] or name.r is None: - return "simulation" - if reference: - if len(reference.parents) > 1: - return "_or_".join(reference.parents) - return reference.parents[0] - return "model" - - def _convert(var: Dict[str, Any], wrap: bool = False) -> Var: - """ - Transform a variable from its original representation in - an input definition to a Python specification appropriate - for generating an input context class. + Uniquely identifies an input context. A context + consists of a left term and optional right term. Notes ----- - This involves expanding nested type hierarchies, mapping - types to roughly equivalent Python primitives/composites, - and other shaping. - - The rules for optional variable defaults are as follows: - If a `default_value` is not provided, keywords are `False` - by default, everything else is `None`. + A single definition may be associated with one or more + contexts. For instance, a model DFN file will produce + both a namefile package class and a model class. - If `wrap` is true, scalars will be wrapped as records. - This is useful to distinguish among choices in unions. + From the context name several other things are derived: - Any filepath variable whose name functions as a foreign key - for another context will be given a pointer to the context. + - the input context class' name + - a description of the context class + - the name of the source file to write + - the base class the context inherits from """ - _name = var["name"] - _type = var.get("type", None) - block = var.get("block", None) - shape = var.get("shape", None) - shape = None if shape == "" else shape - default = var.get("default", None) - descr = var.get("description", "") + l: str + r: Optional[str] - # if the var is a foreign key, register the referenced context - ref = _references.get(_name, None) - if ref: - referenced[_name] = ref - - def _description(descr: str) -> str: + @property + def title(self) -> str: """ - Make substitutions from common variable definitions, - remove backslashes, TODO: generate/insert citations. + The input context's unique title. This is not + identical to `f"{l}{r}` in some cases, but it + remains unique. The title is substituted into + the file name and class name. """ - descr = descr.replace("\\", "") - _, replace, tail = descr.strip().partition("REPLACE") - if replace: - key, _, subs = tail.strip().partition(" ") - subs = literal_eval(subs) - cmn_var = _commonvars.get(key, None) - if cmn_var is None: - raise ValueError(f"Common variable not found: {key}") - descr = cmn_var.get("description", "") - if any(subs): - return descr.replace("\\", "").replace( - "{#1}", subs["{#1}"] - ) - return descr - return descr - def _default(value: str) -> Any: + l, r = self + if self == ("sim", "nam"): + return "simulation" + if l is None: + return r + if r is None: + return l + if l == "sim": + return r + if l in ["sln", "exg"]: + return r + return f"{l}{r}" + + @property + def base(self) -> str: + """Base class from which the input context should inherit.""" + _, r = self + if self == ("sim", "nam"): + return "MFSimulationBase" + if r is None: + return "MFModel" + return "MFPackage" + + @property + def target(self) -> str: + """The source file name to generate.""" + return f"mf{self.title}.py" + + @property + def description(self) -> str: + """A description of the input context.""" + l, r = self + title = self.title.title() + if self.base == "MFPackage": + return f"Modflow{title} defines a {r.upper()} package." + elif self.base == "MFModel": + return f"Modflow{title} defines a {l.upper()} model." + elif self.base == "MFSimulationBase": + return """ + MFSimulation is used to load, build, and/or save a MODFLOW 6 simulation. + A MFSimulation object must be created before creating any of the MODFLOW 6 + model objects.""" + + def parent(self, ref: Optional[Ref] = None) -> Optional[str]: """ - Try to parse a default value as a literal. + Return the name of the parent `__init__` method parameter, + or `None` if the context cannot have parents. Contexts can + have more than one possible parent, in which case the name + of the parameter is of the pattern `name1_or_..._or_nameN`. """ - if _type != "string": - try: - return literal_eval(value) - except (SyntaxError, ValueError): - return value - - def _fields(record_name: str) -> Vars: - """Recursively load/convert a record's fields.""" - record = _definition[record_name] - field_names = record["type"].split()[1:] - fields: Dict[str, Var] = { - n: _convert(field, wrap=False) - for n, field in _definition.items() - if n in field_names - } - field_names = list(fields.keys()) - - # if the record represents a file... - if "file" in record_name: - # remove filein/fileout - for term in ["filein", "fileout"]: - if term in field_names: - fields.pop(term) - - # remove leading keyword - keyword = next(iter(fields), None) - if keyword: - fields.pop(keyword) - - # set the type - n = list(fields.keys())[0] - path_field = fields[n] - path_field._type = Union[str, PathLike] - fields[n] = path_field - - # if tagged, remove the leading keyword - elif record.get("tagged", False): - keyword = next(iter(fields), None) - if keyword: - fields.pop(keyword) - - return fields - - def _var() -> Var: + if ref: + return ref.parent + if self == ("sim", "nam"): + return None + elif ( + self.l is None + or self.r is None + or self.l in ["sim", "exg", "sln"] + ): + return "simulation" + return "model" + + @staticmethod + def from_dfn(dfn: Dfn) -> List["Context.Name"]: """ - Create the variable. + Returns a list of context names this definition produces. Notes ----- - Goes through all the possible input kinds - from top (composites) to bottom (scalars): + An input definition may produce one or more input contexts. - - list - - union - - record - - array - - scalar - - Creates and returs a variable of the proper - kind. This may be a composite variable, in - which case nested variables are recursively - created as needed to produce the composite. + Model definition files produce both a model class context and + a model namefile package context. The same goes for simulation + definition files. All other definition files produce a single + context. """ - - children = dict() - - # list input, child is the item type - if _type.startswith("recarray"): - # make sure columns are defined - names = _type.split()[1:] - n_names = len(names) - if n_names < 1: - raise ValueError(f"Missing recarray definition: {_type}") - - # list input can have records or unions as rows. - # lists which have a consistent record type are - # regular, inconsistent record types irregular. - - # regular tabular/columnar data (1 record type) can be - # defined with a nested record (i.e. explicit) or with - # fields directly inside the recarray (implicit). list - # data for unions/keystrings necessarily comes nested. - - is_explicit_record = len(names) == 1 and _definition[names[0]][ - "type" - ].startswith("record") - - def _is_implicit_scalar_record(): - # if the record is defined implicitly and it has - # only scalar fields - types = [ - v["type"] for n, v in _definition.items() if n in names + if dfn.name.r == "nam": + if dfn.name.l == "sim": + return [ + Context.Name(None, dfn.name.r), # nam pkg + Context.Name(*dfn.name), # simulation ] - return all(t in _SCALAR_TYPES for t in types) - - if is_explicit_record: - record_name = names[0] - record_spec = _definition[record_name] - record = _convert(record_spec, wrap=False) - children = {record_name: record} - kind = VarKind.List - elif _is_implicit_scalar_record(): - record_name = _name - fields = _fields(record_name) - record = Var( - name=record_name, - _type=_type.split()[0], - kind=VarKind.Record, - block=block, - children=fields, - description=descr, - ) - children = {record_name: record} - kind = VarKind.List else: - # implicit complex record (i.e. some fields are records or unions) - fields = { - n: _convert(_definition[n], wrap=False) for n in names - } - first = list(fields.values())[0] - single = len(fields) == 1 - record_name = first.name if single else _name - record = Var( - name=record_name, - _type=_type.split()[0], - kind=VarKind.Record, - block=block, - children=first.children if single else fields, - description=descr, - ) - children = {record_name: record} - kind = VarKind.List - - # union (product), children are record choices - elif _type.startswith("keystring"): - names = _type.split()[1:] - children = { - n: _convert(_definition[n], wrap=True) for n in names - } - kind = VarKind.Union - - # record (sum), children are fields - elif _type.startswith("record"): - children = _fields(_name) - kind = VarKind.Record - - # are we wrapping a var into a record - # as a choice in a union? - elif wrap: - field_name = _name - field = _convert(var, wrap=False) - children = {field_name: field} - kind = VarKind.Record - - # at this point, if it has a shape, it's an array - elif shape is not None: - if _type not in _SCALAR_TYPES: - raise TypeError(f"Unsupported array type: {_type}") - elif _type == "string": - kind = VarKind.List - else: - kind = VarKind.Array - - # finally scalars - else: - kind = VarKind.Scalar - - # create var - return Var( - # if name is a reserved keyword, add a trailing underscore to it. - # convert dashes to underscores since it may become a class attr. - name=(f"{_name}_" if _name in kwlist else _name).replace( - "-", "_" - ), - _type=_type, - kind=kind, - block=block, - description=_description(descr), - default=_default(default), - children=children, - reference=ref, - ) - - return _var() + return [ + Context.Name(*dfn.name), # nam pkg + Context.Name(dfn.name.l, None), # model + ] + elif (dfn.name.l, dfn.name.r) in [ + ("gwf", "mvr"), + ("gwf", "gnc"), + ("gwt", "mvt"), + ]: + return [ + Context.Name(*dfn.name), + Context.Name(None, dfn.name.r), + ] + return [Context.Name(*dfn.name)] + + name: Name + vars: Vars + base: Optional[type] = None + parent: Optional[str] = None + description: Optional[str] = None + meta: Optional[Dict[str, Any]] = None - def _variables() -> Vars: + @classmethod + def from_dfn(cls, dfn: Dfn) -> Iterator["Context"]: """ - Return all input variables for an input context class. - - Notes - ----- - Not all variables become parameters; nested variables - will become components of composite parameters, e.g., - record fields, keystring (union) choices, list items. + Extract context class descriptor(s) from an input definition. + These are structured representations of input context classes. - Variables may be added, depending on the context type. + Each input definition yields one or more input contexts. + The `name` parameter can be used to select the context. """ - vars_ = _definition.copy() - vars_ = { - name: _convert(var, wrap=False) - for name, var in vars_.items() - # skip composites, we already inflated - # their parents in the var hierarchy - if not var.get("in_record", False) - } - - # reset var name since we may have altered - # it when creating the variable e.g. to - # avoid a reserved keyword collision - return {v.name: v for v in vars_.values()} - - return Context( - name=name, - definition=definition, - variables=_variables(), - base=name.base, - parent=_parent(), - description=name.description, - reference=reference, - references=referenced, - ) - - -def make_contexts( - definition: Dfn, - commonvars: Optional[Dfn] = None, - references: Optional[Refs] = None, -) -> Iterator[Context]: - """Generate input contexts from the given input definition.""" - for name in get_context_names(definition.name): - yield make_context( - name=name, - definition=definition, - commonvars=commonvars, - references=references, - ) + meta = dfn.meta.copy() + ref = Ref.from_dfn(dfn) + if ref: + meta["ref"] = ref + + for name in Context.Name.from_dfn(dfn): + yield Context( + name=name, + vars=dfn.data, + base=name.base, + parent=name.parent(ref), + description=name.description, + meta=meta, + ) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index b5de78791f..7708c0588b 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -1,58 +1,124 @@ -from collections import UserList -from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Tuple +from ast import literal_eval +from collections import UserDict +from dataclasses import dataclass +from enum import Enum +from keyword import kwlist +from os import PathLike +from typing import ( + Any, + Dict, + List, + NamedTuple, + Optional, + Tuple, + Union, +) +from warnings import warn +from boltons.dictutils import OMD -class DfnName(NamedTuple): +_SCALARS = { + "keyword", + "integer", + "double precision", + "string", +} + + +Vars = Dict[str, "Var"] +Dfns = Dict[str, "Dfn"] + + +def _try_parse_bool(value): """ - Uniquely identifies an input definition by its name, which - consists of a <= 3-letter left term and an optional right - term. + Try to parse a boolean from a string as represented + in a DFN file, otherwise return the value unaltered. """ + + if isinstance(value, str): + value = value.lower() + if value in ["true", "false"]: + return value == "true" + return value - l: str - r: str +def _try_literal_eval(value: str) -> Any: + """ + Try to parse a string as a literal. If this fails, + return the value unaltered. + """ + try: + return literal_eval(value) + except (SyntaxError, ValueError): + return value -Metadata = List[str] +@dataclass +class Var: + """MODFLOW 6 input variable specification.""" -class Dfn(UserList): - """ - An MF6 input definition. + class Kind(Enum): + """ + An input variable's kind. This is an enumeration + of the general shapes of data MODFLOW 6 accepts. + """ + + Array = "array" + Scalar = "scalar" + Record = "record" + Union = "union" + List = "list" - Notes - ----- - This class is a list rather than a dictionary to - accommodate duplicate variable names. Dictionary - would be nicer; this constraint goes away if the - DFN specifications become nested instead of flat. + name: str + kind: Kind + block: Optional[str] = None + description: Optional[str] = None + default: Optional[Any] = None + children: Optional[Vars] = None + meta: Optional[Dict[str, Any]] = None - With conversion to a standard format we get this - for free, and we could then drop the custom load. + +class Dfn(UserDict): + """ + MODFLOW 6 input definition. An input definition + file specifies a component of an MF6 simulation, + e.g. a model or package. """ - name: Optional[DfnName] - metadata: Optional[Metadata] + class Name(NamedTuple): + """ + Uniquely identifies an input definition. A name + consists of a left term and optional right term. + """ + + l: str + r: str + + name: Optional[Name] + meta: Optional[Dict[str, Any]] def __init__( self, - variables: Optional[Iterable[Tuple[str, Dict[str, Any]]]] = None, - name: Optional[DfnName] = None, - metadata: Optional[Metadata] = None, + vars: Optional[Vars] = None, + name: Optional[Name] = None, + meta: Optional[Dict[str, Any]] = None, ): - super().__init__(variables) + self.data = OMD(vars) self.name = name - self.metadata = metadata or [] + self.meta = meta - @classmethod - def load(cls, f, name: Optional[DfnName] = None) -> "Dfn": + @staticmethod + def _load(f, common: Optional[dict] = None) -> Tuple[OMD, List[str]]: """ - Load an input definition from a definition file. + Internal use only. Loads the DFN as an ordered multi-dictionary, and + a list of string metadata. This is later parsed into more structured + form. We also store the original representation for now so it can be + used by the shim. """ - - meta = None - vars_ = list() var = dict() + vars = list() + meta = list() + common = common or dict() for line in f: # remove whitespace/etc from the line @@ -63,8 +129,6 @@ def load(cls, f, name: Optional[DfnName] = None) -> "Dfn": if line.startswith("#"): _, sep, tail = line.partition("flopy") if sep == "flopy": - if meta is None: - meta = list() tail = tail.strip() if "solution_package" in tail: tail = tail.split() @@ -85,8 +149,7 @@ def load(cls, f, name: Optional[DfnName] = None) -> "Dfn": # block of attributes if not any(line): if any(var): - n = var["name"] - vars_.append((n, var)) + vars.append((var["name"], var)) var = dict() continue @@ -95,16 +158,278 @@ def load(cls, f, name: Optional[DfnName] = None) -> "Dfn": key, _, value = line.partition(" ") if key == "default_value": key = "default" - if value in ["true", "false"]: - value = value == "true" var[key] = value + # make substitutions from common variable definitions, + # remove backslashes, TODO: generate/insert citations. + descr = var.get("description", None) + if descr: + descr = descr.replace("\\", "") + _, replace, tail = descr.strip().partition("REPLACE") + if replace: + key, _, subs = tail.strip().partition(" ") + subs = literal_eval(subs) + cvar = common.get(key, None) + if cvar is None: + warn( + "Can't substitute description text, " + f"common variable not found: {key}" + ) + else: + descr = cvar.get("description", "") + if any(subs): + descr = descr.replace("\\", "").replace( + "{#1}", subs["{#1}"] + ) + var["description"] = descr + # add the final parameter if any(var): - n = var["name"] - vars_.append((n, var)) + vars.append((var["name"], var)) + + return OMD(vars), meta + + @classmethod + def load( + cls, + f, + name: Optional[Name] = None, + refs: Optional[Dfns] = None, + **kwargs, + ) -> "Dfn": + """Load an input definition.""" + + refs = refs or dict() + referenced = dict() + vars, meta = Dfn._load(f, **kwargs) + + def _map(spec: Dict[str, Any], wrap: bool = False) -> Var: + """ + Convert a variable specification from its representation + in an input definition file to a Pythonic form. + + Notes + ----- + This involves expanding nested type hierarchies, mapping + types to roughly equivalent Python primitives/composites, + and other shaping. + + The rules for optional variable defaults are as follows: + If a `default_value` is not provided, keywords are `False` + by default, everything else is `None`. + + If `wrap` is true, scalars will be wrapped as records. + This is useful to distinguish among choices in unions. + + Any filepath variable whose name functions as a foreign key + for another context will be given a pointer to the context. + + """ + + # parse booleans from strings. everything else can + # stay a string except default values, which we'll + # try to parse as arbitrary literals below, and at + # some point types, once we introduce type hinting + spec = {k: _try_parse_bool(v) for k, v in spec.items()} + + # pull off attributes we're interested in + _name = spec["name"] + _type = spec.get("type", None) + block = spec.get("block", None) + shape = spec.get("shape", None) + shape = None if shape == "" else shape + default = spec.get("default", None) + description = spec.get("description", "") + children = dict() + + # if var is a foreign key, register the reference + ref = refs.get(_name, None) + if ref: + referenced[_name] = ref + + def _fields(record_name: str) -> Vars: + """Recursively load/convert a record's fields.""" + record = next(iter(vars.getlist(record_name)), None) + assert record + names = _type.split()[1:] + fields = { + v["name"]: _map(v) + for v in vars.values(multi=True) + if v["name"] in names + and not v["type"].startswith("record") + and v.get("in_record", False) + } + + # if the record represents a file... + if "file" in _name: + # remove filein/fileout + for term in ["filein", "fileout"]: + if term in names: + fields.pop(term) + + # remove leading keyword + keyword = next(iter(fields), None) + if keyword: + fields.pop(keyword) + + # set the type + n = list(fields.keys())[0] + path_field = fields[n] + path_field._type = Union[str, PathLike] + fields[n] = path_field + + # if tagged, remove the leading keyword + elif record.get("tagged", False): + keyword = next(iter(fields), None) + if keyword: + fields.pop(keyword) + + return fields + + # list, child is the item type + if _type.startswith("recarray"): + # make sure columns are defined + names = _type.split()[1:] + n_names = len(names) + if n_names < 1: + raise ValueError(f"Missing recarray definition: {_type}") + + # list input can have records or unions as rows. + # lists which have a consistent record type are + # regular, inconsistent record types irregular. + + # regular tabular/columnar data (1 record type) can be + # defined with a nested record (i.e. explicit) or with + # fields directly inside the recarray (implicit). list + # data for unions/keystrings necessarily comes nested. + + is_explicit_record = n_names == 1 and vars[names[0]][ + "type" + ].startswith("record") + + def _is_implicit_scalar_record(): + # if the record is defined implicitly and it has + # only scalar fields + types = [ + v["type"] + for v in vars.values(multi=True) + if v["name"] in names and v.get("in_record", False) + ] + return all(t in _SCALARS for t in types) + + if is_explicit_record: + record = next(iter(vars.getlist(names[0])), None) + children = {names[0]: _map(record)} + kind = Var.Kind.List + elif _is_implicit_scalar_record(): + children = { + _name: Var( + name=_name, + kind=Var.Kind.Record, + block=block, + children=_fields(_name), + description=description, + ) + } + kind = Var.Kind.List + else: + # implicit complex record (i.e. some fields are records or unions) + fields = { + v["name"]: _map(v) + for v in vars.values(multi=True) + if v["name"] in names and v.get("in_record", False) + } + first = list(fields.values())[0] + single = len(fields) == 1 + name_ = first.name if single else _name + children = { + name_: Var( + name=name_, + kind=Var.Kind.Record, + block=block, + children=first.children if single else fields, + description=description, + ) + } + kind = Var.Kind.List + + # union (product), children are choices. + # scalar choices are wrapped as records. + elif _type.startswith("keystring"): + names = _type.split()[1:] + children = { + v["name"]: _map(v, wrap=True) + for v in vars.values(multi=True) + if v["name"] in names and v.get("in_record", False) + } + kind = Var.Kind.Union + + # record (sum), children are fields + elif _type.startswith("record"): + children = _fields(_name) + kind = Var.Kind.Record + + # are we wrapping a var into a record + # as a choice in a union? + elif wrap: + children = {_name: _map(spec)} + kind = Var.Kind.Record + + # at this point, if it has a shape, it's an array + elif shape is not None: + if _type not in _SCALARS: + raise TypeError(f"Unsupported array type: {_type}") + elif _type == "string": + kind = Var.Kind.List + else: + kind = Var.Kind.Array + + # finally scalars + else: + kind = Var.Kind.Scalar + + # create var + return Var( + # if name is a reserved keyword, add a trailing underscore to it. + # convert dashes to underscores since it may become a class attr. + name=(f"{_name}_" if _name in kwlist else _name).replace( + "-", "_" + ), + kind=kind, + block=block, + description=description, + default=( + _try_literal_eval(default) + if _type != "string" + else default + ), + children=children, + meta={"ref": ref}, + ) + + # pass the original DFN representation as + # metadata so the shim can use it for now + _vars = list(vars.values(multi=True)) - return cls(variables=vars_, name=name, metadata=meta) + # convert input variable specs to + # structured form, descending into + # composites recursively as needed + vars = { + var["name"]: _map(var) + for var in vars.values(multi=True) + if not var.get("in_record", False) + } + # reset the var name. we may have altered + # it when converting the variable e.g. to + # avoid collision with a reserved keyword + vars = {v.name: v for v in vars.values()} -Dfns = Dict[str, Dfn] + return cls( + vars, + name, + { + "dfn": (_vars, meta), + "refs": referenced, + }, + ) diff --git a/flopy/mf6/utils/codegen/make.py b/flopy/mf6/utils/codegen/make.py index 1d81aceb93..3d18ca517e 100644 --- a/flopy/mf6/utils/codegen/make.py +++ b/flopy/mf6/utils/codegen/make.py @@ -1,17 +1,9 @@ from pathlib import Path -from typing import ( - Optional, -) -from warnings import warn from jinja2 import Environment, PackageLoader -# noqa: F401 -from flopy.mf6.utils.codegen.context import ( - get_context_names, - make_contexts, -) -from flopy.mf6.utils.codegen.dfn import Dfn, DfnName, Dfns +from flopy.mf6.utils.codegen.context import Context +from flopy.mf6.utils.codegen.dfn import Dfn, Dfns from flopy.mf6.utils.codegen.ref import Ref, Refs _TEMPLATE_LOADER = PackageLoader("flopy", "mf6/utils/codegen/templates/") @@ -20,18 +12,10 @@ _TEMPLATE = _TEMPLATE_ENV.get_template(_TEMPLATE_NAME) -def make_targets( - definition: Dfn, - outdir: Path, - commonvars: Optional[Dfn] = None, - references: Optional[Refs] = None, - verbose: bool = False, -): +def make_targets(dfn: Dfn, outdir: Path, verbose: bool = False): """Generate Python source file(s) from the given input definition.""" - for context in make_contexts( - definition=definition, commonvars=commonvars, references=references - ): + for context in Context.from_dfn(dfn): target = outdir / context.name.target with open(target, "w") as f: source = _TEMPLATE.render(**context.render()) @@ -51,44 +35,41 @@ def make_all(dfndir: Path, outdir: Path, verbose: bool = False): # try to load common variables common_path = dfndir / "common.dfn" if not common_path.is_file: - warn("No common input definition file...") common = None else: with open(common_path, "r") as f: - common = Dfn.load(f) + common, _ = Dfn._load(f) - # load all the input definitions before we generate input - # contexts so we can create foreign key refs between them. - dfns: Dfns = {} + # load subpackages first so we can pass them as references + # to load() for the rest of the input contexts refs: Refs = {} - for p in paths: - name = DfnName(*p.stem.split("-")) - with open(p) as f: - dfn = Dfn.load(f, name=name) - dfns[name] = dfn + for path in paths: + name = Dfn.Name(*path.stem.split("-")) + with open(path) as f: + dfn = Dfn.load(f, name=name, common=common) ref = Ref.from_dfn(dfn) if ref: - # key is the name of the file record - # that's the reference's foreign key refs[ref.key] = ref - # generate target files + # load all the input definitions before we generate input + # contexts so we can create foreign key refs between them. + dfns: Dfns = {} + for path in paths: + name = Dfn.Name(*path.stem.split("-")) + with open(path) as f: + dfn = Dfn.load(f, name=name, refs=refs, common=common) + dfns[name] = dfn + + # make target files for dfn in dfns.values(): - with open(p) as f: - make_targets( - definition=dfn, - outdir=outdir, - references=refs, - commonvars=common, - verbose=verbose, - ) + make_targets(dfn, outdir, verbose) - # generate __init__.py file + # make __init__.py file init_path = outdir / "__init__.py" with open(init_path, "w") as f: for dfn in dfns.values(): - for ctx in get_context_names(dfn.name): - prefix = "MF" if ctx.base == "MFSimulationBase" else "Modflow" + for name in Context.Name.from_dfn(dfn): + prefix = "MF" if name.base == "MFSimulationBase" else "Modflow" f.write( - f"from .mf{ctx.title} import {prefix}{ctx.title.title()}\n" + f"from .mf{name.title} import {prefix}{name.title.title()}\n" ) diff --git a/flopy/mf6/utils/codegen/ref.py b/flopy/mf6/utils/codegen/ref.py index 1ee3c32973..59a2c067e6 100644 --- a/flopy/mf6/utils/codegen/ref.py +++ b/flopy/mf6/utils/codegen/ref.py @@ -41,7 +41,7 @@ class Ref: val: str abbr: str param: str - parents: List[str] + parent: str description: Optional[str] @classmethod @@ -50,21 +50,17 @@ def from_dfn(cls, dfn: Dfn) -> Optional["Ref"]: Try to load a reference from the definition. Returns `None` if the definition cannot be referenced by other contexts. - - Notes - ----- - Reference info is located in the definition's - metadata in an unstructured form. It would be - easier if we had a structured representation. """ - if not dfn.metadata: + if not dfn.meta or "dfn" not in dfn.meta: return None + _, meta = dfn.meta["dfn"] + lines = { "subpkg": next( iter( m - for m in dfn.metadata + for m in meta if isinstance(m, str) and m.startswith("subpac") ), None, @@ -72,7 +68,7 @@ def from_dfn(cls, dfn: Dfn) -> Optional["Ref"]: "parent": next( iter( m - for m in dfn.metadata + for m in meta if isinstance(m, str) and m.startswith("parent") ), None, @@ -82,14 +78,14 @@ def from_dfn(cls, dfn: Dfn) -> Optional["Ref"]: def _subpkg(): line = lines["subpkg"] _, key, abbr, param, val = line.split() - matches = [v for _, v in dfn if v["name"] == val] + matches = [v for v in dfn.values() if v.name == val] if not any(matches): descr = None else: if len(matches) > 1: warn(f"Multiple matches for referenced variable {val}") match = matches[0] - descr = match.get("description", None) + descr = match.description return { "key": key, @@ -99,13 +95,13 @@ def _subpkg(): "description": descr, } - def _parents(): + def _parent(): line = lines["parent"] - _, _, _type = line.split() - return [t.lower().replace("mf", "") for t in _type.split("/")] + _, param_name, _ = line.split() + return param_name return ( - cls(**_subpkg(), parents=_parents()) + cls(**_subpkg(), parent=_parent()) if all(v for v in lines.values()) else None ) diff --git a/flopy/mf6/utils/codegen/render.py b/flopy/mf6/utils/codegen/render.py index fe228b8ff0..345c110161 100644 --- a/flopy/mf6/utils/codegen/render.py +++ b/flopy/mf6/utils/codegen/render.py @@ -2,10 +2,10 @@ from enum import Enum from typing import Any, Callable, Dict, Iterable, Optional, Tuple -Predicate = Callable[[Any], bool] -Transform = Callable[[Any], Dict[str, str]] -Pair = Tuple[str, Any] -Pairs = Iterable[Pair] +_Predicate = Callable[[Any], bool] +_Transform = Callable[[Any], Dict[str, str]] +_Pair = Tuple[str, Any] +_Pairs = Iterable[_Pair] def _try_get_enum_value(v: Any) -> Any: @@ -17,8 +17,8 @@ def renderable( *, keep_none: Optional[Iterable[str]] = None, quote_str: Optional[Iterable[str]] = None, - set_pairs: Optional[Iterable[Tuple[Predicate, Pairs]]] = None, - transform: Optional[Iterable[Tuple[Predicate, Transform]]] = None, + set_pairs: Optional[Iterable[Tuple[_Predicate, _Pairs]]] = None, + transform: Optional[Iterable[Tuple[_Predicate, _Transform]]] = None, ): """ Decorator for dataclasses which are meant @@ -45,40 +45,23 @@ def renderable( value should be wrapped with quotation marks, use the `quote_str` parameter. - Arbitrary transformations of the instance - to which the decorator is applied can be - specified with the `transform` parameter, - which accepts a set of predicate/function - pairs; see below for more information on - how to use the transformation mechanism. + Arbitrary transformations can be configured + via the `transform` parameter, which accepts + an iterable of predicate / function tuples. + Each of these specifies a condition in which + an instance of a context should be modified, + and a function to make the alteration. Notes ----- This decorator is intended as a convenient way to modify dataclass instances to make them more palatable for templates. It also - aims to keep keep edge cases incidental to + keeps implementation details incidental to the current design of MF6 input framework cleanly isolated from the reimplementation - of which this code is a part. - - The basic idea behind this decorator is for - the developer to specify conditions in which - a given dataclass instance should be altered, - and a function to make the alteration. These - are provided as a collection of `Predicate`/ - `Transform` pairs. - - Transformations might be for convenience, or - to handle special cases where an object has - some other need for modification. - - Edge cases in the MF6 classes, e.g. the logic - determining the members of generated classes, - can be isolated as rendering transformations. - This allows keeping more general templating - infrastructure free of incidental complexity - while we move toward a leaner core framework. + of which this code is a part, which aims + for a more general approach. Jinja supports attribute- and dictionary- based access on arbitrary objects but does @@ -87,10 +70,22 @@ def renderable( can make it awkward to express some things, which transformations can also remedy. + Edge cases in the MF6 classes, e.g. the logic + determining the contents of generated classes, + can also be implemented with transformations. + This allows keeping the templating module as + generic as possible and inserting "shims" to + incrementally rewrite the existing framework. + Because a transformation function accepts an instance of a dataclass and converts it to a dictionary, only one transformation function - (the first predicate to match) is applied. + (of the first matching predicate) is applied. + + References + ---------- + This pattern was heavily inspired by `attrs`' + use of class decorators. """ quote_str = quote_str or list() diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index 9ce338bc7f..07fc49cd52 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -2,18 +2,16 @@ The purpose of this module is to keep special handling necessary to support the current `flopy.mf6` generated classes separate from more general templating and code -generation infrastructure. +generation infrastructure. It has no dependency on the +rest of the `flopy.mf6.utils.codegen` module. """ import os from keyword import kwlist from typing import List, Optional -from flopy.mf6.utils.codegen.dfn import Metadata -from flopy.mf6.utils.codegen.var import VarKind - -def _is_ctx(o) -> bool: +def _is_context(o) -> bool: """Whether the object is an input context.""" d = dict(o) return "name" in d and "base" in d @@ -22,7 +20,7 @@ def _is_ctx(o) -> bool: def _is_var(o) -> bool: """Whether the object is an input context variable.""" d = dict(o) - return "name" in d and "_type" in d + return "name" in d and "kind" in d def _is_init_param(o) -> bool: @@ -51,7 +49,7 @@ def _set_exg_vars(ctx: dict) -> dict: """ Modify variables for an exchange context. """ - vars_ = ctx["variables"].copy() + vars_ = ctx["vars"].copy() vars_ = { "loading_package": { "name": "loading_package", @@ -101,8 +99,9 @@ def _set_exg_vars(ctx: dict) -> dict: }, } - if ctx["references"]: - for key, ref in ctx["references"].items(): + refs = ctx.get("meta", dict()).get("refs", dict()) + if any(refs): + for key, ref in refs.items(): key_var = vars_.get(key, None) if not key_var: continue @@ -110,7 +109,7 @@ def _set_exg_vars(ctx: dict) -> dict: **key_var, "name": ref["val"], "description": ref.get("description", None), - "reference": ref, + "ref": ref, "init_param": True, "default": None, "construct_package": ref["abbr"], @@ -118,13 +117,13 @@ def _set_exg_vars(ctx: dict) -> dict: "parameter_name": ref["param"], } - ctx["variables"] = vars_ + ctx["vars"] = vars_ return ctx def _set_pkg_vars(ctx: dict) -> dict: """Modify variables for a package context.""" - vars_ = ctx["variables"].copy() + vars_ = ctx["vars"].copy() if ctx["name"].r == "nam": init_skip = ["export_netcdf", "nc_filerecord"] @@ -178,8 +177,9 @@ def _set_pkg_vars(ctx: dict) -> dict: ), } - if ctx["references"]: - for key, ref in ctx["references"].items(): + refs = ctx.get("meta", dict()).get("refs", dict()) + if any(refs): + for key, ref in refs.items(): key_var = vars_.get(key, None) if not key_var: continue @@ -187,7 +187,7 @@ def _set_pkg_vars(ctx: dict) -> dict: **key_var, "name": ref["val"], "description": ref.get("description", None), - "reference": ref, + "ref": ref, "init_param": ctx["name"].r != "nam", "default": None, "construct_package": ref["abbr"], @@ -195,13 +195,13 @@ def _set_pkg_vars(ctx: dict) -> dict: "parameter_name": ref["param"], } - ctx["variables"] = vars_ + ctx["vars"] = vars_ return ctx def _set_mdl_vars(ctx: dict) -> dict: """Modify variables for a model context.""" - vars_ = ctx["variables"].copy() + vars_ = ctx["vars"].copy() init_skip = ["packages", "export_netcdf", "nc_filerecord"] for k in init_skip: var = vars_.get(k, None) @@ -250,8 +250,9 @@ def _set_mdl_vars(ctx: dict) -> dict: **vars_, } - if ctx["references"]: - for key, ref in ctx["references"].items(): + refs = ctx.get("meta", dict()).get("refs", dict()) + if any(refs): + for key, ref in refs.items(): key_var = vars_.get(key, None) if not key_var: continue @@ -259,19 +260,19 @@ def _set_mdl_vars(ctx: dict) -> dict: **key_var, "name": ref["val"], "description": ref.get("description", None), - "reference": ref, + "ref": ref, "construct_package": ref["abbr"], "construct_data": ref["val"], "parameter_name": ref["param"], } - ctx["variables"] = vars_ + ctx["vars"] = vars_ return ctx def _set_sim_vars(ctx: dict) -> dict: """Modify variables for a simulation context.""" - vars_ = ctx["variables"].copy() + vars_ = ctx["vars"].copy() init_skip = [ "tdis6", "models", @@ -338,8 +339,9 @@ def _set_sim_vars(ctx: dict) -> dict: **vars_, } - if ctx["references"] and ctx["name"] != (None, "nam"): - for key, ref in ctx["references"].items(): + refs = ctx.get("meta", dict()).get("refs", dict()) + if any(refs) and ctx["name"] != (None, "nam"): + for key, ref in refs.items(): key_var = vars_.get(key, None) if not key_var: continue @@ -347,22 +349,20 @@ def _set_sim_vars(ctx: dict) -> dict: **key_var, "name": ref["param"], "description": ref.get("description", None), - "reference": ref, + "ref": ref, "init_param": True, "init_skip": True, "default": None, } - ctx["variables"] = vars_ + ctx["vars"] = vars_ return ctx def _set_parent(ctx: dict) -> dict: - vars_ = ctx["variables"] + vars_ = ctx["vars"] parent = ctx["parent"] - if ctx.get("reference"): - parent = f"parent_{parent}" - ctx["variables"] = { + ctx["vars"] = { parent: { "name": parent, "_type": str(ctx["parent"]), @@ -374,9 +374,9 @@ def _set_parent(ctx: dict) -> dict: return ctx -def _map_ctx(o): +def _map_context(o): """ - Transform an input context's as needed depending on its type. + Transform an input context as needed depending on its type. Notes ----- @@ -384,6 +384,7 @@ def _map_ctx(o): This is done as a transform instead of with `set_pairs` so we can control the order they appear in the method signature. """ + ctx = dict(o) if ctx["name"].base == "MFSimulationBase": ctx = _set_sim_vars(ctx) @@ -410,9 +411,9 @@ def _attr(var: dict) -> Optional[str]: var_name = var["name"] var_kind = var.get("kind", None) var_block = var.get("block", None) - var_ref = var.get("reference", None) + var_ref = var.get("meta", dict()).get("ref", None) - if var_kind is None or var_kind == VarKind.Scalar.value: + if var_kind is None or var_kind == "scalar": return None if var_name in ["cvoptions", "output"]: @@ -426,11 +427,7 @@ def _attr(var: dict) -> Optional[str]: if ctx_name.r == "dis" and var_name == "packagedata": return None - if var_kind in [ - VarKind.List.value, - VarKind.Record.value, - VarKind.Union.value, - ]: + if var_kind in ["list", "record", "union"]: if not var_block: raise ValueError("Need block") @@ -461,7 +458,7 @@ def _attr(var: dict) -> Optional[str]: args.insert(0, f"'{ctx_name.l}6'") return f"{var_name} = ListTemplateGenerator(({', '.join(args)}))" - elif var_kind == VarKind.Array.value: + elif var_kind == "array": if not var_block: raise ValueError("Need block") args = [f"'{ctx_name.r}'", f"'{var_block}'", f"'{var_name}'"] @@ -476,7 +473,7 @@ def _attr(var: dict) -> Optional[str]: return None - attrs = [_attr(v) for v in ctx["variables"].values()] + attrs = [_attr(v) for v in ctx["vars"].values()] return "\n ".join([a for a in attrs if a]) @@ -492,24 +489,20 @@ def _super() -> Optional[str]: """ if ctx["base"] == "MFPackage": - parent = ctx["parent"] - if ctx["reference"]: - parent = f"parent_{parent}" - pkgtyp = ctx["name"].r args = [ - parent, - f"'{pkgtyp}'", + ctx["parent"] + if ctx.get("meta", dict()).get("ref", None) + else ctx['parent'], + f"'{ctx['name'].r}'", "filename", "pname", "loading_package", "**kwargs", ] elif ctx["base"] == "MFModel": - parent = ctx["parent"] - mdltyp = ctx["name"].l args = [ - parent, - f"'{mdltyp}6'", + ctx["parent"], + f"'{ctx['name'].l}6'", "modelname=modelname", "model_nam_file=model_nam_file", "version=version", @@ -545,7 +538,10 @@ def _build(var: dict) -> bool: Whether to call `build_mfdata()` on the variable. in the `__init__` method. """ - if var.get("reference", None) and ctx["name"] != (None, "nam"): + if var.get("meta", dict()).get("ref", None) and ctx["name"] != ( + None, + "nam", + ): return False name = var["name"] if name in [ @@ -587,8 +583,8 @@ def _body() -> Optional[str]: if ctx["base"] in ["MFSimulationBase", "MFModel"]: statements = [] references = {} - for var in ctx["variables"].values(): - ref = var.get("reference", None) + for var in ctx["vars"].values(): + ref = var.get("meta", dict()).get("ref", None) if not var.get("kind", None): continue @@ -609,9 +605,9 @@ def _body() -> Optional[str]: else: statements = [] references = {} - for var in ctx["variables"].values(): + for var in ctx["vars"].values(): name = var["name"] - ref = var.get("reference", None) + ref = var.get("meta", dict()).get("ref", None) if name in kwlist: name = f"{name}_" @@ -659,7 +655,7 @@ def _body() -> Optional[str]: return "\n".join(sections) -def _dfn(o) -> List[Metadata]: +def _dfn(o) -> List[List[str]]: """ Get a list of the class' original definition attributes as a partial, internal reproduction of the DFN contents. @@ -668,41 +664,33 @@ def _dfn(o) -> List[Metadata]: ----- Currently, generated classes have a `.dfn` property that reproduces the corresponding DFN sans a few attributes. - This represents the DFN in raw form, before adapting to - Python, consolidating nested types, etc. + Once `mfstructure.py` etc is reworked to introspect the + context classes instead of this property, it can go. """ ctx = dict(o) - dfn = ctx["definition"] + dfn, meta = ctx["meta"]["dfn"] def _meta(): - meta = dfn.metadata or list() exclude = ["subpackage", "parent_name_type"] - return [m for m in meta if not any(p in m for p in exclude)] - - def _var(var: dict) -> List[str]: - exclude = ["longname", "description"] - - def _fmt_name(k, v): - return v - # return v.replace("-", "_") if k == "name" else v - - return [ - " ".join([k, str(_fmt_name(k, v))]).strip() - for k, v in var.items() - if k not in exclude - ] + return [v for v in meta if not any(p in v for p in exclude)] def _dfn(): - dfn_ = [] - for name, var in dfn: - var_ = ctx["variables"].get(name, None) - if var_ and "construct_package" in var_: - var["construct_package"] = var_["construct_package"] - var["construct_data"] = var_["construct_data"] - var["parameter_name"] = var_["parameter_name"] - dfn_.append((name, var)) - return [_var(v) for _, v in dfn_] + def _var(var: dict) -> List[str]: + exclude = ["longname", "description"] + name = var["name"] + var_ = ctx["vars"].get(name, None) + keys = ["construct_package", "construct_data", "parameter_name"] + if var_ and keys[0] in var_: + for k in keys: + var[k] = var_[k] + return [ + " ".join([k, v]).strip() + for k, v in var.items() + if k not in exclude + ] + + return [_var(var) for var in dfn] return [["header"] + _meta()] + _dfn() @@ -723,7 +711,7 @@ def _qual_base(ctx: dict): "quote_str": ["default"], "set_pairs": [ ( - _is_ctx, + _is_context, [ ("dfn", _dfn), ("qual_base", _qual_base), @@ -739,7 +727,7 @@ def _qual_base(ctx: dict): ], ), ], - "transform": [(_is_ctx, _map_ctx)], + "transform": [(_is_context, _map_context)], } """ Arguments for `renderable` as applied to `Context` diff --git a/flopy/mf6/utils/codegen/templates/attrs.jinja b/flopy/mf6/utils/codegen/templates/attrs.py.jinja similarity index 100% rename from flopy/mf6/utils/codegen/templates/attrs.jinja rename to flopy/mf6/utils/codegen/templates/attrs.py.jinja diff --git a/flopy/mf6/utils/codegen/templates/package_container.jinja b/flopy/mf6/utils/codegen/templates/container.py.jinja similarity index 69% rename from flopy/mf6/utils/codegen/templates/package_container.jinja rename to flopy/mf6/utils/codegen/templates/container.py.jinja index a8060b1770..a61ceb0f9b 100644 --- a/flopy/mf6/utils/codegen/templates/package_container.jinja +++ b/flopy/mf6/utils/codegen/templates/container.py.jinja @@ -17,19 +17,15 @@ class {{ name.title.title() }}Packages(mfpackage.MFChildPackages): def initialize( self, - {%- for n, var in variables.items() if var.container_init_param and not var.init_skip %} - {%- if var.default is defined %} - {{ n }}={{ var.default }}, - {%- else -%} - {{ n }}, - {% endif -%} + {%- for n, var in vars.items() if var.container_init_param and not var.init_skip %} + {{ n }}{%- if var.default is defined %}={{ var.default }}{% endif -%}, {%- endfor %} filename=None, pname=None, ): new_package = Modflow{{ name.title.title() }}( self._cpparent, - {%- for n, var in variables.items() if var.container_init_param and not var.init_skip %} + {%- for n, var in vars.items() if var.container_init_param and not var.init_skip %} {{ n }}={{ n }}, {%- endfor %} filename=filename, @@ -41,19 +37,15 @@ class {{ name.title.title() }}Packages(mfpackage.MFChildPackages): {% if name.r != "obs" %} def append_package( self, - {%- for n, var in variables.items() if var.container_init_param and not var.init_skip %} - {%- if var.default is defined %} - {{ n }}={{ var.default }}, - {%- else -%} - {{ n }}, - {% endif -%} + {%- for n, var in vars.items() if var.container_init_param and not var.init_skip %} + {{ n }}{%- if var.default is defined %}={{ var.default }}{% endif -%}, {%- endfor %} filename=None, pname=None, ): new_package = Modflow{{ name.title.title() }}( self._cpparent, - {%- for n, var in variables.items() if var.container_init_param and not var.init_skip %} + {%- for n, var in vars.items() if var.container_init_param and not var.init_skip %} {{ n }}={{ n }}, {%- endfor %} filename=filename, diff --git a/flopy/mf6/utils/codegen/templates/context.py.jinja b/flopy/mf6/utils/codegen/templates/context.py.jinja index 9140664690..75d3271db6 100644 --- a/flopy/mf6/utils/codegen/templates/context.py.jinja +++ b/flopy/mf6/utils/codegen/templates/context.py.jinja @@ -13,13 +13,13 @@ from flopy.mf6 import mfsimbase class {% if base == "MFSimulationBase" %}MF{% else %}Modflow{% endif %}{{ name.title.title() }}({{ qual_base }}): {% include "docstring.jinja" %} - {% include "attrs.jinja" %} + {% include "attrs.py.jinja" %} - {% include "init.jinja" %} + {% include "init.py.jinja" %} - {% include "load.jinja" %} + {% include "load.py.jinja" %} -{# TODO: cleaner way to filter out hpc subpkgs? #} -{% if reference is defined and name.r != "hpc" %} -{% include "package_container.jinja" %} +{# TODO: cleaner way to filter out hpc subpkg? #} +{% if "ref" in meta and name.r != "hpc" %} +{% include "container.py.jinja" %} {% endif %} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/docstring_params.jinja b/flopy/mf6/utils/codegen/templates/docstring_params.jinja index 3afb3be69f..d8d5641d02 100644 --- a/flopy/mf6/utils/codegen/templates/docstring_params.jinja +++ b/flopy/mf6/utils/codegen/templates/docstring_params.jinja @@ -1,4 +1,4 @@ -{%- for v in variables.values() recursive %} +{%- for v in vars.values() recursive %} {% if loop.depth > 1 %}* {% endif %}{{ v.name }}{% if v._type is defined and v._type is not none %} : {{ v._type }}{% endif %} {%- if v.description is defined and v.description is not none %} {{ v.description|wordwrap|indent(4 + (loop.depth * 4), first=True) }} diff --git a/flopy/mf6/utils/codegen/templates/init.jinja b/flopy/mf6/utils/codegen/templates/init.py.jinja similarity index 84% rename from flopy/mf6/utils/codegen/templates/init.jinja rename to flopy/mf6/utils/codegen/templates/init.py.jinja index 72b98aa522..9ed03e647e 100644 --- a/flopy/mf6/utils/codegen/templates/init.jinja +++ b/flopy/mf6/utils/codegen/templates/init.py.jinja @@ -1,6 +1,6 @@ def __init__( self, - {%- for var in variables.values() if var.init_param %} + {%- for var in vars.values() if var.init_param %} {%- if var.default is defined %} {{ var.name }}={{ var.default }}, {%- else %} diff --git a/flopy/mf6/utils/codegen/templates/load.jinja b/flopy/mf6/utils/codegen/templates/load.py.jinja similarity index 100% rename from flopy/mf6/utils/codegen/templates/load.jinja rename to flopy/mf6/utils/codegen/templates/load.py.jinja diff --git a/flopy/mf6/utils/codegen/var.py b/flopy/mf6/utils/codegen/var.py deleted file mode 100644 index d90bb57ae6..0000000000 --- a/flopy/mf6/utils/codegen/var.py +++ /dev/null @@ -1,59 +0,0 @@ -from dataclasses import dataclass -from enum import Enum -from typing import Any, Dict, Optional - -from flopy.mf6.utils.codegen.dfn import Metadata -from flopy.mf6.utils.codegen.ref import Ref - - -class VarKind(Enum): - """ - An input variable's kind. This is an enumeration - of the general shapes of data MODFLOW 6 accepts. - """ - - Array = "array" - Scalar = "scalar" - Record = "record" - Union = "union" - List = "list" - - -@dataclass -class Var: - """An input variable specification.""" - - name: str - _type: str - kind: VarKind - block: Optional[str] - description: Optional[str] - default: Optional[Any] - children: Optional[Dict[str, "Var"]] - metadata: Optional[Metadata] - reference: Optional[Ref] - - def __init__( - self, - name: str, - _type: str, - kind: VarKind, - block: Optional[str] = None, - description: Optional[str] = None, - default: Optional[Any] = None, - children: Optional["Vars"] = None, - metadata: Optional[Metadata] = None, - reference: Optional[Ref] = None, - ): - self.name = name - self._type = _type - self.kind = kind - self.block = block - self.description = description - self.default = default - self.children = children - self.metadata = metadata - self.reference = reference - - -Vars = Dict[str, Var] diff --git a/pyproject.toml b/pyproject.toml index 3c4f370fb9..8e57b38f29 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ classifiers = [ ] requires-python = ">=3.9" dependencies = [ + "boltons", "Jinja2>=3.0", "numpy>=1.20.3", "matplotlib >=1.4.0", From 159cff71a375b3d3b55f3dc65fe22813311550dd Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 15 Oct 2024 13:04:35 -0400 Subject: [PATCH 18/75] ruff --- flopy/mf6/utils/codegen/dfn.py | 2 +- flopy/mf6/utils/codegen/shim.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index 7708c0588b..667f312908 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -34,7 +34,7 @@ def _try_parse_bool(value): Try to parse a boolean from a string as represented in a DFN file, otherwise return the value unaltered. """ - + if isinstance(value, str): value = value.lower() if value in ["true", "false"]: diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index 07fc49cd52..4ca953552f 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -492,7 +492,7 @@ def _super() -> Optional[str]: args = [ ctx["parent"] if ctx.get("meta", dict()).get("ref", None) - else ctx['parent'], + else ctx["parent"], f"'{ctx['name'].r}'", "filename", "pname", From 0513ed61ba34da7ac9323fa64563c064b0bab5fd Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 15 Oct 2024 13:27:11 -0400 Subject: [PATCH 19/75] cleanup --- flopy/mf6/utils/codegen/context.py | 10 +++------- flopy/mf6/utils/codegen/make.py | 6 ++---- flopy/mf6/utils/codegen/ref.py | 9 ++++----- flopy/mf6/utils/codegen/render.py | 12 ++++++------ 4 files changed, 15 insertions(+), 22 deletions(-) diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py index cf51c07eaa..77318d0109 100644 --- a/flopy/mf6/utils/codegen/context.py +++ b/flopy/mf6/utils/codegen/context.py @@ -25,11 +25,7 @@ class Context: Notes ----- - A context class minimally consists of a name, a definition, and a - map of variables. The definition and variables are redundant (the - latter are generated from the former) but for now, the definition - is needed. When generated classes no longer reproduce definitions - verbatim, it can be removed. + A context minimally consists of a name and a map of variables. The context class may inherit from a base class, and may specify a parent context within which it can be created (the parent then @@ -58,6 +54,8 @@ class Name(NamedTuple): - a description of the context class - the name of the source file to write - the base class the context inherits from + - the name of the parent parameter in the context + class' `__init__` method, if it can have a parent """ @@ -183,9 +181,7 @@ def from_dfn(cls, dfn: Dfn) -> Iterator["Context"]: """ Extract context class descriptor(s) from an input definition. These are structured representations of input context classes. - Each input definition yields one or more input contexts. - The `name` parameter can be used to select the context. """ meta = dfn.meta.copy() diff --git a/flopy/mf6/utils/codegen/make.py b/flopy/mf6/utils/codegen/make.py index 3d18ca517e..74af3b70e0 100644 --- a/flopy/mf6/utils/codegen/make.py +++ b/flopy/mf6/utils/codegen/make.py @@ -40,8 +40,7 @@ def make_all(dfndir: Path, outdir: Path, verbose: bool = False): with open(common_path, "r") as f: common, _ = Dfn._load(f) - # load subpackages first so we can pass them as references - # to load() for the rest of the input contexts + # load subpackage references first refs: Refs = {} for path in paths: name = Dfn.Name(*path.stem.split("-")) @@ -51,8 +50,7 @@ def make_all(dfndir: Path, outdir: Path, verbose: bool = False): if ref: refs[ref.key] = ref - # load all the input definitions before we generate input - # contexts so we can create foreign key refs between them. + # load all the input definitions dfns: Dfns = {} for path in paths: name = Dfn.Name(*path.stem.split("-")) diff --git a/flopy/mf6/utils/codegen/ref.py b/flopy/mf6/utils/codegen/ref.py index 59a2c067e6..2637f0ba36 100644 --- a/flopy/mf6/utils/codegen/ref.py +++ b/flopy/mf6/utils/codegen/ref.py @@ -1,5 +1,5 @@ from dataclasses import dataclass -from typing import Dict, List, Optional +from typing import Dict, Optional from warnings import warn from flopy.mf6.utils.codegen.dfn import Dfn @@ -12,10 +12,9 @@ class Ref: and another input definition. This allows an input context to refer to another input context, by including a filepath variable whose name acts as a foreign key for a different - input context. Extra parameters are added to the referring - context's `__init__` method so a selected "value" variable - defined in the referenced context can be provided directly - instead of the file path (foreign key) variable. + input context. The referring context's `__init__` method + is modified such that the variable named `val` replaces + the `key` variable. Notes ----- diff --git a/flopy/mf6/utils/codegen/render.py b/flopy/mf6/utils/codegen/render.py index 345c110161..af8075cc29 100644 --- a/flopy/mf6/utils/codegen/render.py +++ b/flopy/mf6/utils/codegen/render.py @@ -2,10 +2,10 @@ from enum import Enum from typing import Any, Callable, Dict, Iterable, Optional, Tuple -_Predicate = Callable[[Any], bool] -_Transform = Callable[[Any], Dict[str, str]] -_Pair = Tuple[str, Any] -_Pairs = Iterable[_Pair] +Predicate = Callable[[Any], bool] +Transform = Callable[[Any], Dict[str, str]] +Pair = Tuple[str, Any] +Pairs = Iterable[Pair] def _try_get_enum_value(v: Any) -> Any: @@ -17,8 +17,8 @@ def renderable( *, keep_none: Optional[Iterable[str]] = None, quote_str: Optional[Iterable[str]] = None, - set_pairs: Optional[Iterable[Tuple[_Predicate, _Pairs]]] = None, - transform: Optional[Iterable[Tuple[_Predicate, _Transform]]] = None, + set_pairs: Optional[Iterable[Tuple[Predicate, Pairs]]] = None, + transform: Optional[Iterable[Tuple[Predicate, Transform]]] = None, ): """ Decorator for dataclasses which are meant From cde850f2e71c9e33810e771dfc8cd8472082ac5b Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 15 Oct 2024 18:28:10 -0400 Subject: [PATCH 20/75] docstring fixes --- flopy/mf6/utils/codegen/dfn.py | 32 ++++++++++--------- .../codegen/templates/docstring_params.jinja | 2 +- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index 667f312908..b30537f574 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -203,7 +203,7 @@ def load( referenced = dict() vars, meta = Dfn._load(f, **kwargs) - def _map(spec: Dict[str, Any], wrap: bool = False) -> Var: + def _map(spec: Dict[str, Any]) -> Var: """ Convert a variable specification from its representation in an input definition file to a Pythonic form. @@ -218,9 +218,6 @@ def _map(spec: Dict[str, Any], wrap: bool = False) -> Var: If a `default_value` is not provided, keywords are `False` by default, everything else is `None`. - If `wrap` is true, scalars will be wrapped as records. - This is useful to distinguish among choices in unions. - Any filepath variable whose name functions as a foreign key for another context will be given a pointer to the context. @@ -322,13 +319,17 @@ def _is_implicit_scalar_record(): children = {names[0]: _map(record)} kind = Var.Kind.List elif _is_implicit_scalar_record(): + fields = _fields(_name) children = { _name: Var( name=_name, kind=Var.Kind.Record, block=block, - children=_fields(_name), + children=fields, description=description, + meta={ + "type": f"[{', '.join([f.meta["type"] for f in fields.values()])}]" + }, ) } kind = Var.Kind.List @@ -349,31 +350,30 @@ def _is_implicit_scalar_record(): block=block, children=first.children if single else fields, description=description, + meta={ + "type": f"[{', '.join([v.meta["type"] for v in fields.values()])}]" + }, ) } kind = Var.Kind.List + type_ = f"[{', '.join([v.name for v in children.values()])}]" - # union (product), children are choices. - # scalar choices are wrapped as records. + # union (product), children are choices elif _type.startswith("keystring"): names = _type.split()[1:] children = { - v["name"]: _map(v, wrap=True) + v["name"]: _map(v) for v in vars.values(multi=True) if v["name"] in names and v.get("in_record", False) } kind = Var.Kind.Union + type_ = f"[{', '.join([v.name for v in children.values()])}]" # record (sum), children are fields elif _type.startswith("record"): children = _fields(_name) kind = Var.Kind.Record - - # are we wrapping a var into a record - # as a choice in a union? - elif wrap: - children = {_name: _map(spec)} - kind = Var.Kind.Record + type_ = f"[{', '.join([v.meta["type"] for v in children.values()])}]" # at this point, if it has a shape, it's an array elif shape is not None: @@ -383,10 +383,12 @@ def _is_implicit_scalar_record(): kind = Var.Kind.List else: kind = Var.Kind.Array + type_ = f"[{_type}]" # finally scalars else: kind = Var.Kind.Scalar + type_ = _type # create var return Var( @@ -404,7 +406,7 @@ def _is_implicit_scalar_record(): else default ), children=children, - meta={"ref": ref}, + meta={"ref": ref, "type": type_}, ) # pass the original DFN representation as diff --git a/flopy/mf6/utils/codegen/templates/docstring_params.jinja b/flopy/mf6/utils/codegen/templates/docstring_params.jinja index d8d5641d02..0b9d85d591 100644 --- a/flopy/mf6/utils/codegen/templates/docstring_params.jinja +++ b/flopy/mf6/utils/codegen/templates/docstring_params.jinja @@ -1,5 +1,5 @@ {%- for v in vars.values() recursive %} - {% if loop.depth > 1 %}* {% endif %}{{ v.name }}{% if v._type is defined and v._type is not none %} : {{ v._type }}{% endif %} + {% if loop.depth > 1 %}* {% endif %}{{ v.name }}{% if v.meta is defined and v.meta.type is defined %} : {{ v.meta.type }}{% endif %} {%- if v.description is defined and v.description is not none %} {{ v.description|wordwrap|indent(4 + (loop.depth * 4), first=True) }} {%- endif %} From d128c5a8206cc6c258ada6b590a657cc5bbf2acb Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 15 Oct 2024 18:36:00 -0400 Subject: [PATCH 21/75] restore test_generate_classes.py --- autotest/test_generate_classes.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/autotest/test_generate_classes.py b/autotest/test_generate_classes.py index c1de3bc706..c145ab65ab 100644 --- a/autotest/test_generate_classes.py +++ b/autotest/test_generate_classes.py @@ -2,7 +2,10 @@ from collections.abc import Iterable from os import environ from pathlib import Path +<<<<<<< HEAD from platform import system +======= +>>>>>>> 5cf4ed2a (restore test_generate_classes.py) from pprint import pprint from warnings import warn @@ -31,6 +34,10 @@ def pytest_generate_tests(metafunc): against all of the versions of mf6io flopy guarantees support for- maybe develop and latest release? Though some backwards compatibility seems ideal if possible. +<<<<<<< HEAD +======= + This would need changes in GH Actions CI test matrix. +>>>>>>> 5cf4ed2a (restore test_generate_classes.py) """ owner = "MODFLOW-USGS" From 04f40175b21fc1156fd5ca823546e3b250d4a26b Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 15 Oct 2024 18:39:23 -0400 Subject: [PATCH 22/75] 3.9 syntax? --- flopy/mf6/utils/codegen/dfn.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index b30537f574..ef0754ddbf 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -320,6 +320,7 @@ def _is_implicit_scalar_record(): kind = Var.Kind.List elif _is_implicit_scalar_record(): fields = _fields(_name) + types = [f.meta["type"] for f in fields.values()] children = { _name: Var( name=_name, @@ -327,9 +328,7 @@ def _is_implicit_scalar_record(): block=block, children=fields, description=description, - meta={ - "type": f"[{', '.join([f.meta["type"] for f in fields.values()])}]" - }, + meta={"type": f"[{', '.join(types)}]"}, ) } kind = Var.Kind.List From bd3e9eae998d1060b7d3d5bd9af6f146563fad43 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 15 Oct 2024 18:52:15 -0400 Subject: [PATCH 23/75] cleanup --- flopy/mf6/utils/codegen/make.py | 3 +-- flopy/mf6/utils/codegen/ref.py | 7 +++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/flopy/mf6/utils/codegen/make.py b/flopy/mf6/utils/codegen/make.py index 74af3b70e0..54b8ca55ce 100644 --- a/flopy/mf6/utils/codegen/make.py +++ b/flopy/mf6/utils/codegen/make.py @@ -18,8 +18,7 @@ def make_targets(dfn: Dfn, outdir: Path, verbose: bool = False): for context in Context.from_dfn(dfn): target = outdir / context.name.target with open(target, "w") as f: - source = _TEMPLATE.render(**context.render()) - f.write(source) + f.write(_TEMPLATE.render(**context.render())) if verbose: print(f"Wrote {target}") diff --git a/flopy/mf6/utils/codegen/ref.py b/flopy/mf6/utils/codegen/ref.py index 2637f0ba36..178d6b77d3 100644 --- a/flopy/mf6/utils/codegen/ref.py +++ b/flopy/mf6/utils/codegen/ref.py @@ -49,7 +49,14 @@ def from_dfn(cls, dfn: Dfn) -> Optional["Ref"]: Try to load a reference from the definition. Returns `None` if the definition cannot be referenced by other contexts. + """ + + # TODO: all this won't be necessary once we + # structure DFN format; we can then support + # subpackage references directly instead of + # by making assumptions about `dfn.meta` + if not dfn.meta or "dfn" not in dfn.meta: return None From 9a1171d6a1893aedfb448e77281c1d2c42d25954 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 15 Oct 2024 18:53:26 -0400 Subject: [PATCH 24/75] cleanup --- flopy/mf6/utils/codegen/dfn.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index ef0754ddbf..00561b7df0 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -320,7 +320,6 @@ def _is_implicit_scalar_record(): kind = Var.Kind.List elif _is_implicit_scalar_record(): fields = _fields(_name) - types = [f.meta["type"] for f in fields.values()] children = { _name: Var( name=_name, @@ -328,7 +327,9 @@ def _is_implicit_scalar_record(): block=block, children=fields, description=description, - meta={"type": f"[{', '.join(types)}]"}, + meta={ + "type": f"[{', '.join([f.meta['type'] for f in fields.values()])}]" + }, ) } kind = Var.Kind.List @@ -350,7 +351,7 @@ def _is_implicit_scalar_record(): children=first.children if single else fields, description=description, meta={ - "type": f"[{', '.join([v.meta["type"] for v in fields.values()])}]" + "type": f"[{', '.join([v.meta['type'] for v in fields.values()])}]" }, ) } @@ -372,7 +373,7 @@ def _is_implicit_scalar_record(): elif _type.startswith("record"): children = _fields(_name) kind = Var.Kind.Record - type_ = f"[{', '.join([v.meta["type"] for v in children.values()])}]" + type_ = f"[{', '.join([v.meta['type'] for v in children.values()])}]" # at this point, if it has a shape, it's an array elif shape is not None: From 323bfe6f3023e3cad02d6e172306253b3cc342a0 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 15 Oct 2024 19:03:49 -0400 Subject: [PATCH 25/75] docs/comments --- flopy/mf6/utils/codegen/dfn.py | 17 +++++++++++------ flopy/mf6/utils/codegen/templates/init.py.jinja | 6 +----- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index 00561b7df0..cbf527e848 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -110,10 +110,15 @@ def __init__( @staticmethod def _load(f, common: Optional[dict] = None) -> Tuple[OMD, List[str]]: """ - Internal use only. Loads the DFN as an ordered multi-dictionary, and + Internal use only. Loads the DFN as an ordered multi-dictionary* and a list of string metadata. This is later parsed into more structured form. We also store the original representation for now so it can be used by the shim. + + *The point of the OMD is to handle duplicate variable names; the only + case of this right now is 'auxiliary' which can appear in the options + block and again as a keyword in a record in a package data variable. + """ var = dict() vars = list() @@ -211,12 +216,12 @@ def _map(spec: Dict[str, Any]) -> Var: Notes ----- This involves expanding nested type hierarchies, mapping - types to roughly equivalent Python primitives/composites, - and other shaping. + types to roughly equivalent Python primitives/composites. + The composite inflation step will not be necessary after + DFNs move to a structured format. - The rules for optional variable defaults are as follows: - If a `default_value` is not provided, keywords are `False` - by default, everything else is `None`. + If a variable does not have a `default` attribute, it will + default to `False` if it is a keyword, otherwise to `None`. Any filepath variable whose name functions as a foreign key for another context will be given a pointer to the context. diff --git a/flopy/mf6/utils/codegen/templates/init.py.jinja b/flopy/mf6/utils/codegen/templates/init.py.jinja index 9ed03e647e..b49ea53239 100644 --- a/flopy/mf6/utils/codegen/templates/init.py.jinja +++ b/flopy/mf6/utils/codegen/templates/init.py.jinja @@ -1,11 +1,7 @@ def __init__( self, {%- for var in vars.values() if var.init_param %} - {%- if var.default is defined %} - {{ var.name }}={{ var.default }}, - {%- else %} - {{ var.name }}, - {%- endif -%} + {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, {%- endfor %} **kwargs, ): From 7eaaaf4ef9f8967ea92bf16faddb6cd345e4cccc Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 15 Oct 2024 19:26:44 -0400 Subject: [PATCH 26/75] backcompat fix? --- flopy/mf6/utils/codegen/ref.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flopy/mf6/utils/codegen/ref.py b/flopy/mf6/utils/codegen/ref.py index 178d6b77d3..e03c1a5127 100644 --- a/flopy/mf6/utils/codegen/ref.py +++ b/flopy/mf6/utils/codegen/ref.py @@ -103,8 +103,8 @@ def _subpkg(): def _parent(): line = lines["parent"] - _, param_name, _ = line.split() - return param_name + split = line.split() + return split[1] return ( cls(**_subpkg(), parent=_parent()) From a6ca045aedc3d03b2d02f76c671a2a4f05c6829a Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 16 Oct 2024 08:00:00 -0400 Subject: [PATCH 27/75] remove unused statement, fix mermaid diagrams in mf6_dev_guide, add boltons to readme --- README.md | 1 + autotest/test_codegen.py | 1 - docs/mf6_dev_guide.md | 40 ++++++++++++++++++++-------------------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index e22f58134a..6ac2c34a3e 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,7 @@ Installation FloPy requires **Python** 3.9+ with: ``` +boltons numpy >=1.20.3 matplotlib >=1.4.0 pandas >=2.0.0 diff --git a/autotest/test_codegen.py b/autotest/test_codegen.py index eaf7bb57b4..7143a5627f 100644 --- a/autotest/test_codegen.py +++ b/autotest/test_codegen.py @@ -7,7 +7,6 @@ PROJ_ROOT = get_project_root_path() MF6_PATH = PROJ_ROOT / "flopy" / "mf6" -TGT_PATH = MF6_PATH / "modflow" DFN_PATH = MF6_PATH / "data" / "dfn" DFN_NAMES = [ dfn.stem diff --git a/docs/mf6_dev_guide.md b/docs/mf6_dev_guide.md index 8f95ee9bbb..cb4e041ed2 100644 --- a/docs/mf6_dev_guide.md +++ b/docs/mf6_dev_guide.md @@ -12,12 +12,12 @@ All meta-data can be accessed from the flopy.mf6.data.mfstructure.MFStructure cl ```mermaid classDiagram - MFStructure --* "1" MFSimulationStructure : has - MFSimulationStructure --* "1+" MFModelStructure : has - MFModelStructure --* "1" MFInputFileStructure : has - MFInputFileStructure --* "1+" MFBlockStructure : has - MFBlockStructure --* "1+" MFDataStructure : has - MFDataStructure --* "1+" MFDataItemStructure : has + MFStructure *-- "1" MFSimulationStructure : has + MFSimulationStructure *-- "1+" MFModelStructure : has + MFModelStructure *-- "1" MFInputFileStructure : has + MFInputFileStructure *-- "1+" MFBlockStructure : has + MFBlockStructure *-- "1+" MFDataStructure : has + MFDataStructure *-- "1+" MFDataItemStructure : has ``` Figure 1: Generic data structure hierarchy. Connections show composition relationships. @@ -31,20 +31,20 @@ The package and data classes are related as shown below in figure 2. On the top ```mermaid classDiagram -MFPackage --* "1+" MFBlock : has -MFBlock --* "1+" MFData : has -MFPackage --* "1" MFInputFileStructure : has -MFBlock --* "1" MFBlockStructure : has -MFData --* "1" MFDataStructure : has -MFData --|> MFArray -MFArray --|> MFTransientArray -MFData --|> MFList -MFList --|> MFTransientList -MFData --|> MFScalar -MFScalar --|> MFTransientScalar -MFTransientData --|> MFTransientArray -MFTransientData --|> MFTransientList -MFTransientData --|> MFTransientScalar +MFPackage *-- "1+" MFBlock : has +MFBlock *-- "1+" MFData : has +MFPackage *-- "1" MFInputFileStructure : has +MFBlock *-- "1" MFBlockStructure : has +MFData *-- "1" MFDataStructure : has +MFData <|-- MFArray +MFArray <|-- MFTransientArray +MFData <|-- MFList +MFList <|-- MFTransientList +MFData <|-- MFScalar +MFScalar <|-- MFTransientScalar +MFTransientData <|-- MFTransientArray +MFTransientData <|-- MFTransientList +MFTransientData <|-- MFTransientScalar ``` Figure 2: FPMF6 package and data classes. Lines connecting classes show a relationship defined between the two connected classes. A "*" next to the class means that the class is a sub-class of the connected class. A "+" next to the class means that the class is contained within the connected class. From ed30e1c267d98e604c08ffc8c97fe52a0f55ff15 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 16 Oct 2024 08:00:42 -0400 Subject: [PATCH 28/75] bound boltons >= 1 --- README.md | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 6ac2c34a3e..23c5fe3b2d 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Installation FloPy requires **Python** 3.9+ with: ``` -boltons +boltons >=1.0 numpy >=1.20.3 matplotlib >=1.4.0 pandas >=2.0.0 diff --git a/pyproject.toml b/pyproject.toml index 8e57b38f29..f4d62f0160 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,7 +29,7 @@ classifiers = [ ] requires-python = ">=3.9" dependencies = [ - "boltons", + "boltons>=1.0", "Jinja2>=3.0", "numpy>=1.20.3", "matplotlib >=1.4.0", From 16a21de9bb55bdb83859f328b73c248dfd0d7037 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 16 Oct 2024 14:05:15 -0400 Subject: [PATCH 29/75] hints --- autotest/test_generate_classes.py | 7 ------- flopy/mf6/utils/codegen/dfn.py | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/autotest/test_generate_classes.py b/autotest/test_generate_classes.py index c145ab65ab..c1de3bc706 100644 --- a/autotest/test_generate_classes.py +++ b/autotest/test_generate_classes.py @@ -2,10 +2,7 @@ from collections.abc import Iterable from os import environ from pathlib import Path -<<<<<<< HEAD from platform import system -======= ->>>>>>> 5cf4ed2a (restore test_generate_classes.py) from pprint import pprint from warnings import warn @@ -34,10 +31,6 @@ def pytest_generate_tests(metafunc): against all of the versions of mf6io flopy guarantees support for- maybe develop and latest release? Though some backwards compatibility seems ideal if possible. -<<<<<<< HEAD -======= - This would need changes in GH Actions CI test matrix. ->>>>>>> 5cf4ed2a (restore test_generate_classes.py) """ owner = "MODFLOW-USGS" diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index cbf527e848..f6d7c66e0b 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -29,7 +29,7 @@ Dfns = Dict[str, "Dfn"] -def _try_parse_bool(value): +def _try_parse_bool(value: Any) -> Any: """ Try to parse a boolean from a string as represented in a DFN file, otherwise return the value unaltered. From 51f8ae348039a8ab68b65f5f0f106c8c5111c831 Mon Sep 17 00:00:00 2001 From: Marnix Date: Wed, 16 Oct 2024 11:44:38 +0200 Subject: [PATCH 30/75] Moving code from shim to jinja Removed qual_base Removed load and attrs, moved it to model, package and simulation. --- flopy/mf6/utils/codegen/shim.py | 14 +---- .../utils/codegen/templates/attrs.py.jinja | 9 --- .../codegen/templates/container.py.jinja | 4 +- .../utils/codegen/templates/context.py.jinja | 27 ++------- .../mf6/utils/codegen/templates/init.py.jinja | 4 +- .../mf6/utils/codegen/templates/load.py.jinja | 58 ------------------- .../utils/codegen/templates/model.py.jinja | 40 +++++++++++++ .../utils/codegen/templates/package.py.jinja | 24 ++++++++ .../codegen/templates/simulation.py.jinja | 40 +++++++++++++ 9 files changed, 113 insertions(+), 107 deletions(-) delete mode 100644 flopy/mf6/utils/codegen/templates/attrs.py.jinja delete mode 100644 flopy/mf6/utils/codegen/templates/load.py.jinja create mode 100644 flopy/mf6/utils/codegen/templates/model.py.jinja create mode 100644 flopy/mf6/utils/codegen/templates/package.py.jinja create mode 100644 flopy/mf6/utils/codegen/templates/simulation.py.jinja diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index 4ca953552f..5935750b8c 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -474,7 +474,7 @@ def _attr(var: dict) -> Optional[str]: return None attrs = [_attr(v) for v in ctx["vars"].values()] - return "\n ".join([a for a in attrs if a]) + return [a for a in attrs if a] def _init_body(ctx: dict) -> str: @@ -695,17 +695,6 @@ def _var(var: dict) -> List[str]: return [["header"] + _meta()] + _dfn() -def _qual_base(ctx: dict): - base = ctx["base"] - if base == "MFSimulationBase": - module = "mfsimbase" - elif base == "MFModel": - module = "mfmodel" - else: - module = "mfpackage" - return f"{module}.{base}" - - SHIM = { "keep_none": ["default", "block", "metadata"], "quote_str": ["default"], @@ -714,7 +703,6 @@ def _qual_base(ctx: dict): _is_context, [ ("dfn", _dfn), - ("qual_base", _qual_base), ("class_attrs", _class_attrs), ("init_body", _init_body), ], diff --git a/flopy/mf6/utils/codegen/templates/attrs.py.jinja b/flopy/mf6/utils/codegen/templates/attrs.py.jinja deleted file mode 100644 index 55653dc42d..0000000000 --- a/flopy/mf6/utils/codegen/templates/attrs.py.jinja +++ /dev/null @@ -1,9 +0,0 @@ - {%- if base == "MFModel" %} - model_type = "{{ name.title }}" - {%- elif base == "MFPackage" %} - {{ class_attrs }} - package_abbr = "{% if name.l != "sln" and name.l != "sim" and name.l != "exg" and name.l is not none %}{{ name.l }}{% endif %}{{ name.r }}" - _package_type = "{{ name.r }}" - dfn_file_name = "{% if name.l is not none %}{{ name.l }}-{% elif name.l is none %}sim-{% endif %}{{ name.r }}.dfn" - dfn = {{ dfn|pprint|indent(10) }} - {% endif -%} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/container.py.jinja b/flopy/mf6/utils/codegen/templates/container.py.jinja index a61ceb0f9b..ac4f7997d6 100644 --- a/flopy/mf6/utils/codegen/templates/container.py.jinja +++ b/flopy/mf6/utils/codegen/templates/container.py.jinja @@ -1,4 +1,4 @@ -class {{ name.title.title() }}Packages(mfpackage.MFChildPackages): +class {{ name.title.title() }}Packages(MFChildPackages): """ {{ name.title.title() }}Packages is a container class for the Modflow{{ name.title.title() }} class. @@ -53,4 +53,4 @@ class {{ name.title.title() }}Packages(mfpackage.MFChildPackages): child_builder_call=True, ) self._append_package(new_package, filename) -{% endif %} \ No newline at end of file +{% endif %} diff --git a/flopy/mf6/utils/codegen/templates/context.py.jinja b/flopy/mf6/utils/codegen/templates/context.py.jinja index 75d3271db6..34537ea427 100644 --- a/flopy/mf6/utils/codegen/templates/context.py.jinja +++ b/flopy/mf6/utils/codegen/templates/context.py.jinja @@ -1,25 +1,8 @@ # autogenerated file, do not modify -from os import PathLike, curdir -from typing import Union - -from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator -from flopy.mf6 import mfpackage -from flopy.mf6 import mfmodel -{# avoid circular import; some pkgs (e.g. mfnam) are used by mfsimbase.py #} {% if base == "MFSimulationBase" %} -from flopy.mf6 import mfsimbase +{% include "simulation.py.jinja" %} +{% elif base == "MFModel" %} +{% include "model.py.jinja" %} +{% else %} +{% include "package.py.jinja" %} {% endif %} - -class {% if base == "MFSimulationBase" %}MF{% else %}Modflow{% endif %}{{ name.title.title() }}({{ qual_base }}): - {% include "docstring.jinja" %} - - {% include "attrs.py.jinja" %} - - {% include "init.py.jinja" %} - - {% include "load.py.jinja" %} - -{# TODO: cleaner way to filter out hpc subpkg? #} -{% if "ref" in meta and name.r != "hpc" %} -{% include "container.py.jinja" %} -{% endif %} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/init.py.jinja b/flopy/mf6/utils/codegen/templates/init.py.jinja index b49ea53239..78da10e5e6 100644 --- a/flopy/mf6/utils/codegen/templates/init.py.jinja +++ b/flopy/mf6/utils/codegen/templates/init.py.jinja @@ -6,6 +6,4 @@ def __init__( **kwargs, ): {{ init_body }} - {% if base == "MFPackage" %} - self._init_complete = True - {% endif %} \ No newline at end of file + \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/load.py.jinja b/flopy/mf6/utils/codegen/templates/load.py.jinja deleted file mode 100644 index e36e13c64e..0000000000 --- a/flopy/mf6/utils/codegen/templates/load.py.jinja +++ /dev/null @@ -1,58 +0,0 @@ -{% if base == "MFSimulationBase" %} - @classmethod - def load( - cls, - sim_name="modflowsim", - version="mf6", - exe_name: Union[str, PathLike] = "mf6", - sim_ws: Union[str, PathLike] = curdir, - strict=True, - verbosity_level=1, - load_only=None, - verify_data=False, - write_headers=True, - lazy_io=False, - use_pandas=True, - ): - return mfsimbase.MFSimulationBase.load( - cls, - sim_name, - version, - exe_name, - sim_ws, - strict, - verbosity_level, - load_only, - verify_data, - write_headers, - lazy_io, - use_pandas, - ) -{% elif base == "MFModel" %} - @classmethod - def load( - cls, - simulation, - structure, - modelname="NewModel", - model_nam_file="modflowtest.nam", - version="mf6", - exe_name="mf6", - strict=True, - model_rel_path=curdir, - load_only=None, - ): - return mfmodel.MFModel.load_base( - cls, - simulation, - structure, - modelname, - model_nam_file, - "{{ name.title }}6", - version, - exe_name, - strict, - model_rel_path, - load_only, - ) -{% endif %} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/model.py.jinja b/flopy/mf6/utils/codegen/templates/model.py.jinja new file mode 100644 index 0000000000..ccb9b8ecf6 --- /dev/null +++ b/flopy/mf6/utils/codegen/templates/model.py.jinja @@ -0,0 +1,40 @@ +from os import PathLike, curdir +from typing import Union + +from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator +from flopy.mf6.mfmodel import MFModel + + +class Modflow{{ name.title.title() }}(MFModel): + {% include "docstring.jinja" %} + + model_type = "{{ name.title }}" + + {% include "init.py.jinja" %} + + @classmethod + def load( + cls, + simulation, + structure, + modelname="NewModel", + model_nam_file="modflowtest.nam", + version="mf6", + exe_name="mf6", + strict=True, + model_rel_path=curdir, + load_only=None, + ): + return MFModel.load_base( + cls, + simulation, + structure, + modelname, + model_nam_file, + "{{ name.title }}6", + version, + exe_name, + strict, + model_rel_path, + load_only, + ) diff --git a/flopy/mf6/utils/codegen/templates/package.py.jinja b/flopy/mf6/utils/codegen/templates/package.py.jinja new file mode 100644 index 0000000000..50da2d813c --- /dev/null +++ b/flopy/mf6/utils/codegen/templates/package.py.jinja @@ -0,0 +1,24 @@ +from os import PathLike, curdir +from typing import Union + +from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator +from flopy.mf6.mfpackage import MFPackage, MFChildPackages + +class Modflow{{ name.title.title() }}(MFPackage): + {% include "docstring.jinja" %} + + {%- for item in class_attrs %} + {{ item }} + {%- endfor %} + package_abbr = "{{ '' if name.l in ["sln", "sim", "exg", none] else name.l }}{{ name.r }}" + _package_type = "{{ name.r }}" + dfn_file_name = "{{ name.l | default('sim', true) }}-{{ name.r }}.dfn" + dfn = {{ dfn|pprint|indent(10) }} + + {% include "init.py.jinja" %} + self._init_complete = True + +{# TODO: cleaner way to filter out hpc subpkg? #} +{% if "ref" in meta and name.r != "hpc" %} +{% include "container.py.jinja" %} +{% endif %} diff --git a/flopy/mf6/utils/codegen/templates/simulation.py.jinja b/flopy/mf6/utils/codegen/templates/simulation.py.jinja new file mode 100644 index 0000000000..84437a0c43 --- /dev/null +++ b/flopy/mf6/utils/codegen/templates/simulation.py.jinja @@ -0,0 +1,40 @@ +from os import PathLike, curdir +from typing import Union + +from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator +from flopy.mf6.mfsimbase import MFSimulationBase + +class MF{{ name.title.title() }}(MFSimulationBase): + {% include "docstring.jinja" %} + + {% include "init.py.jinja" %} + + @classmethod + def load( + cls, + sim_name="modflowsim", + version="mf6", + exe_name: Union[str, PathLike] = "mf6", + sim_ws: Union[str, PathLike] = curdir, + strict=True, + verbosity_level=1, + load_only=None, + verify_data=False, + write_headers=True, + lazy_io=False, + use_pandas=True, + ): + return MFSimulationBase.load( + cls, + sim_name, + version, + exe_name, + sim_ws, + strict, + verbosity_level, + load_only, + verify_data, + write_headers, + lazy_io, + use_pandas, + ) From 1e12080347ff2d9d8a46c2ad2bbca39d939b537e Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Sun, 27 Oct 2024 19:07:59 -0400 Subject: [PATCH 31/75] move more from shim to jinja --- autotest/test_codegen.py | 52 +- .../utils/codegen/{make.py => __init__.py} | 13 +- flopy/mf6/utils/codegen/context.py | 59 +- flopy/mf6/utils/codegen/dfn.py | 155 ++- flopy/mf6/utils/codegen/ref.py | 116 --- .../codegen/{render.py => renderable.py} | 74 +- flopy/mf6/utils/codegen/shim.py | 942 ++++++------------ .../codegen/templates/container.py.jinja | 56 -- .../utils/codegen/templates/context.py.jinja | 8 - .../utils/codegen/templates/docstring.jinja | 12 - .../codegen/templates/docstring_methods.jinja | 13 - .../codegen/templates/docstring_params.jinja | 9 - .../utils/codegen/templates/exchange.py.jinja | 82 ++ .../mf6/utils/codegen/templates/init.py.jinja | 9 - .../mf6/utils/codegen/templates/macros.jinja | 11 + .../utils/codegen/templates/model.py.jinja | 66 +- .../utils/codegen/templates/package.py.jinja | 124 ++- .../codegen/templates/simulation.py.jinja | 72 +- flopy/mf6/utils/codegen/utils.py | 34 + flopy/mf6/utils/createpackages.py | 6 +- 20 files changed, 898 insertions(+), 1015 deletions(-) rename flopy/mf6/utils/codegen/{make.py => __init__.py} (87%) delete mode 100644 flopy/mf6/utils/codegen/ref.py rename flopy/mf6/utils/codegen/{render.py => renderable.py} (68%) delete mode 100644 flopy/mf6/utils/codegen/templates/container.py.jinja delete mode 100644 flopy/mf6/utils/codegen/templates/context.py.jinja delete mode 100644 flopy/mf6/utils/codegen/templates/docstring.jinja delete mode 100644 flopy/mf6/utils/codegen/templates/docstring_methods.jinja delete mode 100644 flopy/mf6/utils/codegen/templates/docstring_params.jinja create mode 100644 flopy/mf6/utils/codegen/templates/exchange.py.jinja delete mode 100644 flopy/mf6/utils/codegen/templates/init.py.jinja create mode 100644 flopy/mf6/utils/codegen/templates/macros.jinja create mode 100644 flopy/mf6/utils/codegen/utils.py diff --git a/autotest/test_codegen.py b/autotest/test_codegen.py index 7143a5627f..9dbd771f9b 100644 --- a/autotest/test_codegen.py +++ b/autotest/test_codegen.py @@ -1,9 +1,9 @@ import pytest from autotest.conftest import get_project_root_path +from flopy.mf6.utils.codegen import make_all, make_targets from flopy.mf6.utils.codegen.context import Context from flopy.mf6.utils.codegen.dfn import Dfn -from flopy.mf6.utils.codegen.make import make_all, make_targets PROJ_ROOT = get_project_root_path() MF6_PATH = PROJ_ROOT / "flopy" / "mf6" @@ -17,34 +17,40 @@ @pytest.mark.parametrize("dfn_name", DFN_NAMES) def test_dfn_load(dfn_name): - dfn_path = DFN_PATH / f"{dfn_name}.dfn" - - common_path = DFN_PATH / "common.dfn" - with open(common_path, "r") as f: - common, _ = Dfn._load(f) - - with open(dfn_path, "r") as f: - dfn = Dfn.load(f, name=Dfn.Name(*dfn_name.split("-")), common=common) - if dfn_name in ["sln-ems", "exg-gwfprt", "exg-gwfgwe", "exg-gwfgwt"]: - assert not any(dfn) - else: - assert any(dfn) + with ( + open(DFN_PATH / "common.dfn", "r") as common_file, + open(DFN_PATH / f"{dfn_name}.dfn", "r") as dfn_file, + ): + name = Dfn.Name.parse(dfn_name) + common, _ = Dfn._load(common_file) + dfn = Dfn.load(dfn_file, name=name, common=common) + + if name in [ + ("sln", "ems"), + ("exg", "gwfprt"), + ("exg", "gwfgwe"), + ("exg", "gwfgwt"), + ]: + assert not any(dfn) + else: + assert any(dfn) @pytest.mark.parametrize("dfn_name", DFN_NAMES) def test_make_targets(dfn_name, function_tmpdir): - common_path = DFN_PATH / "common.dfn" - with open(common_path, "r") as f: - common, _ = Dfn._load(f) - - with open(DFN_PATH / f"{dfn_name}.dfn", "r") as f: - dfn = Dfn.load(f, name=Dfn.Name(*dfn_name.split("-")), common=common) + with ( + open(DFN_PATH / "common.dfn", "r") as common_file, + open(DFN_PATH / f"{dfn_name}.dfn", "r") as dfn_file, + ): + name = Dfn.Name.parse(dfn_name) + common, _ = Dfn._load(common_file) + dfn = Dfn.load(dfn_file, name=name, common=common) make_targets(dfn, function_tmpdir, verbose=True) - - for name in Context.Name.from_dfn(dfn): - source_path = function_tmpdir / name.target - assert source_path.is_file() + assert all( + (function_tmpdir / name.target).is_file() + for name in Context.Name.from_dfn(dfn) + ) def test_make_all(function_tmpdir): diff --git a/flopy/mf6/utils/codegen/make.py b/flopy/mf6/utils/codegen/__init__.py similarity index 87% rename from flopy/mf6/utils/codegen/make.py rename to flopy/mf6/utils/codegen/__init__.py index 54b8ca55ce..68466b91ed 100644 --- a/flopy/mf6/utils/codegen/make.py +++ b/flopy/mf6/utils/codegen/__init__.py @@ -3,22 +3,23 @@ from jinja2 import Environment, PackageLoader from flopy.mf6.utils.codegen.context import Context -from flopy.mf6.utils.codegen.dfn import Dfn, Dfns -from flopy.mf6.utils.codegen.ref import Ref, Refs +from flopy.mf6.utils.codegen.dfn import Dfn, Dfns, Ref, Refs + +__all__ = ["make_targets", "make_all"] _TEMPLATE_LOADER = PackageLoader("flopy", "mf6/utils/codegen/templates/") _TEMPLATE_ENV = Environment(loader=_TEMPLATE_LOADER) -_TEMPLATE_NAME = "context.py.jinja" -_TEMPLATE = _TEMPLATE_ENV.get_template(_TEMPLATE_NAME) def make_targets(dfn: Dfn, outdir: Path, verbose: bool = False): """Generate Python source file(s) from the given input definition.""" for context in Context.from_dfn(dfn): - target = outdir / context.name.target + name = context.name + target = outdir / name.target + template = _TEMPLATE_ENV.get_template(name.template) with open(target, "w") as f: - f.write(_TEMPLATE.render(**context.render())) + f.write(template.render(**context.render())) if verbose: print(f"Wrote {target}") diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py index 77318d0109..91fd165570 100644 --- a/flopy/mf6/utils/codegen/context.py +++ b/flopy/mf6/utils/codegen/context.py @@ -8,9 +8,8 @@ Optional, ) -from flopy.mf6.utils.codegen.dfn import Dfn, Vars -from flopy.mf6.utils.codegen.ref import Ref -from flopy.mf6.utils.codegen.render import renderable +from flopy.mf6.utils.codegen.dfn import Dfn, Ref, Vars +from flopy.mf6.utils.codegen.renderable import renderable from flopy.mf6.utils.codegen.shim import SHIM @@ -39,7 +38,7 @@ class Context: class Name(NamedTuple): """ - Uniquely identifies an input context. A context + Uniquely identifies an input context. The name consists of a left term and optional right term. Notes @@ -50,10 +49,11 @@ class Name(NamedTuple): From the context name several other things are derived: + - a description of the context - the input context class' name - - a description of the context class - - the name of the source file to write + - the template the context will populate - the base class the context inherits from + - the name of the source file the context is in - the name of the parent parameter in the context class' `__init__` method, if it can have a parent @@ -70,7 +70,6 @@ def title(self) -> str: remains unique. The title is substituted into the file name and class name. """ - l, r = self if self == ("sim", "nam"): return "simulation" @@ -82,7 +81,7 @@ def title(self) -> str: return r if l in ["sln", "exg"]: return r - return f"{l}{r}" + return l + r @property def base(self) -> str: @@ -99,6 +98,18 @@ def target(self) -> str: """The source file name to generate.""" return f"mf{self.title}.py" + @property + def template(self) -> str: + """The template file to use.""" + if self.base == "MFSimulationBase": + return "simulation.py.jinja" + elif self.base == "MFModel": + return "model.py.jinja" + elif self.base == "MFPackage": + if self.l == "exg": + return "exchange.py.jinja" + return "package.py.jinja" + @property def description(self) -> str: """A description of the input context.""" @@ -109,29 +120,11 @@ def description(self) -> str: elif self.base == "MFModel": return f"Modflow{title} defines a {l.upper()} model." elif self.base == "MFSimulationBase": - return """ - MFSimulation is used to load, build, and/or save a MODFLOW 6 simulation. - A MFSimulation object must be created before creating any of the MODFLOW 6 - model objects.""" - - def parent(self, ref: Optional[Ref] = None) -> Optional[str]: - """ - Return the name of the parent `__init__` method parameter, - or `None` if the context cannot have parents. Contexts can - have more than one possible parent, in which case the name - of the parameter is of the pattern `name1_or_..._or_nameN`. - """ - if ref: - return ref.parent - if self == ("sim", "nam"): - return None - elif ( - self.l is None - or self.r is None - or self.l in ["sim", "exg", "sln"] - ): - return "simulation" - return "model" + return ( + "MFSimulation is used to load, build, and/or save a MODFLOW 6 simulation." + " A MFSimulation object must be created before creating any of the MODFLOW" + " 6 model objects." + ) @staticmethod def from_dfn(dfn: Dfn) -> List["Context.Name"]: @@ -172,7 +165,6 @@ def from_dfn(dfn: Dfn) -> List["Context.Name"]: name: Name vars: Vars base: Optional[type] = None - parent: Optional[str] = None description: Optional[str] = None meta: Optional[Dict[str, Any]] = None @@ -183,18 +175,15 @@ def from_dfn(cls, dfn: Dfn) -> Iterator["Context"]: These are structured representations of input context classes. Each input definition yields one or more input contexts. """ - meta = dfn.meta.copy() ref = Ref.from_dfn(dfn) if ref: meta["ref"] = ref - for name in Context.Name.from_dfn(dfn): yield Context( name=name, vars=dfn.data, base=name.base, - parent=name.parent(ref), description=name.description, meta=meta, ) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index f6d7c66e0b..dd4b545611 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -17,6 +17,8 @@ from boltons.dictutils import OMD +from flopy.mf6.utils.codegen.utils import try_literal_eval, try_parse_bool + _SCALARS = { "keyword", "integer", @@ -27,30 +29,7 @@ Vars = Dict[str, "Var"] Dfns = Dict[str, "Dfn"] - - -def _try_parse_bool(value: Any) -> Any: - """ - Try to parse a boolean from a string as represented - in a DFN file, otherwise return the value unaltered. - """ - - if isinstance(value, str): - value = value.lower() - if value in ["true", "false"]: - return value == "true" - return value - - -def _try_literal_eval(value: str) -> Any: - """ - Try to parse a string as a literal. If this fails, - return the value unaltered. - """ - try: - return literal_eval(value) - except (SyntaxError, ValueError): - return value +Refs = Dict[str, "Ref"] @dataclass @@ -87,13 +66,20 @@ class Dfn(UserDict): class Name(NamedTuple): """ - Uniquely identifies an input definition. A name - consists of a left term and optional right term. + Uniquely identifies an input definition. + Consists of a left term and a right term. """ l: str r: str + @classmethod + def parse(cls, v: str) -> "Dfn.Name": + try: + return cls(*v.split("-")) + except: + raise ValueError(f"Bad DFN name format: {v}") + name: Optional[Name] meta: Optional[Dict[str, Any]] @@ -232,9 +218,8 @@ def _map(spec: Dict[str, Any]) -> Var: # stay a string except default values, which we'll # try to parse as arbitrary literals below, and at # some point types, once we introduce type hinting - spec = {k: _try_parse_bool(v) for k, v in spec.items()} + spec = {k: try_parse_bool(v) for k, v in spec.items()} - # pull off attributes we're interested in _name = spec["name"] _type = spec.get("type", None) block = spec.get("block", None) @@ -406,11 +391,12 @@ def _is_implicit_scalar_record(): block=block, description=description, default=( - _try_literal_eval(default) - if _type != "string" - else default + try_literal_eval(default) if _type != "string" else default ), children=children, + # type is a string for now, when + # introducing type hints make it + # a proper type... meta={"ref": ref, "type": type_}, ) @@ -440,3 +426,110 @@ def _is_implicit_scalar_record(): "refs": referenced, }, ) + + +@dataclass +class Ref: + """ + A foreign-key-like reference between a file input variable + and another input definition. This allows an input context + to refer to another input context, by including a filepath + variable whose name acts as a foreign key for a different + input context. The referring context's `__init__` method + is modified such that the variable named `val` replaces + the `key` variable. + + Notes + ----- + This class is used to represent subpackage references. + + Parameters + ---------- + key : str + The name of the foreign key file input variable. + val : str + The name of the data variable in the referenced context. + abbr : str + An abbreviation of the referenced context's name. + param : str + The referenced parameter name. + parents : List[str] + The referenced context's supported parents. + description : Optional[str] + The reference's description. + """ + + key: str + val: str + abbr: str + param: str + parent: str + description: Optional[str] + + @classmethod + def from_dfn(cls, dfn: Dfn) -> Optional["Ref"]: + """ + Try to load a reference from the definition. + Returns `None` if the definition cannot be + referenced by other contexts. + """ + + # TODO: all this won't be necessary once we + # structure DFN format; we can then support + # subpackage references directly instead of + # by making assumptions about `dfn.meta` + + if not dfn.meta or "dfn" not in dfn.meta: + return None + + _, meta = dfn.meta["dfn"] + + lines = { + "subpkg": next( + iter( + m + for m in meta + if isinstance(m, str) and m.startswith("subpac") + ), + None, + ), + "parent": next( + iter( + m + for m in meta + if isinstance(m, str) and m.startswith("parent") + ), + None, + ), + } + + def _subpkg(): + line = lines["subpkg"] + _, key, abbr, param, val = line.split() + matches = [v for v in dfn.values() if v.name == val] + if not any(matches): + descr = None + else: + if len(matches) > 1: + warn(f"Multiple matches for referenced variable {val}") + match = matches[0] + descr = match.description + + return { + "key": key, + "val": val, + "abbr": abbr, + "param": param, + "description": descr, + } + + def _parent(): + line = lines["parent"] + split = line.split() + return split[1] + + return ( + cls(**_subpkg(), parent=_parent()) + if all(v for v in lines.values()) + else None + ) diff --git a/flopy/mf6/utils/codegen/ref.py b/flopy/mf6/utils/codegen/ref.py deleted file mode 100644 index e03c1a5127..0000000000 --- a/flopy/mf6/utils/codegen/ref.py +++ /dev/null @@ -1,116 +0,0 @@ -from dataclasses import dataclass -from typing import Dict, Optional -from warnings import warn - -from flopy.mf6.utils.codegen.dfn import Dfn - - -@dataclass -class Ref: - """ - A foreign-key-like reference between a file input variable - and another input definition. This allows an input context - to refer to another input context, by including a filepath - variable whose name acts as a foreign key for a different - input context. The referring context's `__init__` method - is modified such that the variable named `val` replaces - the `key` variable. - - Notes - ----- - This class is used to represent subpackage references. - - Parameters - ---------- - key : str - The name of the foreign key file input variable. - val : str - The name of the data variable in the referenced context. - abbr : str - An abbreviation of the referenced context's name. - param : str - The referenced parameter name. - parents : List[str] - The referenced context's supported parents. - description : Optional[str] - The reference's description. - """ - - key: str - val: str - abbr: str - param: str - parent: str - description: Optional[str] - - @classmethod - def from_dfn(cls, dfn: Dfn) -> Optional["Ref"]: - """ - Try to load a reference from the definition. - Returns `None` if the definition cannot be - referenced by other contexts. - - """ - - # TODO: all this won't be necessary once we - # structure DFN format; we can then support - # subpackage references directly instead of - # by making assumptions about `dfn.meta` - - if not dfn.meta or "dfn" not in dfn.meta: - return None - - _, meta = dfn.meta["dfn"] - - lines = { - "subpkg": next( - iter( - m - for m in meta - if isinstance(m, str) and m.startswith("subpac") - ), - None, - ), - "parent": next( - iter( - m - for m in meta - if isinstance(m, str) and m.startswith("parent") - ), - None, - ), - } - - def _subpkg(): - line = lines["subpkg"] - _, key, abbr, param, val = line.split() - matches = [v for v in dfn.values() if v.name == val] - if not any(matches): - descr = None - else: - if len(matches) > 1: - warn(f"Multiple matches for referenced variable {val}") - match = matches[0] - descr = match.description - - return { - "key": key, - "val": val, - "abbr": abbr, - "param": param, - "description": descr, - } - - def _parent(): - line = lines["parent"] - split = line.split() - return split[1] - - return ( - cls(**_subpkg(), parent=_parent()) - if all(v for v in lines.values()) - else None - ) - - -Refs = Dict[str, Ref] diff --git a/flopy/mf6/utils/codegen/render.py b/flopy/mf6/utils/codegen/renderable.py similarity index 68% rename from flopy/mf6/utils/codegen/render.py rename to flopy/mf6/utils/codegen/renderable.py index af8075cc29..3aeaafd6cd 100644 --- a/flopy/mf6/utils/codegen/render.py +++ b/flopy/mf6/utils/codegen/renderable.py @@ -1,17 +1,36 @@ +""" +This module contains a decorator intended to +allow modifying dataclass instances to make +them more palatable for templates. It also +keeps implementation details incidental to +the current design of MF6 input framework +cleanly isolated from the reimplementation +of which this code is a part, which aims +for a more general approach. + +Jinja supports attribute- and dictionary- +based access on arbitrary objects but does +not support arbitrary expressions, and has +only a limited set of custom filters; this +can make it awkward to express some things, +which transformations can also remedy. + +Edge cases in the MF6 classes, e.g. the logic +determining the contents of generated classes, +can also be implemented with transformations. +""" + from dataclasses import asdict -from enum import Enum from typing import Any, Callable, Dict, Iterable, Optional, Tuple +from flopy.mf6.utils.codegen.utils import try_get_enum_value + Predicate = Callable[[Any], bool] Transform = Callable[[Any], Dict[str, str]] Pair = Tuple[str, Any] Pairs = Iterable[Pair] -def _try_get_enum_value(v: Any) -> Any: - return v.value if isinstance(v, Enum) else v - - def renderable( maybe_cls=None, *, @@ -54,38 +73,12 @@ def renderable( Notes ----- - This decorator is intended as a convenient - way to modify dataclass instances to make - them more palatable for templates. It also - keeps implementation details incidental to - the current design of MF6 input framework - cleanly isolated from the reimplementation - of which this code is a part, which aims - for a more general approach. - - Jinja supports attribute- and dictionary- - based access on arbitrary objects but does - not support arbitrary expressions, and has - only a limited set of custom filters; this - can make it awkward to express some things, - which transformations can also remedy. - - Edge cases in the MF6 classes, e.g. the logic - determining the contents of generated classes, - can also be implemented with transformations. - This allows keeping the templating module as - generic as possible and inserting "shims" to - incrementally rewrite the existing framework. - Because a transformation function accepts an instance of a dataclass and converts it to a dictionary, only one transformation function (of the first matching predicate) is applied. - References - ---------- - This pattern was heavily inspired by `attrs`' - use of class decorators. + This was inspired by `attrs` class decorators. """ quote_str = quote_str or list() @@ -95,13 +88,8 @@ def renderable( def __renderable(cls): def _render(d: dict) -> dict: - """ - Render the dictionary recursively, - with requested value modifications. - """ - def _render_val(k, v): - v = _try_get_enum_value(v) + v = try_get_enum_value(v) if ( k in quote_str and isinstance(v, str) @@ -115,16 +103,10 @@ def _render_val(k, v): return { k: _render_val(k, v) for k, v in d.items() - # drop nones except where requested if (k in keep_none or v is not None) } def _dict(o): - """ - Convert the dataclass instance to a dictionary, - applying a transformation if applicable and any - extra key/value pairs if provided. - """ d = dict(o) for p, t in transform: if p(o): @@ -145,9 +127,7 @@ def _dict_factory(o): return _render(_dict(o)) def render(self) -> dict: - """ - Recursively render the dataclass instance. - """ + """Recursively render the dataclass instance.""" return _render(asdict(self, dict_factory=_dict_factory)) setattr(cls, "render", render) diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index 5935750b8c..2b02ecd020 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -2,409 +2,15 @@ The purpose of this module is to keep special handling necessary to support the current `flopy.mf6` generated classes separate from more general templating and code -generation infrastructure. It has no dependency on the -rest of the `flopy.mf6.utils.codegen` module. +generation infrastructure. """ -import os from keyword import kwlist +from pprint import pformat from typing import List, Optional -def _is_context(o) -> bool: - """Whether the object is an input context.""" - d = dict(o) - return "name" in d and "base" in d - - -def _is_var(o) -> bool: - """Whether the object is an input context variable.""" - d = dict(o) - return "name" in d and "kind" in d - - -def _is_init_param(o) -> bool: - """Whether the object is an `__init__` method parameter.""" - d = dict(o) - if d.get("ref", None): - return False - if d["name"] in ["output"]: - return False - return True - - -def _is_container_init_param(o) -> bool: - """ - Whether the object is a parameter of the corresponding - package container class. This is only relevant for some - subpackage contexts. - """ - d = dict(o) - if d["name"] in ["output"]: - return False - return True - - -def _set_exg_vars(ctx: dict) -> dict: - """ - Modify variables for an exchange context. - """ - vars_ = ctx["vars"].copy() - vars_ = { - "loading_package": { - "name": "loading_package", - "_type": "bool", - "description": ( - "Do not set this parameter. It is intended for " - "debugging and internal processing purposes only." - ), - "default": False, - "init_param": True, - }, - "exgtype": { - "name": "exgtype", - "_type": "str", - "default": f"{ctx['name'].r[:3].upper()}6-{ctx['name'].r[:3].upper()}6", - "description": "The exchange type.", - "init_param": True, - }, - "exgmnamea": { - "name": "exgmnamea", - "_type": "str", - "description": "The name of the first model in the exchange.", - "default": None, - "init_param": True, - }, - "exgmnameb": { - "name": "exgmnameb", - "_type": "str", - "description": "The name of the second model in the exchange.", - "default": None, - "init_param": True, - }, - **vars_, - "filename": { - "name": "filename", - "_type": "pathlike", - "description": "File name for this package.", - "default": None, - "init_param": True, - }, - "pname": { - "name": "pname", - "_type": "str", - "description": "Package name for this package.", - "default": None, - "init_param": True, - }, - } - - refs = ctx.get("meta", dict()).get("refs", dict()) - if any(refs): - for key, ref in refs.items(): - key_var = vars_.get(key, None) - if not key_var: - continue - vars_[key] = { - **key_var, - "name": ref["val"], - "description": ref.get("description", None), - "ref": ref, - "init_param": True, - "default": None, - "construct_package": ref["abbr"], - "construct_data": ref["val"], - "parameter_name": ref["param"], - } - - ctx["vars"] = vars_ - return ctx - - -def _set_pkg_vars(ctx: dict) -> dict: - """Modify variables for a package context.""" - vars_ = ctx["vars"].copy() - - if ctx["name"].r == "nam": - init_skip = ["export_netcdf", "nc_filerecord"] - elif ctx["name"] == ("utl", "ts"): - init_skip = ["method", "interpolation_method_single", "sfac"] - else: - init_skip = [] - for k in init_skip: - var = vars_.get(k, None) - if var: - var["init_param"] = False - var["init_skip"] = True - vars_[k] = var - - vars_ = { - "loading_package": { - "name": "loading_package", - "_type": "bool", - "description": ( - "Do not set this variable. It is intended for debugging " - "and internal processing purposes only." - ), - "default": False, - "init_param": True, - }, - **vars_, - "filename": { - "name": "filename", - "_type": "str", - "description": "File name for this package.", - "default": None, - "init_param": True, - }, - "pname": { - "name": "pname", - "_type": "str", - "description": "Package name for this package.", - "default": None, - "init_param": True, - }, - } - - if ctx["name"].l == "utl": - vars_["parent_file"] = { - "name": "parent_file", - "_type": "pathlike", - "description": ( - "Parent package file that references this package. Only needed " - "for utility packages (mfutl*). For example, mfutllaktab package " - "must have a mfgwflak package parent_file." - ), - } - - refs = ctx.get("meta", dict()).get("refs", dict()) - if any(refs): - for key, ref in refs.items(): - key_var = vars_.get(key, None) - if not key_var: - continue - vars_[key] = { - **key_var, - "name": ref["val"], - "description": ref.get("description", None), - "ref": ref, - "init_param": ctx["name"].r != "nam", - "default": None, - "construct_package": ref["abbr"], - "construct_data": ref["val"], - "parameter_name": ref["param"], - } - - ctx["vars"] = vars_ - return ctx - - -def _set_mdl_vars(ctx: dict) -> dict: - """Modify variables for a model context.""" - vars_ = ctx["vars"].copy() - init_skip = ["packages", "export_netcdf", "nc_filerecord"] - for k in init_skip: - var = vars_.get(k, None) - if var: - var["init_param"] = False - var["init_skip"] = True - vars_[k] = var - vars_ = { - "modelname": { - "name": "modelname", - "_type": "str", - "description": "The name of the model.", - "default": "model", - "init_param": True, - }, - "model_nam_file": { - "name": "model_nam_file", - "_type": "pathlike", - "default": None, - "description": ( - "The relative path to the model name file from model working folder." - ), - "init_param": True, - }, - "version": { - "name": "version", - "_type": "str", - "description": "The version of modflow", - "default": "mf6", - "init_param": True, - }, - "exe_name": { - "name": "exe_name", - "_type": "str", - "description": "The executable name.", - "default": "mf6", - "init_param": True, - }, - "model_rel_path": { - "name": "model_rel_path", - "_type": "pathlike", - "description": "The model working folder path.", - "default": os.curdir, - "init_param": True, - }, - **vars_, - } - - refs = ctx.get("meta", dict()).get("refs", dict()) - if any(refs): - for key, ref in refs.items(): - key_var = vars_.get(key, None) - if not key_var: - continue - vars_[key] = { - **key_var, - "name": ref["val"], - "description": ref.get("description", None), - "ref": ref, - "construct_package": ref["abbr"], - "construct_data": ref["val"], - "parameter_name": ref["param"], - } - - ctx["vars"] = vars_ - return ctx - - -def _set_sim_vars(ctx: dict) -> dict: - """Modify variables for a simulation context.""" - vars_ = ctx["vars"].copy() - init_skip = [ - "tdis6", - "models", - "exchanges", - "mxiter", - "solutiongroup", - ] - for k in init_skip: - var = vars_.get(k, None) - if var: - var["init_param"] = False - var["init_skip"] = True - vars_[k] = var - vars_ = { - "sim_name": { - "name": "sim_name", - "_type": "str", - "default": "sim", - "description": "Name of the simulation.", - "init_param": True, - }, - "version": { - "name": "version", - "_type": "str", - "default": "mf6", - "init_param": True, - }, - "exe_name": { - "name": "exe_name", - "_type": "pathlike", - "default": "mf6", - "init_param": True, - }, - "sim_ws": { - "name": "sim_ws", - "_type": "pathlike", - "default": ".", - "init_param": True, - }, - "verbosity_level": { - "name": "verbosity_level", - "_type": "int", - "default": 1, - "init_param": True, - }, - "write_headers": { - "name": "write_headers", - "_type": "bool", - "default": True, - "init_param": True, - }, - "use_pandas": { - "name": "use_pandas", - "_type": "bool", - "default": True, - "init_param": True, - }, - "lazy_io": { - "name": "lazy_io", - "_type": "bool", - "default": False, - "init_param": True, - }, - **vars_, - } - - refs = ctx.get("meta", dict()).get("refs", dict()) - if any(refs) and ctx["name"] != (None, "nam"): - for key, ref in refs.items(): - key_var = vars_.get(key, None) - if not key_var: - continue - vars_[key] = { - **key_var, - "name": ref["param"], - "description": ref.get("description", None), - "ref": ref, - "init_param": True, - "init_skip": True, - "default": None, - } - - ctx["vars"] = vars_ - return ctx - - -def _set_parent(ctx: dict) -> dict: - vars_ = ctx["vars"] - parent = ctx["parent"] - ctx["vars"] = { - parent: { - "name": parent, - "_type": str(ctx["parent"]), - "description": f"Parent {parent} that this package is part of.", - "init_param": True, - }, - **vars_, - } - return ctx - - -def _map_context(o): - """ - Transform an input context as needed depending on its type. - - Notes - ----- - This includes adding extra variables for the `__init__` method; - This is done as a transform instead of with `set_pairs` so we - can control the order they appear in the method signature. - """ - - ctx = dict(o) - if ctx["name"].base == "MFSimulationBase": - ctx = _set_sim_vars(ctx) - elif ctx["name"].base == "MFModel": - ctx = _set_mdl_vars(ctx) - ctx = _set_parent(ctx) - elif ctx["name"].base == "MFPackage": - ctx = ( - _set_exg_vars(ctx) - if ctx["name"].l == "exg" - else _set_pkg_vars(ctx) - ) - ctx = _set_parent(ctx) - return ctx - - -def _class_attrs(ctx: dict) -> str: - """ - Get class attributes for the context. - """ +def _cls_attrs(ctx: dict) -> List[str]: ctx_name = ctx["name"] def _attr(var: dict) -> Optional[str]: @@ -413,32 +19,42 @@ def _attr(var: dict) -> Optional[str]: var_block = var.get("block", None) var_ref = var.get("meta", dict()).get("ref", None) - if var_kind is None or var_kind == "scalar": - return None - - if var_name in ["cvoptions", "output"]: - return None - if ( - ctx_name.l is not None and ctx_name.r == "nam" - ) and var_name != "packages": - return None - - if ctx_name.r == "dis" and var_name == "packagedata": + var_kind is None + or var_kind == "scalar" + or var_name in ["cvoptions", "output"] + or (ctx_name.r == "dis" and var_name == "packagedata") + or ( + var_name != "packages" + and (ctx_name.l is not None and ctx_name.r == "nam") + ) + ): return None - if var_kind in ["list", "record", "union"]: + if var_kind in ["list", "record", "union", "array"]: if not var_block: raise ValueError("Need block") - if var_ref: - # if the variable is a subpackage reference, use the original key - # (which has been replaced already with the referenced variable) - args = [ - f"'{ctx_name.r}'", - f"'{var_block}'", - f"'{var_ref['key']}'", - ] + if var_kind != "array": + if var_ref: + # if the variable is a subpackage reference, use the original key + # (which has been replaced already with the referenced variable) + args = [ + f"'{ctx_name.r}'", + f"'{var_block}'", + f"'{var_ref['key']}'", + ] + if ctx_name.l is not None and ctx_name.l not in [ + "sim", + "sln", + "utl", + "exg", + ]: + args.insert(0, f"'{ctx_name.l}6'") + return f"{var_ref['key']} = ListTemplateGenerator(({', '.join(args)}))" + + def _args(): + args = [f"'{ctx_name.r}'", f"'{var_block}'", f"'{var_name}'"] if ctx_name.l is not None and ctx_name.l not in [ "sim", "sln", @@ -446,143 +62,81 @@ def _attr(var: dict) -> Optional[str]: "exg", ]: args.insert(0, f"'{ctx_name.l}6'") - return f"{var_ref['key']} = ListTemplateGenerator(({', '.join(args)}))" - - args = [f"'{ctx_name.r}'", f"'{var_block}'", f"'{var_name}'"] - if ctx_name.l is not None and ctx_name.l not in [ - "sim", - "sln", - "utl", - "exg", - ]: - args.insert(0, f"'{ctx_name.l}6'") - return f"{var_name} = ListTemplateGenerator(({', '.join(args)}))" - - elif var_kind == "array": - if not var_block: - raise ValueError("Need block") - args = [f"'{ctx_name.r}'", f"'{var_block}'", f"'{var_name}'"] - if ctx_name.l is not None and ctx_name.l not in [ - "sim", - "sln", - "utl", - "exg", - ]: - args.insert(0, f"'{ctx_name.l}6'") - return f"{var_name} = ArrayTemplateGenerator(({', '.join(args)}))" + return args + + kind = var_kind if var_kind == "array" else "list" + return f"{var_name} = {kind.title()}TemplateGenerator(({', '.join(_args())}))" return None - attrs = [_attr(v) for v in ctx["vars"].values()] - return [a for a in attrs if a] - - -def _init_body(ctx: dict) -> str: - """ - Get the `__init__` method body for the context. - """ - - def _super() -> Optional[str]: - """ - Whether to pass the variable to `super().__init__()` - by name in the `__init__` method. - """ - - if ctx["base"] == "MFPackage": - args = [ - ctx["parent"] - if ctx.get("meta", dict()).get("ref", None) - else ctx["parent"], - f"'{ctx['name'].r}'", - "filename", - "pname", - "loading_package", - "**kwargs", - ] - elif ctx["base"] == "MFModel": - args = [ - ctx["parent"], - f"'{ctx['name'].l}6'", - "modelname=modelname", - "model_nam_file=model_nam_file", - "version=version", - "exe_name=exe_name", - "model_rel_path=model_rel_path", - "**kwargs", - ] - elif ctx["base"] == "MFSimulationBase": - args = [ - "sim_name=sim_name", - "version=version", - "exe_name=exe_name", - "sim_ws=sim_ws", - "verbosity_level=verbosity_level", - "write_headers=write_headers", - "lazy_io=lazy_io", - "use_pandas=use_pandas", + def _dfn() -> List[List[str]]: + dfn, meta = ctx["meta"]["dfn"] + + def _meta(): + exclude = ["subpackage", "parent_name_type"] + return [v for v in meta if not any(p in v for p in exclude)] + + def _dfn(): + def _var(var: dict) -> List[str]: + exclude = ["longname", "description"] + name = var["name"] + var_ = ctx["vars"].get(name, None) + keys = [ + "construct_package", + "construct_data", + "parameter_name", + ] + if var_ and keys[0] in var_: + for k in keys: + var[k] = var_[k] + return [ + " ".join([k, v]).strip() + for k, v in var.items() + if k not in exclude + ] + + return [_var(var) for var in dfn] + + return [["header"] + _meta()] + _dfn() + + attrs = list(filter(None, [_attr(v) for v in ctx["vars"].values()])) + + if ctx["base"] == "MFModel": + attrs.append(f"model_type = {ctx_name.l}") + elif ctx["base"] == "MFPackage": + attrs.extend( + [ + f"package_abbr = '{ctx_name.r}'" + if ctx_name.l == "exg" + else f"package_abbr = '{'' if ctx_name.l in ['sln', 'sim', 'exg', None] else ctx_name.l}{ctx_name.r}'", + f"_package_type = '{ctx_name.r}'", + f"dfn_file_name = '{ctx_name.l}-{ctx_name.r}.dfn'" + if ctx_name.l == "exg" + else f"dfn_file_name = '{ctx_name.l or 'sim'}-{ctx_name.r}.dfn'", + f"dfn = {pformat(_dfn(), indent=10)}", ] + ) - return f"super().__init__({', '.join(args)})" - - def _assign(var: dict) -> bool: - """ - Whether to assign arguments to self in the - `__init__` method. if this is false, assume - the template has conditionals for any more - involved initialization needs. - """ - return var["name"] in ["exgtype", "exgmnamea", "exgmnameb"] - - def _build(var: dict) -> bool: - """ - Whether to call `build_mfdata()` on the variable. - in the `__init__` method. - """ - if var.get("meta", dict()).get("ref", None) and ctx["name"] != ( - None, - "nam", - ): - return False - name = var["name"] - if name in [ - "simulation", - "model", - "package", - "parent_model", - "parent_package", - "loading_package", - "parent_model_or_package", - "exgtype", - "exgmnamea", - "exgmnameb", - "filename", - "pname", - "parent_file", - "modelname", - "model_nam_file", - "version", - "exe_name", - "model_rel_path", - "sim_name", - "sim_ws", - "verbosity_level", - "write_headers", - "use_pandas", - "lazy_io", - "export_netcdf", - "nc_filerecord", - "method", - "interpolation_method_single", - "sfac", - "output", - ]: - return False - return True - - def _body() -> Optional[str]: - if ctx["base"] in ["MFSimulationBase", "MFModel"]: - statements = [] - references = {} + return attrs + + +def _init_body(ctx: dict) -> List[str]: + def _statements() -> Optional[List[str]]: + base = ctx["base"] + if base == "MFSimulationBase": + + def _should_set(var: dict) -> bool: + return var["name"] not in [ + "tdis6", + "models", + "exchanges", + "mxiter", + "solutiongroup", + "hpc_data", + ] + + stmts = [] + refs = {} for var in ctx["vars"].values(): ref = var.get("meta", dict()).get("ref", None) if not var.get("kind", None): @@ -592,34 +146,84 @@ def _body() -> Optional[str]: if name in kwlist: name = f"{name}_" - if not var.get("init_skip", False): - statements.append( - f"self.name_file.{name}.set_data({name})" + if _should_set(var): + stmts.append(f"self.name_file.{name}.set_data({name})") + stmts.append(f"self.{name} = self.name_file.{name}") + if ref and ref["key"] not in refs: + refs[ref["key"]] = ref + stmts.append( + f"self.{ref['param']} = self._create_package('{ref['abbr']}', {ref['param']})" ) - statements.append(f"self.{name} = self.name_file.{name}") - if ref and ref["key"] not in references: - references[ref["key"]] = ref - statements.append( + elif base == "MFModel": + + def _should_set(var: dict) -> bool: + return var["name"] not in [ + "export_netcdf", + "nc_filerecord", + "packages", + ] + + stmts = [] + refs = {} + for var in ctx["vars"].values(): + ref = var.get("meta", dict()).get("ref", None) + if not var.get("kind", None): + continue + + name = var["name"] + if name in kwlist: + name = f"{name}_" + + if _should_set(var): + stmts.append(f"self.name_file.{name}.set_data({name})") + stmts.append(f"self.{name} = self.name_file.{name}") + if ref and ref["key"] not in refs: + refs[ref["key"]] = ref + stmts.append( f"self.{ref['param']} = self._create_package('{ref['abbr']}', {ref['param']})" ) - else: - statements = [] - references = {} + elif base == "MFPackage": + + def _should_build(var: dict) -> bool: + if var.get("meta", dict()).get("ref", None) and ctx[ + "name" + ] != ( + None, + "nam", + ): + return False + name = var["name"] + if name in [ + "simulation", + "model", + "package", + "parent_model", + "parent_package", + "parent_model_or_package", + "parent_file", + "modelname", + "model_nam_file", + "export_netcdf", + "nc_filerecord", + "method", + "interpolation_method_single", + "sfac", + "output", + ]: + return False + return True + + stmts = [] + refs = {} for var in ctx["vars"].values(): name = var["name"] ref = var.get("meta", dict()).get("ref", None) if name in kwlist: name = f"{name}_" - if _assign(var): - statements.append(f"self.{name} = {name}") - if name == "exgmnameb": - statements.append( - "simulation.register_exchange_file(self)" - ) - elif _build(var): + if _should_build(var): if ref and ctx["name"] == (None, "nam"): - statements.append( + stmts.append( f"self.{'_' if ref else ''}{ref['key']} = self.build_mfdata('{ref['key']}', None)" ) else: @@ -627,72 +231,156 @@ def _body() -> Optional[str]: _name = name[:-1] if name.endswith("_") else name if _name == "steady_state": _name = "steady-state" - statements.append( + stmts.append( f"self.{'_' if ref else ''}{name} = self.build_mfdata('{_name}', {name if var.get('init_param', True) else 'None'})" ) - if ( - ref - and ref["key"] not in references - and ctx["name"].r != "nam" - ): - references[ref["key"]] = ref - statements.append( + if ref and ref["key"] not in refs and ctx["name"].r != "nam": + refs[ref["key"]] = ref + stmts.append( f"self._{ref['key']} = self.build_mfdata('{ref['key']}', None)" ) - statements.append( + stmts.append( f"self._{ref['abbr']}_package = self.build_child_package('{ref['abbr']}', {ref['val']}, '{ref['param']}', self._{ref['key']})" ) - return ( - None - if not any(statements) - else "\n".join([" " + s for s in statements]) - ) + return stmts + + return list(filter(None, _statements())) + + +def _init_skip(ctx: dict) -> List[str]: + name = ctx["name"] + base = name.base + if base == "MFSimulationBase": + skip = [ + "tdis6", + "models", + "exchanges", + "mxiter", + "solutiongroup", + ] + refs = ctx.get("meta", dict()).get("refs", dict()) + return skip + elif base == "MFModel": + skip = ["packages", "export_netcdf", "nc_filerecord"] + refs = ctx.get("meta", dict()).get("refs", dict()) + if any(refs) and ctx["name"] != (None, "nam"): + for key in refs.keys(): + if ctx["vars"].get(key, None): + skip.append(key) + return skip + elif base == "MFPackage": + if name.r == "nam": + return ["export_netcdf", "nc_filerecord"] + elif name == ("utl", "ts"): + return ["method", "interpolation_method_single", "sfac"] + else: + return [] - sections = [_super(), _body()] - sections = [s for s in sections if s] - return "\n".join(sections) +def _is_context(o) -> bool: + d = dict(o) + return "name" in d and "base" in d -def _dfn(o) -> List[List[str]]: - """ - Get a list of the class' original definition attributes - as a partial, internal reproduction of the DFN contents. - Notes - ----- - Currently, generated classes have a `.dfn` property that - reproduces the corresponding DFN sans a few attributes. - Once `mfstructure.py` etc is reworked to introspect the - context classes instead of this property, it can go. - """ +def _parent(ctx: dict) -> str: + ref = ctx["meta"].get("ref", None) + if ref: + return ref["parent"] + name = ctx["name"] + ref = ctx["meta"].get("ref", None) + if name == ("sim", "nam"): + return None + elif name.l is None or name.r is None or name.l in ["sim", "exg", "sln"]: + return "simulation" + elif ref: + if name.l == "utl" and name.r == "hpc": + return "simulation" + return "package" + return "model" - ctx = dict(o) - dfn, meta = ctx["meta"]["dfn"] - - def _meta(): - exclude = ["subpackage", "parent_name_type"] - return [v for v in meta if not any(p in v for p in exclude)] - - def _dfn(): - def _var(var: dict) -> List[str]: - exclude = ["longname", "description"] - name = var["name"] - var_ = ctx["vars"].get(name, None) - keys = ["construct_package", "construct_data", "parameter_name"] - if var_ and keys[0] in var_: - for k in keys: - var[k] = var_[k] - return [ - " ".join([k, v]).strip() - for k, v in var.items() - if k not in exclude - ] - return [_var(var) for var in dfn] +def _replace_refs_exg(ctx: dict) -> dict: + refs = ctx.get("meta", dict()).get("refs", dict()) + if any(refs): + for key, ref in refs.items(): + key_var = ctx["vars"].get(key, None) + if not key_var: + continue + ctx["vars"][key] = { + **key_var, + "name": ref["val"], + "description": ref.get("description", None), + "ref": ref, + "default": None, + } + return ctx + + +def _replace_refs_pkg(ctx: dict) -> dict: + refs = ctx.get("meta", dict()).get("refs", dict()) + if any(refs): + for key, ref in refs.items(): + key_var = ctx["vars"].get(key, None) + if not key_var: + continue + ctx["vars"][key] = { + **key_var, + "name": ref["val"], + "description": ref.get("description", None), + "ref": ref, + "default": None, + } + return ctx + - return [["header"] + _meta()] + _dfn() +def _replace_refs_mdl(ctx: dict) -> dict: + refs = ctx.get("meta", dict()).get("refs", dict()) + if any(refs): + for key, ref in refs.items(): + key_var = ctx["vars"].get(key, None) + if not key_var: + continue + ctx["vars"][key] = { + **key_var, + "name": ref["val"], + "description": ref.get("description", None), + "ref": ref, + } + return ctx + + +def _replace_refs_sim(ctx: dict) -> dict: + refs = ctx.get("meta", dict()).get("refs", dict()) + if any(refs) and ctx["name"] != (None, "nam"): + for key, ref in refs.items(): + key_var = ctx["vars"].get(key, None) + if not key_var: + continue + ctx["vars"][key] = { + **key_var, + "name": ref["param"], + "description": ref.get("description", None), + "ref": ref, + "default": None, + } + return ctx + + +def _transform_context(o): + ctx = dict(o) + ctx_name = ctx["name"] + ctx_base = ctx_name.base + if ctx_base == "MFSimulationBase": + return _replace_refs_sim(ctx) + elif ctx_base == "MFModel": + return _replace_refs_mdl(ctx) + elif ctx_base == "MFPackage": + if ctx_name.l == "exg": + return _replace_refs_exg(ctx) + else: + return _replace_refs_pkg(ctx) SHIM = { @@ -702,22 +390,12 @@ def _var(var: dict) -> List[str]: ( _is_context, [ - ("dfn", _dfn), - ("class_attrs", _class_attrs), + ("cls_attrs", _cls_attrs), + ("init_skip", _init_skip), ("init_body", _init_body), - ], - ), - ( - _is_var, - [ - ("init_param", _is_init_param), - ("container_init_param", _is_container_init_param), + ("parent", _parent), ], ), ], - "transform": [(_is_context, _map_context)], + "transform": [(_is_context, _transform_context)], } -""" -Arguments for `renderable` as applied to `Context` -to support the current `flopy.mf6` input framework. -""" diff --git a/flopy/mf6/utils/codegen/templates/container.py.jinja b/flopy/mf6/utils/codegen/templates/container.py.jinja deleted file mode 100644 index ac4f7997d6..0000000000 --- a/flopy/mf6/utils/codegen/templates/container.py.jinja +++ /dev/null @@ -1,56 +0,0 @@ -class {{ name.title.title() }}Packages(MFChildPackages): - """ - {{ name.title.title() }}Packages is a container class for the Modflow{{ name.title.title() }} class. - - Methods - ------- - initialize - Initializes a new Modflow{{ name.title.title() }} package removing any sibling child - packages attached to the same parent package. See Modflow{{ name.title.title() }} init - documentation for definition of parameters. - append_package - Adds a new Modflow{{ name.title.title() }} package to the container. See Modflow{{ name.title.title() }} - init documentation for definition of parameters. - """ - - package_abbr = "{{ name.title.lower() }}packages" - - def initialize( - self, - {%- for n, var in vars.items() if var.container_init_param and not var.init_skip %} - {{ n }}{%- if var.default is defined %}={{ var.default }}{% endif -%}, - {%- endfor %} - filename=None, - pname=None, - ): - new_package = Modflow{{ name.title.title() }}( - self._cpparent, - {%- for n, var in vars.items() if var.container_init_param and not var.init_skip %} - {{ n }}={{ n }}, - {%- endfor %} - filename=filename, - pname=pname, - child_builder_call=True, - ) - self.init_package(new_package, filename) - -{% if name.r != "obs" %} - def append_package( - self, - {%- for n, var in vars.items() if var.container_init_param and not var.init_skip %} - {{ n }}{%- if var.default is defined %}={{ var.default }}{% endif -%}, - {%- endfor %} - filename=None, - pname=None, - ): - new_package = Modflow{{ name.title.title() }}( - self._cpparent, - {%- for n, var in vars.items() if var.container_init_param and not var.init_skip %} - {{ n }}={{ n }}, - {%- endfor %} - filename=filename, - pname=pname, - child_builder_call=True, - ) - self._append_package(new_package, filename) -{% endif %} diff --git a/flopy/mf6/utils/codegen/templates/context.py.jinja b/flopy/mf6/utils/codegen/templates/context.py.jinja deleted file mode 100644 index 34537ea427..0000000000 --- a/flopy/mf6/utils/codegen/templates/context.py.jinja +++ /dev/null @@ -1,8 +0,0 @@ -# autogenerated file, do not modify -{% if base == "MFSimulationBase" %} -{% include "simulation.py.jinja" %} -{% elif base == "MFModel" %} -{% include "model.py.jinja" %} -{% else %} -{% include "package.py.jinja" %} -{% endif %} diff --git a/flopy/mf6/utils/codegen/templates/docstring.jinja b/flopy/mf6/utils/codegen/templates/docstring.jinja deleted file mode 100644 index 488b567d45..0000000000 --- a/flopy/mf6/utils/codegen/templates/docstring.jinja +++ /dev/null @@ -1,12 +0,0 @@ -""" - {{ description }} - - Parameters - ---------- - {% include "docstring_params.jinja" %} - - Methods - ------- - {% include "docstring_methods.jinja" %} - """ - diff --git a/flopy/mf6/utils/codegen/templates/docstring_methods.jinja b/flopy/mf6/utils/codegen/templates/docstring_methods.jinja deleted file mode 100644 index 41daf5715d..0000000000 --- a/flopy/mf6/utils/codegen/templates/docstring_methods.jinja +++ /dev/null @@ -1,13 +0,0 @@ -{% if base == "MFSimulationBase" %} - load : (sim_name : str, version : string, - exe_name : str or PathLike, sim_ws : str or PathLike, strict : bool, - verbosity_level : int, load_only : list, verify_data : bool, - write_headers : bool, lazy_io : bool, use_pandas : bool, - ) : MFSimulation - a class method that loads a simulation from files -{% elif base == "MFModel" %} - load : (simulation : MFSimulationData, model_name : string, - namfile : string, version : string, exe_name : string, - model_ws : string, strict : boolean) : MFSimulation - a class method that loads a model from files -{% endif %} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/docstring_params.jinja b/flopy/mf6/utils/codegen/templates/docstring_params.jinja deleted file mode 100644 index 0b9d85d591..0000000000 --- a/flopy/mf6/utils/codegen/templates/docstring_params.jinja +++ /dev/null @@ -1,9 +0,0 @@ -{%- for v in vars.values() recursive %} - {% if loop.depth > 1 %}* {% endif %}{{ v.name }}{% if v.meta is defined and v.meta.type is defined %} : {{ v.meta.type }}{% endif %} -{%- if v.description is defined and v.description is not none %} -{{ v.description|wordwrap|indent(4 + (loop.depth * 4), first=True) }} -{%- endif %} -{%- if v.children is defined and v.children is not none -%} -{{ loop(v.children.values())|indent(4) }} -{%- endif %} -{% endfor -%} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/exchange.py.jinja b/flopy/mf6/utils/codegen/templates/exchange.py.jinja new file mode 100644 index 0000000000..b01b2e3a7d --- /dev/null +++ b/flopy/mf6/utils/codegen/templates/exchange.py.jinja @@ -0,0 +1,82 @@ +{% import 'macros.jinja' as macros %} + +from os import PathLike, curdir +from typing import Union + +from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator +from flopy.mf6.mfpackage import MFPackage + +class Modflow{{ name.title.title() }}(MFPackage): + """ + {{ description }} + + Parameters + ---------- + {{ macros.vars_docs(vars, start_indent=4) }} + """ + + {%- for attr in cls_attrs %} + {{ attr }} + {%- endfor %} + + def __init__( + self, + simulation, + loading_package=False, + exgtype="{{ name.r[:3].upper() }}6-{{ name.r[3:].upper() }}6", + exgmnamea=None, + exgmnameb=None, + {%- for n, var in vars.items() if n not in init_skip %} + {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, + {%- endfor %} + filename=None, + pname=None, + **kwargs, + ): + """ + {{ description }} + + simulation : MFSimulation + Simulation that this package is a part of. Package is automatically + added to simulation when it is initialized. + loading_package : bool + Do not set this parameter. It is intended for debugging and internal + processing purposes only. + exgtype : str + The exchange type (GWF-GWF or GWF-GWT). + exgmnamea : str + The name of the first model that is part of this exchange. + exgmnameb : str + The name of the second model that is part of this exchange. + gwfmodelname1 : str + Name of first GWF Model. In the simulation name file, the GWE6-GWE6 + entry contains names for GWE Models (exgmnamea and exgmnameb). The + GWE Model with the name exgmnamea must correspond to the GWF Model + with the name gwfmodelname1. + gwfmodelname2 : str + Name of second GWF Model. In the simulation name file, the GWE6-GWE6 + entry contains names for GWE Models (exgmnamea and exgmnameb). The + GWE Model with the name exgmnameb must correspond to the GWF Model + with the name gwfmodelname2. + {{ macros.vars_docs(vars, start_indent=8) }} + """ + + super().__init__( + {{ parent }}, + "{{ name.r }}", + filename, + pname, + loading_package, + **kwargs + ) + + self.exgtype = exgtype + self.exgmnamea = exgmnamea + self.exgmnameb = exgmnameb + simulation.register_exchange_file(self) + + {% for statement in init_body %} + {{ statement }} + {% endfor %} + + self._init_complete = True \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/init.py.jinja b/flopy/mf6/utils/codegen/templates/init.py.jinja deleted file mode 100644 index 78da10e5e6..0000000000 --- a/flopy/mf6/utils/codegen/templates/init.py.jinja +++ /dev/null @@ -1,9 +0,0 @@ -def __init__( - self, - {%- for var in vars.values() if var.init_param %} - {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, - {%- endfor %} - **kwargs, - ): - {{ init_body }} - \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/macros.jinja b/flopy/mf6/utils/codegen/templates/macros.jinja new file mode 100644 index 0000000000..8ebe95111f --- /dev/null +++ b/flopy/mf6/utils/codegen/templates/macros.jinja @@ -0,0 +1,11 @@ +{% macro vars_docs(vars, start_indent=0) %} +{%- for v in vars.values() recursive %} + {% if loop.depth > 1 %}* {% endif %}{% if v.name|last == "_" %}{{ v.name.replace("_", "\\\_") }}{% else %}{{ v.name }}{% endif %}{% if v.meta is defined and v.meta.type is defined %} : {{ v.meta.type }}{% endif %} + {%- if v.description is defined and v.description is not none %} +{{ v.description|wordwrap|indent(start_indent + (loop.depth * 4), first=true) }} + {%- endif %} + {%- if v.children is defined and v.children is not none -%} +{{ loop(v.children.values())|indent(start_indent, first=true) }} + {%- endif %} +{% endfor -%} +{% endmacro %} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/model.py.jinja b/flopy/mf6/utils/codegen/templates/model.py.jinja index ccb9b8ecf6..407e425cfa 100644 --- a/flopy/mf6/utils/codegen/templates/model.py.jinja +++ b/flopy/mf6/utils/codegen/templates/model.py.jinja @@ -1,3 +1,5 @@ +{% import 'macros.jinja' as macros %} + from os import PathLike, curdir from typing import Union @@ -6,11 +8,71 @@ from flopy.mf6.mfmodel import MFModel class Modflow{{ name.title.title() }}(MFModel): - {% include "docstring.jinja" %} + """ + {{ description }} + + Parameters + ---------- + {{ macros.vars_docs(vars, start_indent=4) }} + + Methods + ------- + load : (simulation : MFSimulationData, model_name : string, + namfile : string, version : string, exe_name : string, + model_ws : string, strict : boolean) : MFSimulation + a class method that loads a model from files + """ model_type = "{{ name.title }}" - {% include "init.py.jinja" %} + def __init__( + self, + simulation, + modelname="model", + model_nam_file=None, + version="mf6", + exe_name="mf6", + model_rel_path=".", + {%- for n, var in vars.items() if n not in init_skip %} + {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, + {%- endfor %} + **kwargs, + ): + """ + {{ description }} + + Parameters + ---------- + modelname : string + name of the model + model_nam_file : string + relative path to the model name file from model working folder + version : string + version of modflow + exe_name : string + model executable name + model_ws : string + model working folder path + sim : MFSimulation + Simulation that this model is a part of. Model is automatically + added to simulation when it is initialized. + {{ macros.vars_docs(vars, start_indent=8) }} + """ + + super().__init__( + simulation, + model_type="{{ name.title }}6", + modelname=modelname, + model_nam_file=model_nam_file, + version=version, + exe_name=exe_name, + model_rel_path=model_rel_path, + **kwargs, + ) + + {% for statement in init_body %} + {{ statement }} + {% endfor %} @classmethod def load( diff --git a/flopy/mf6/utils/codegen/templates/package.py.jinja b/flopy/mf6/utils/codegen/templates/package.py.jinja index 50da2d813c..6cd7e142e9 100644 --- a/flopy/mf6/utils/codegen/templates/package.py.jinja +++ b/flopy/mf6/utils/codegen/templates/package.py.jinja @@ -1,24 +1,128 @@ +{% import 'macros.jinja' as macros %} + from os import PathLike, curdir from typing import Union from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator from flopy.mf6.mfpackage import MFPackage, MFChildPackages + class Modflow{{ name.title.title() }}(MFPackage): - {% include "docstring.jinja" %} + """ + {{ description }} + + Parameters + ---------- + {{ macros.vars_docs(vars, start_indent=4) }} + """ - {%- for item in class_attrs %} - {{ item }} + {%- for attr in cls_attrs %} + {{ attr }} {%- endfor %} - package_abbr = "{{ '' if name.l in ["sln", "sim", "exg", none] else name.l }}{{ name.r }}" - _package_type = "{{ name.r }}" - dfn_file_name = "{{ name.l | default('sim', true) }}-{{ name.r }}.dfn" - dfn = {{ dfn|pprint|indent(10) }} - {% include "init.py.jinja" %} + def __init__( + self, + {{ parent }}, + loading_package=False, + {%- for n, var in vars.items() if n not in init_skip %} + {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, + {%- endfor %} + filename=None, + pname=None, + **kwargs, + ): + """ + {{ description }} + + Parameters + ---------- + {{ parent }} : {{ name.parent }} + Parent_package that this package is a part of. Package is automatically + added to parent_package when it is initialized. + loading_package : bool + Do not set this parameter. It is intended for debugging and internal + processing purposes only. + {{ macros.vars_docs(vars, start_indent=8) }} + filename : str + File name for this package. + pname : str + Package name for this package. + parent_file : MFPackage + Parent package file that references this package. Only needed for + utility packages (mfutl*). For example, mfutllaktab package must have + a mfgwflak package parent_file. + """ + + super().__init__( + {{ parent }}, + "{{ name.r }}", + filename, + pname, + loading_package, + **kwargs + ) + + {% for statement in init_body %} + {{ statement }} + {%- endfor %} + self._init_complete = True -{# TODO: cleaner way to filter out hpc subpkg? #} {% if "ref" in meta and name.r != "hpc" %} -{% include "container.py.jinja" %} +class {{ name.title.title() }}Packages(MFChildPackages): + """ + {{ name.title.title() }}Packages is a container class for the Modflow{{ name.title.title() }} class. + + Methods + ------- + initialize + Initializes a new Modflow{{ name.title.title() }} package removing any sibling child + packages attached to the same parent package. See Modflow{{ name.title.title() }} init + documentation for definition of parameters. + append_package + Adds a new Modflow{{ name.title.title() }} package to the container. See Modflow{{ name.title.title() }} + init documentation for definition of parameters. + """ + + package_abbr = "{{ name.title.lower() }}packages" + + def initialize( + self, + {%- for n, var in vars.items() if n not in init_skip %} + {{ n }}{%- if var.default is defined %}={{ var.default }}{% endif -%}, + {%- endfor %} + filename=None, + pname=None, + ): + new_package = Modflow{{ name.title.title() }}( + self._cpparent, + {%- for n, var in vars.items() if n not in init_skip %} + {{ n }}={{ n }}, + {%- endfor %} + filename=filename, + pname=pname, + child_builder_call=True, + ) + self.init_package(new_package, filename) + + {% if name.r != "obs" %} + def append_package( + self, + {%- for n, var in vars.items() if n not in init_skip %} + {{ n }}{%- if var.default is defined %}={{ var.default }}{% endif -%}, + {%- endfor %} + filename=None, + pname=None, + ): + new_package = Modflow{{ name.title.title() }}( + self._cpparent, + {%- for n, var in vars.items() if n not in init_skip %} + {{ n }}={{ n }}, + {%- endfor %} + filename=filename, + pname=pname, + child_builder_call=True, + ) + self._append_package(new_package, filename) + {% endif %} {% endif %} diff --git a/flopy/mf6/utils/codegen/templates/simulation.py.jinja b/flopy/mf6/utils/codegen/templates/simulation.py.jinja index 84437a0c43..aa6762bce7 100644 --- a/flopy/mf6/utils/codegen/templates/simulation.py.jinja +++ b/flopy/mf6/utils/codegen/templates/simulation.py.jinja @@ -1,3 +1,5 @@ +{% import 'macros.jinja' as macros %} + from os import PathLike, curdir from typing import Union @@ -5,9 +7,75 @@ from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenera from flopy.mf6.mfsimbase import MFSimulationBase class MF{{ name.title.title() }}(MFSimulationBase): - {% include "docstring.jinja" %} + """ + {{ description }} + + Parameters + ---------- + {{ macros.vars_docs(vars, start_indent=4) }} + + Methods + ------- + load : (sim_name : str, version : string, + exe_name : str or PathLike, sim_ws : str or PathLike, strict : bool, + verbosity_level : int, load_only : list, verify_data : bool, + write_headers : bool, lazy_io : bool, use_pandas : bool, + ) : MFSimulation + a class method that loads a simulation from files + """ + + def __init__( + self, + sim_name: str = "sim", + version: str = "mf6", + exe_name: Union[str, PathLike] = "mf6", + sim_ws: Union[str, PathLike] = curdir, + verbosity_level: int = 1, + write_headers: bool = True, + use_pandas: bool = True, + lazy_io: bool = False, + {%- for n, var in vars.items() if n not in init_skip %} + {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, + {%- endfor %} + ): + """ + {{ description }} + + Parameters + ---------- + sim_name + The name of the simulation + version + The simulation version + exe_name + The executable name + sim_ws + The simulation workspace + verbosity_level + The verbosity level + write_headers + Whether to write + use_pandas + Whether to use pandas + lazy_io + Whether to use lazy IO + {{ macros.vars_docs(vars, start_indent=8) }} + """ + + super().__init__( + sim_name=sim_name, + version=version, + exe_name=exe_name, + sim_ws=sim_ws, + verbosity_level=verbosity_level, + write_headers=write_headers, + lazy_io=lazy_io, + use_pandas=use_pandas + ) - {% include "init.py.jinja" %} + {% for statement in init_body %} + {{ statement }} + {% endfor %} @classmethod def load( diff --git a/flopy/mf6/utils/codegen/utils.py b/flopy/mf6/utils/codegen/utils.py new file mode 100644 index 0000000000..556f96b399 --- /dev/null +++ b/flopy/mf6/utils/codegen/utils.py @@ -0,0 +1,34 @@ +from ast import literal_eval +from enum import Enum +from typing import Any + + +def try_get_enum_value(v: Any) -> Any: + """ + Get the enum's value if the object is an instance + of an enumeration, otherwise return it unaltered. + """ + return v.value if isinstance(v, Enum) else v + + +def try_literal_eval(value: str) -> Any: + """ + Try to parse a string as a literal. If this fails, + return the value unaltered. + """ + try: + return literal_eval(value) + except (SyntaxError, ValueError): + return value + + +def try_parse_bool(value: Any) -> Any: + """ + Try to parse a boolean from a string as represented + in a DFN file, otherwise return the value unaltered. + """ + if isinstance(value, str): + value = value.lower() + if value in ["true", "false"]: + return value == "true" + return value diff --git a/flopy/mf6/utils/createpackages.py b/flopy/mf6/utils/createpackages.py index 6d76aa8cf6..9d565e6feb 100644 --- a/flopy/mf6/utils/createpackages.py +++ b/flopy/mf6/utils/createpackages.py @@ -83,12 +83,10 @@ from pathlib import Path -from flopy.mf6.utils.codegen.make import make_all +from flopy.mf6.utils.codegen import make_all _MF6_PATH = Path(__file__).parents[1] -_DFN_PATH = _MF6_PATH / "data" / "dfn" -_TGT_PATH = _MF6_PATH / "modflow" if __name__ == "__main__": - make_all(_DFN_PATH, _TGT_PATH) + make_all(dfndir=_MF6_PATH / "data" / "dfn", outdir=_MF6_PATH / "modflow") From 70012161c7e52636f04df8fe776dfefdec3d982c Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Mon, 28 Oct 2024 18:17:00 -0400 Subject: [PATCH 32/75] codacy fixes --- flopy/mf6/utils/codegen/dfn.py | 38 +++++++++++++++++----------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index dd4b545611..5db2703bac 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -85,11 +85,11 @@ def parse(cls, v: str) -> "Dfn.Name": def __init__( self, - vars: Optional[Vars] = None, + data: Optional[Vars] = None, name: Optional[Name] = None, meta: Optional[Dict[str, Any]] = None, ): - self.data = OMD(vars) + self.data = OMD(data) self.name = name self.meta = meta @@ -107,7 +107,7 @@ def _load(f, common: Optional[dict] = None) -> Tuple[OMD, List[str]]: """ var = dict() - vars = list() + flat = list() meta = list() common = common or dict() @@ -140,7 +140,7 @@ def _load(f, common: Optional[dict] = None) -> Tuple[OMD, List[str]]: # block of attributes if not any(line): if any(var): - vars.append((var["name"], var)) + flat.append((var["name"], var)) var = dict() continue @@ -176,9 +176,9 @@ def _load(f, common: Optional[dict] = None) -> Tuple[OMD, List[str]]: # add the final parameter if any(var): - vars.append((var["name"], var)) + flat.append((var["name"], var)) - return OMD(vars), meta + return OMD(flat), meta @classmethod def load( @@ -192,7 +192,7 @@ def load( refs = refs or dict() referenced = dict() - vars, meta = Dfn._load(f, **kwargs) + flat, meta = Dfn._load(f, **kwargs) def _map(spec: Dict[str, Any]) -> Var: """ @@ -236,12 +236,12 @@ def _map(spec: Dict[str, Any]) -> Var: def _fields(record_name: str) -> Vars: """Recursively load/convert a record's fields.""" - record = next(iter(vars.getlist(record_name)), None) + record = next(iter(flat.getlist(record_name)), None) assert record names = _type.split()[1:] fields = { v["name"]: _map(v) - for v in vars.values(multi=True) + for v in flat.values(multi=True) if v["name"] in names and not v["type"].startswith("record") and v.get("in_record", False) @@ -290,7 +290,7 @@ def _fields(record_name: str) -> Vars: # fields directly inside the recarray (implicit). list # data for unions/keystrings necessarily comes nested. - is_explicit_record = n_names == 1 and vars[names[0]][ + is_explicit_record = n_names == 1 and flat[names[0]][ "type" ].startswith("record") @@ -299,13 +299,13 @@ def _is_implicit_scalar_record(): # only scalar fields types = [ v["type"] - for v in vars.values(multi=True) + for v in flat.values(multi=True) if v["name"] in names and v.get("in_record", False) ] return all(t in _SCALARS for t in types) if is_explicit_record: - record = next(iter(vars.getlist(names[0])), None) + record = next(iter(flat.getlist(names[0])), None) children = {names[0]: _map(record)} kind = Var.Kind.List elif _is_implicit_scalar_record(): @@ -327,7 +327,7 @@ def _is_implicit_scalar_record(): # implicit complex record (i.e. some fields are records or unions) fields = { v["name"]: _map(v) - for v in vars.values(multi=True) + for v in flat.values(multi=True) if v["name"] in names and v.get("in_record", False) } first = list(fields.values())[0] @@ -353,7 +353,7 @@ def _is_implicit_scalar_record(): names = _type.split()[1:] children = { v["name"]: _map(v) - for v in vars.values(multi=True) + for v in flat.values(multi=True) if v["name"] in names and v.get("in_record", False) } kind = Var.Kind.Union @@ -402,24 +402,24 @@ def _is_implicit_scalar_record(): # pass the original DFN representation as # metadata so the shim can use it for now - _vars = list(vars.values(multi=True)) + _vars = list(flat.values(multi=True)) # convert input variable specs to # structured form, descending into # composites recursively as needed - vars = { + flat = { var["name"]: _map(var) - for var in vars.values(multi=True) + for var in flat.values(multi=True) if not var.get("in_record", False) } # reset the var name. we may have altered # it when converting the variable e.g. to # avoid collision with a reserved keyword - vars = {v.name: v for v in vars.values()} + flat = {v.name: v for v in flat.values()} return cls( - vars, + flat, name, { "dfn": (_vars, meta), From 80cfa624dfff5d71d2d9ee0a8408abec69b9e7b6 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 29 Oct 2024 08:17:48 -0400 Subject: [PATCH 33/75] boltons and jinja as dev dependencies, cleanup --- README.md | 1 - flopy/mf6/utils/codegen/dfn.py | 2 +- pyproject.toml | 8 +++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 23c5fe3b2d..e22f58134a 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,6 @@ Installation FloPy requires **Python** 3.9+ with: ``` -boltons >=1.0 numpy >=1.20.3 matplotlib >=1.4.0 pandas >=2.0.0 diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index 5db2703bac..b6fcef441b 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -51,9 +51,9 @@ class Kind(Enum): name: str kind: Kind block: Optional[str] = None - description: Optional[str] = None default: Optional[Any] = None children: Optional[Vars] = None + description: Optional[str] = None meta: Optional[Dict[str, Any]] = None diff --git a/pyproject.toml b/pyproject.toml index f4d62f0160..9946027346 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,8 +29,6 @@ classifiers = [ ] requires-python = ">=3.9" dependencies = [ - "boltons>=1.0", - "Jinja2>=3.0", "numpy>=1.20.3", "matplotlib >=1.4.0", "pandas >=2.0.0", @@ -38,7 +36,11 @@ dependencies = [ dynamic = ["version", "readme"] [project.optional-dependencies] -dev = ["flopy[lint,test,optional,doc]", "tach"] +dev = ["flopy[gen,lint,test,optional,doc]", "tach"] +gen = [ + "boltons>=1.0", + "Jinja2>=3.0" +] lint = ["cffconvert", "codespell[toml] >=2.2.2", "ruff"] test = [ "flopy[lint]", From 54caf59289b778ce58039aa73730008e2599a2f0 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 29 Oct 2024 08:19:36 -0400 Subject: [PATCH 34/75] gen -> codegen dep group --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9946027346..e48e592a1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ dynamic = ["version", "readme"] [project.optional-dependencies] dev = ["flopy[gen,lint,test,optional,doc]", "tach"] -gen = [ +codegen = [ "boltons>=1.0", "Jinja2>=3.0" ] From 23f57af7e7a7ef68a836a274e8c4beb14dec793f Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 29 Oct 2024 08:22:25 -0400 Subject: [PATCH 35/75] mention deps in generate_classes docs --- .docs/md/generate_classes.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.docs/md/generate_classes.md b/.docs/md/generate_classes.md index 05c2300d40..3e91086fa4 100644 --- a/.docs/md/generate_classes.md +++ b/.docs/md/generate_classes.md @@ -10,7 +10,11 @@ MODFLOW 6 input continues to evolve as new models, packages, and options are developed, updated, and supported. All MODFLOW 6 input is described by DFN (definition) files, which are simple text files that describe the blocks and keywords in each input file. These definition files are used to build the input and output guide for MODFLOW 6. These definition files are also used to automatically generate FloPy classes for creating, reading and writing MODFLOW 6 models, packages, and options. FloPy and MODFLOW 6 are kept in sync by these DFN (definition) files, and therefore, it may be necessary for a user to update FloPy using a custom set of definition files, or a set of definition files from a previous release. -The FloPy classes for MODFLOW 6 are largely generated by a utility which converts DFN files in a modflow6 repository on GitHub or on the local machine into Python source files in your local FloPy install. For instance (output much abbreviated): +The FloPy classes for MODFLOW 6 are largely generated by a utility which converts DFN files in a modflow6 repository on GitHub or on the local machine into Python source files in your local FloPy install. + +**Note**: to use this functionality, the `codegen` optional dependency group must be installed. + +For instance (output much abbreviated): ```bash $ python -m flopy.mf6.utils.generate_classes From 318d7d1b3af1c78ab724d5a466c4f394e1f88238 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 29 Oct 2024 08:30:58 -0400 Subject: [PATCH 36/75] handle missing jinja --- .docs/md/generate_classes.md | 2 -- .github/workflows/commit.yml | 6 ++++-- .github/workflows/examples.yml | 4 +++- .github/workflows/mf6.yml | 4 ++-- flopy/mf6/utils/codegen/__init__.py | 26 +++++++++++++++++++------- 5 files changed, 28 insertions(+), 14 deletions(-) diff --git a/.docs/md/generate_classes.md b/.docs/md/generate_classes.md index 3e91086fa4..e7b3025771 100644 --- a/.docs/md/generate_classes.md +++ b/.docs/md/generate_classes.md @@ -14,8 +14,6 @@ The FloPy classes for MODFLOW 6 are largely generated by a utility which convert **Note**: to use this functionality, the `codegen` optional dependency group must be installed. -For instance (output much abbreviated): - ```bash $ python -m flopy.mf6.utils.generate_classes diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index 68f310fa10..70302fae1d 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -104,7 +104,7 @@ jobs: run: | pip install --upgrade pip pip install . - pip install ".[test, optional]" + pip install ".[dev]" - name: Install Modflow executables uses: modflowpy/install-modflow-action@v1 @@ -159,7 +159,9 @@ jobs: powershell - name: Install FloPy - run: pip install . + run: | + pip install . + pip install ".[codegen]" - name: Install Modflow-related executables uses: modflowpy/install-modflow-action@v1 diff --git a/.github/workflows/examples.yml b/.github/workflows/examples.yml index 04ccd791ff..37ae546305 100644 --- a/.github/workflows/examples.yml +++ b/.github/workflows/examples.yml @@ -34,7 +34,9 @@ jobs: powershell - name: Install FloPy - run: pip install . + run: | + pip install . + pip install ".[codegen]" - name: OpenGL workaround on Linux if: runner.os == 'Linux' diff --git a/.github/workflows/mf6.yml b/.github/workflows/mf6.yml index 50a3ccce12..8234d0861c 100644 --- a/.github/workflows/mf6.yml +++ b/.github/workflows/mf6.yml @@ -37,7 +37,7 @@ jobs: pip install https://github.com/modflowpy/pymake/zipball/master pip install https://github.com/Deltares/xmipy/zipball/develop pip install https://github.com/MODFLOW-USGS/modflowapi/zipball/develop - pip install .[test,optional] + pip install .[codegen,test,optional] pip install meson ninja - name: Setup GNU Fortran @@ -120,7 +120,7 @@ jobs: pip install https://github.com/modflowpy/pymake/zipball/master pip install https://github.com/Deltares/xmipy/zipball/develop pip install https://github.com/MODFLOW-USGS/modflowapi/zipball/develop - pip install .[test,optional] + pip install .[codegen,test,optional] pip install meson ninja pip install -r modflow6-examples/etc/requirements.pip.txt diff --git a/flopy/mf6/utils/codegen/__init__.py b/flopy/mf6/utils/codegen/__init__.py index 68466b91ed..78ba16e234 100644 --- a/flopy/mf6/utils/codegen/__init__.py +++ b/flopy/mf6/utils/codegen/__init__.py @@ -1,19 +1,25 @@ from pathlib import Path +from warnings import warn -from jinja2 import Environment, PackageLoader - -from flopy.mf6.utils.codegen.context import Context -from flopy.mf6.utils.codegen.dfn import Dfn, Dfns, Ref, Refs +from flopy.utils import import_optional_dependency __all__ = ["make_targets", "make_all"] -_TEMPLATE_LOADER = PackageLoader("flopy", "mf6/utils/codegen/templates/") -_TEMPLATE_ENV = Environment(loader=_TEMPLATE_LOADER) +jinja = import_optional_dependency("jinja2", errors="ignore") +if jinja: + _TEMPLATES_PATH = "mf6/utils/codegen/templates/" + _TEMPLATE_LOADER = jinja.PackageLoader("flopy", _TEMPLATES_PATH) + _TEMPLATE_ENV = jinja.Environment(loader=_TEMPLATE_LOADER) -def make_targets(dfn: Dfn, outdir: Path, verbose: bool = False): +def make_targets(dfn, outdir: Path, verbose: bool = False): """Generate Python source file(s) from the given input definition.""" + from flopy.mf6.utils.codegen.context import Context + + if not jinja: + raise RuntimeError("Jinja2 not installed, can't make targets") + for context in Context.from_dfn(dfn): name = context.name target = outdir / name.target @@ -27,6 +33,12 @@ def make_targets(dfn: Dfn, outdir: Path, verbose: bool = False): def make_all(dfndir: Path, outdir: Path, verbose: bool = False): """Generate Python source files from the DFN files in the given location.""" + from flopy.mf6.utils.codegen.context import Context + from flopy.mf6.utils.codegen.dfn import Dfn, Dfns, Ref, Refs + + if not jinja: + raise RuntimeError("Jinja2 not installed, can't make targets") + # find definition files paths = [ p for p in dfndir.glob("*.dfn") if p.stem not in ["common", "flopy"] From 973c3e5c874b7f54e18a794f2719695c9077cbec Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 29 Oct 2024 08:54:31 -0400 Subject: [PATCH 37/75] fixes --- .github/workflows/commit.yml | 2 +- autotest/test_generate_classes.py | 2 +- pyproject.toml | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/commit.yml b/.github/workflows/commit.yml index 70302fae1d..5de30462a2 100644 --- a/.github/workflows/commit.yml +++ b/.github/workflows/commit.yml @@ -161,7 +161,7 @@ jobs: - name: Install FloPy run: | pip install . - pip install ".[codegen]" + pip install ".[dev]" - name: Install Modflow-related executables uses: modflowpy/install-modflow-action@v1 diff --git a/autotest/test_generate_classes.py b/autotest/test_generate_classes.py index c1de3bc706..f3a93ab4df 100644 --- a/autotest/test_generate_classes.py +++ b/autotest/test_generate_classes.py @@ -94,7 +94,7 @@ def test_generate_classes_from_github_refs( print(f"Using temp venv at {venv} to test class generation from {ref}") # install flopy and dependencies - deps = [str(project_root_path), "modflow-devtools"] + deps = [str(project_root_path), "modflow-devtools", "Jinja2", "boltons"] for dep in deps: out, err, ret = run_cmd(str(pip), "install", dep, verbose=True) assert not ret, out + err diff --git a/pyproject.toml b/pyproject.toml index e48e592a1c..1c023e39d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,7 +39,8 @@ dynamic = ["version", "readme"] dev = ["flopy[gen,lint,test,optional,doc]", "tach"] codegen = [ "boltons>=1.0", - "Jinja2>=3.0" + "Jinja2>=3.0", + "modflow-devtools" ] lint = ["cffconvert", "codespell[toml] >=2.2.2", "ruff"] test = [ From 346507660d6f8604220b08d951cbabcbcdd20953 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 30 Oct 2024 10:22:04 -0400 Subject: [PATCH 38/75] fixes after review --- autotest/test_codegen.py | 10 ---------- flopy/mf6/utils/codegen/context.py | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/autotest/test_codegen.py b/autotest/test_codegen.py index 9dbd771f9b..2a0f98e8ff 100644 --- a/autotest/test_codegen.py +++ b/autotest/test_codegen.py @@ -25,16 +25,6 @@ def test_dfn_load(dfn_name): common, _ = Dfn._load(common_file) dfn = Dfn.load(dfn_file, name=name, common=common) - if name in [ - ("sln", "ems"), - ("exg", "gwfprt"), - ("exg", "gwfgwe"), - ("exg", "gwfgwt"), - ]: - assert not any(dfn) - else: - assert any(dfn) - @pytest.mark.parametrize("dfn_name", DFN_NAMES) def test_make_targets(dfn_name, function_tmpdir): diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py index 91fd165570..d874edf408 100644 --- a/flopy/mf6/utils/codegen/context.py +++ b/flopy/mf6/utils/codegen/context.py @@ -151,7 +151,7 @@ def from_dfn(dfn: Dfn) -> List["Context.Name"]: Context.Name(*dfn.name), # nam pkg Context.Name(dfn.name.l, None), # model ] - elif (dfn.name.l, dfn.name.r) in [ + elif dfn.name in [ ("gwf", "mvr"), ("gwf", "gnc"), ("gwt", "mvt"), From ad965a9157072c68e968350e5643b03083e50a4a Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 30 Oct 2024 11:10:23 -0400 Subject: [PATCH 39/75] docstring indentation fix --- flopy/mf6/utils/codegen/templates/macros.jinja | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flopy/mf6/utils/codegen/templates/macros.jinja b/flopy/mf6/utils/codegen/templates/macros.jinja index 8ebe95111f..4d5c6310f9 100644 --- a/flopy/mf6/utils/codegen/templates/macros.jinja +++ b/flopy/mf6/utils/codegen/templates/macros.jinja @@ -1,6 +1,6 @@ {% macro vars_docs(vars, start_indent=0) %} {%- for v in vars.values() recursive %} - {% if loop.depth > 1 %}* {% endif %}{% if v.name|last == "_" %}{{ v.name.replace("_", "\\\_") }}{% else %}{{ v.name }}{% endif %}{% if v.meta is defined and v.meta.type is defined %} : {{ v.meta.type }}{% endif %} +{{ ""|indent(start_indent, first=true) }}{% if loop.depth > 1 %}* {% endif %}{% if v.name|last == "_" %}{{ v.name.replace("_", "\\\_") }}{% else %}{{ v.name }}{% endif %}{% if v.meta is defined and v.meta.type is defined %} : {{ v.meta.type }}{% endif %} {%- if v.description is defined and v.description is not none %} {{ v.description|wordwrap|indent(start_indent + (loop.depth * 4), first=true) }} {%- endif %} From 01d1c3529d65ff2856e137e072b6f1bdac291824 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 30 Oct 2024 11:25:27 -0400 Subject: [PATCH 40/75] formatting fixes --- autotest/test_codegen.py | 2 +- flopy/mf6/utils/codegen/templates/exchange.py.jinja | 13 ++++++++++--- flopy/mf6/utils/codegen/templates/model.py.jinja | 8 +++++++- flopy/mf6/utils/codegen/templates/package.py.jinja | 4 +++- .../mf6/utils/codegen/templates/simulation.py.jinja | 10 +++++++++- 5 files changed, 30 insertions(+), 7 deletions(-) diff --git a/autotest/test_codegen.py b/autotest/test_codegen.py index 2a0f98e8ff..7d5b58df7e 100644 --- a/autotest/test_codegen.py +++ b/autotest/test_codegen.py @@ -23,7 +23,7 @@ def test_dfn_load(dfn_name): ): name = Dfn.Name.parse(dfn_name) common, _ = Dfn._load(common_file) - dfn = Dfn.load(dfn_file, name=name, common=common) + Dfn.load(dfn_file, name=name, common=common) @pytest.mark.parametrize("dfn_name", DFN_NAMES) diff --git a/flopy/mf6/utils/codegen/templates/exchange.py.jinja b/flopy/mf6/utils/codegen/templates/exchange.py.jinja index b01b2e3a7d..23158aa452 100644 --- a/flopy/mf6/utils/codegen/templates/exchange.py.jinja +++ b/flopy/mf6/utils/codegen/templates/exchange.py.jinja @@ -15,7 +15,7 @@ class Modflow{{ name.title.title() }}(MFPackage): {{ macros.vars_docs(vars, start_indent=4) }} """ - {%- for attr in cls_attrs %} + {% for attr in cls_attrs %} {{ attr }} {%- endfor %} @@ -39,25 +39,32 @@ class Modflow{{ name.title.title() }}(MFPackage): simulation : MFSimulation Simulation that this package is a part of. Package is automatically added to simulation when it is initialized. + loading_package : bool Do not set this parameter. It is intended for debugging and internal processing purposes only. + exgtype : str The exchange type (GWF-GWF or GWF-GWT). + exgmnamea : str The name of the first model that is part of this exchange. + exgmnameb : str The name of the second model that is part of this exchange. + gwfmodelname1 : str Name of first GWF Model. In the simulation name file, the GWE6-GWE6 entry contains names for GWE Models (exgmnamea and exgmnameb). The GWE Model with the name exgmnamea must correspond to the GWF Model with the name gwfmodelname1. + gwfmodelname2 : str Name of second GWF Model. In the simulation name file, the GWE6-GWE6 entry contains names for GWE Models (exgmnamea and exgmnameb). The GWE Model with the name exgmnameb must correspond to the GWF Model - with the name gwfmodelname2. + with the name gwfmodelname2. + {{ macros.vars_docs(vars, start_indent=8) }} """ @@ -77,6 +84,6 @@ class Modflow{{ name.title.title() }}(MFPackage): {% for statement in init_body %} {{ statement }} - {% endfor %} + {%- endfor %} self._init_complete = True \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/model.py.jinja b/flopy/mf6/utils/codegen/templates/model.py.jinja index 407e425cfa..ed32c37497 100644 --- a/flopy/mf6/utils/codegen/templates/model.py.jinja +++ b/flopy/mf6/utils/codegen/templates/model.py.jinja @@ -45,17 +45,23 @@ class Modflow{{ name.title.title() }}(MFModel): ---------- modelname : string name of the model + model_nam_file : string relative path to the model name file from model working folder + version : string version of modflow + exe_name : string model executable name + model_ws : string model working folder path + sim : MFSimulation Simulation that this model is a part of. Model is automatically added to simulation when it is initialized. + {{ macros.vars_docs(vars, start_indent=8) }} """ @@ -72,7 +78,7 @@ class Modflow{{ name.title.title() }}(MFModel): {% for statement in init_body %} {{ statement }} - {% endfor %} + {%- endfor %} @classmethod def load( diff --git a/flopy/mf6/utils/codegen/templates/package.py.jinja b/flopy/mf6/utils/codegen/templates/package.py.jinja index 6cd7e142e9..401b40cde4 100644 --- a/flopy/mf6/utils/codegen/templates/package.py.jinja +++ b/flopy/mf6/utils/codegen/templates/package.py.jinja @@ -16,7 +16,7 @@ class Modflow{{ name.title.title() }}(MFPackage): {{ macros.vars_docs(vars, start_indent=4) }} """ - {%- for attr in cls_attrs %} + {% for attr in cls_attrs %} {{ attr }} {%- endfor %} @@ -79,9 +79,11 @@ class {{ name.title.title() }}Packages(MFChildPackages): Initializes a new Modflow{{ name.title.title() }} package removing any sibling child packages attached to the same parent package. See Modflow{{ name.title.title() }} init documentation for definition of parameters. + append_package Adds a new Modflow{{ name.title.title() }} package to the container. See Modflow{{ name.title.title() }} init documentation for definition of parameters. + """ package_abbr = "{{ name.title.lower() }}packages" diff --git a/flopy/mf6/utils/codegen/templates/simulation.py.jinja b/flopy/mf6/utils/codegen/templates/simulation.py.jinja index aa6762bce7..aab32746b1 100644 --- a/flopy/mf6/utils/codegen/templates/simulation.py.jinja +++ b/flopy/mf6/utils/codegen/templates/simulation.py.jinja @@ -45,20 +45,28 @@ class MF{{ name.title.title() }}(MFSimulationBase): ---------- sim_name The name of the simulation + version The simulation version + exe_name The executable name + sim_ws The simulation workspace + verbosity_level The verbosity level + write_headers Whether to write + use_pandas Whether to use pandas + lazy_io Whether to use lazy IO + {{ macros.vars_docs(vars, start_indent=8) }} """ @@ -75,7 +83,7 @@ class MF{{ name.title.title() }}(MFSimulationBase): {% for statement in init_body %} {{ statement }} - {% endfor %} + {%- endfor %} @classmethod def load( From d0d9703c7213d641116c71da7f5855b3f1fbd7df Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 30 Oct 2024 11:47:25 -0400 Subject: [PATCH 41/75] fmt --- flopy/mf6/utils/codegen/templates/package.py.jinja | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/flopy/mf6/utils/codegen/templates/package.py.jinja b/flopy/mf6/utils/codegen/templates/package.py.jinja index 401b40cde4..857a0d4162 100644 --- a/flopy/mf6/utils/codegen/templates/package.py.jinja +++ b/flopy/mf6/utils/codegen/templates/package.py.jinja @@ -39,14 +39,19 @@ class Modflow{{ name.title.title() }}(MFPackage): {{ parent }} : {{ name.parent }} Parent_package that this package is a part of. Package is automatically added to parent_package when it is initialized. + loading_package : bool Do not set this parameter. It is intended for debugging and internal processing purposes only. + {{ macros.vars_docs(vars, start_indent=8) }} + filename : str File name for this package. + pname : str Package name for this package. + parent_file : MFPackage Parent package file that references this package. Only needed for utility packages (mfutl*). For example, mfutllaktab package must have From 5a8c83c03f7e381d27f86ab7cfcf860a8eea733a Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 31 Oct 2024 09:59:07 -0400 Subject: [PATCH 42/75] install dev dep group in mf6.yml --- .github/workflows/mf6.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mf6.yml b/.github/workflows/mf6.yml index 8234d0861c..08133b4de0 100644 --- a/.github/workflows/mf6.yml +++ b/.github/workflows/mf6.yml @@ -37,7 +37,7 @@ jobs: pip install https://github.com/modflowpy/pymake/zipball/master pip install https://github.com/Deltares/xmipy/zipball/develop pip install https://github.com/MODFLOW-USGS/modflowapi/zipball/develop - pip install .[codegen,test,optional] + pip install .[dev] pip install meson ninja - name: Setup GNU Fortran @@ -120,7 +120,7 @@ jobs: pip install https://github.com/modflowpy/pymake/zipball/master pip install https://github.com/Deltares/xmipy/zipball/develop pip install https://github.com/MODFLOW-USGS/modflowapi/zipball/develop - pip install .[codegen,test,optional] + pip install .[dev] pip install meson ninja pip install -r modflow6-examples/etc/requirements.pip.txt From 4418d1900ef963a8c7caf6a466e9c66b59a33156 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 31 Oct 2024 10:08:11 -0400 Subject: [PATCH 43/75] remove description from context, just use name.description --- flopy/mf6/utils/codegen/context.py | 2 -- flopy/mf6/utils/codegen/templates/exchange.py.jinja | 4 ++-- flopy/mf6/utils/codegen/templates/macros.jinja | 8 ++++---- flopy/mf6/utils/codegen/templates/model.py.jinja | 4 ++-- flopy/mf6/utils/codegen/templates/package.py.jinja | 4 ++-- flopy/mf6/utils/codegen/templates/simulation.py.jinja | 4 ++-- 6 files changed, 12 insertions(+), 14 deletions(-) diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py index d874edf408..92d2f17778 100644 --- a/flopy/mf6/utils/codegen/context.py +++ b/flopy/mf6/utils/codegen/context.py @@ -165,7 +165,6 @@ def from_dfn(dfn: Dfn) -> List["Context.Name"]: name: Name vars: Vars base: Optional[type] = None - description: Optional[str] = None meta: Optional[Dict[str, Any]] = None @classmethod @@ -184,6 +183,5 @@ def from_dfn(cls, dfn: Dfn) -> Iterator["Context"]: name=name, vars=dfn.data, base=name.base, - description=name.description, meta=meta, ) diff --git a/flopy/mf6/utils/codegen/templates/exchange.py.jinja b/flopy/mf6/utils/codegen/templates/exchange.py.jinja index 23158aa452..a5d4a1e98f 100644 --- a/flopy/mf6/utils/codegen/templates/exchange.py.jinja +++ b/flopy/mf6/utils/codegen/templates/exchange.py.jinja @@ -8,7 +8,7 @@ from flopy.mf6.mfpackage import MFPackage class Modflow{{ name.title.title() }}(MFPackage): """ - {{ description }} + {{ name.description }} Parameters ---------- @@ -34,7 +34,7 @@ class Modflow{{ name.title.title() }}(MFPackage): **kwargs, ): """ - {{ description }} + {{ name.description }} simulation : MFSimulation Simulation that this package is a part of. Package is automatically diff --git a/flopy/mf6/utils/codegen/templates/macros.jinja b/flopy/mf6/utils/codegen/templates/macros.jinja index 4d5c6310f9..be5b1ab1a9 100644 --- a/flopy/mf6/utils/codegen/templates/macros.jinja +++ b/flopy/mf6/utils/codegen/templates/macros.jinja @@ -1,11 +1,11 @@ {% macro vars_docs(vars, start_indent=0) %} {%- for v in vars.values() recursive %} {{ ""|indent(start_indent, first=true) }}{% if loop.depth > 1 %}* {% endif %}{% if v.name|last == "_" %}{{ v.name.replace("_", "\\\_") }}{% else %}{{ v.name }}{% endif %}{% if v.meta is defined and v.meta.type is defined %} : {{ v.meta.type }}{% endif %} - {%- if v.description is defined and v.description is not none %} +{%- if v.description is defined and v.description is not none %} {{ v.description|wordwrap|indent(start_indent + (loop.depth * 4), first=true) }} - {%- endif %} - {%- if v.children is defined and v.children is not none -%} +{%- endif %} +{%- if v.children is defined and v.children is not none -%} {{ loop(v.children.values())|indent(start_indent, first=true) }} - {%- endif %} +{%- endif %} {% endfor -%} {% endmacro %} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/model.py.jinja b/flopy/mf6/utils/codegen/templates/model.py.jinja index ed32c37497..a922bad9e0 100644 --- a/flopy/mf6/utils/codegen/templates/model.py.jinja +++ b/flopy/mf6/utils/codegen/templates/model.py.jinja @@ -9,7 +9,7 @@ from flopy.mf6.mfmodel import MFModel class Modflow{{ name.title.title() }}(MFModel): """ - {{ description }} + {{ name.description }} Parameters ---------- @@ -39,7 +39,7 @@ class Modflow{{ name.title.title() }}(MFModel): **kwargs, ): """ - {{ description }} + {{ name.description }} Parameters ---------- diff --git a/flopy/mf6/utils/codegen/templates/package.py.jinja b/flopy/mf6/utils/codegen/templates/package.py.jinja index 857a0d4162..576ba79a78 100644 --- a/flopy/mf6/utils/codegen/templates/package.py.jinja +++ b/flopy/mf6/utils/codegen/templates/package.py.jinja @@ -9,7 +9,7 @@ from flopy.mf6.mfpackage import MFPackage, MFChildPackages class Modflow{{ name.title.title() }}(MFPackage): """ - {{ description }} + {{ name.description }} Parameters ---------- @@ -32,7 +32,7 @@ class Modflow{{ name.title.title() }}(MFPackage): **kwargs, ): """ - {{ description }} + {{ name.description }} Parameters ---------- diff --git a/flopy/mf6/utils/codegen/templates/simulation.py.jinja b/flopy/mf6/utils/codegen/templates/simulation.py.jinja index aab32746b1..39f807574e 100644 --- a/flopy/mf6/utils/codegen/templates/simulation.py.jinja +++ b/flopy/mf6/utils/codegen/templates/simulation.py.jinja @@ -8,7 +8,7 @@ from flopy.mf6.mfsimbase import MFSimulationBase class MF{{ name.title.title() }}(MFSimulationBase): """ - {{ description }} + {{ name.description }} Parameters ---------- @@ -39,7 +39,7 @@ class MF{{ name.title.title() }}(MFSimulationBase): {%- endfor %} ): """ - {{ description }} + {{ name.description }} Parameters ---------- From 0344c6232cf39516c480e6bb6c74866d2cef3e7a Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 31 Oct 2024 10:25:40 -0400 Subject: [PATCH 44/75] replace latex quotes in descriptions --- flopy/mf6/utils/codegen/dfn.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index b6fcef441b..7c0c84d26b 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -155,7 +155,11 @@ def _load(f, common: Optional[dict] = None) -> Tuple[OMD, List[str]]: # remove backslashes, TODO: generate/insert citations. descr = var.get("description", None) if descr: - descr = descr.replace("\\", "") + descr = ( + descr.replace("\\", "") + .replace("``", "'") + .replace("''", "'") + ) _, replace, tail = descr.strip().partition("REPLACE") if replace: key, _, subs = tail.strip().partition(" ") From 3d7580263c1f9b825aceabe406997591c1bef8eb Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 31 Oct 2024 10:30:24 -0400 Subject: [PATCH 45/75] consolidate ref param replacement in shim, remove children, fix description --- flopy/mf6/utils/codegen/shim.py | 75 ++++++--------------------------- 1 file changed, 14 insertions(+), 61 deletions(-) diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index 2b02ecd020..20d9c0fcb5 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -301,7 +301,7 @@ def _parent(ctx: dict) -> str: return "model" -def _replace_refs_exg(ctx: dict) -> dict: +def _replace_refs(ctx: dict, name_param: str = "val") -> dict: refs = ctx.get("meta", dict()).get("refs", dict()) if any(refs): for key, ref in refs.items(): @@ -310,60 +310,18 @@ def _replace_refs_exg(ctx: dict) -> dict: continue ctx["vars"][key] = { **key_var, - "name": ref["val"], - "description": ref.get("description", None), - "ref": ref, - "default": None, - } - return ctx - - -def _replace_refs_pkg(ctx: dict) -> dict: - refs = ctx.get("meta", dict()).get("refs", dict()) - if any(refs): - for key, ref in refs.items(): - key_var = ctx["vars"].get(key, None) - if not key_var: - continue - ctx["vars"][key] = { - **key_var, - "name": ref["val"], - "description": ref.get("description", None), - "ref": ref, - "default": None, - } - return ctx - - -def _replace_refs_mdl(ctx: dict) -> dict: - refs = ctx.get("meta", dict()).get("refs", dict()) - if any(refs): - for key, ref in refs.items(): - key_var = ctx["vars"].get(key, None) - if not key_var: - continue - ctx["vars"][key] = { - **key_var, - "name": ref["val"], - "description": ref.get("description", None), - "ref": ref, - } - return ctx - - -def _replace_refs_sim(ctx: dict) -> dict: - refs = ctx.get("meta", dict()).get("refs", dict()) - if any(refs) and ctx["name"] != (None, "nam"): - for key, ref in refs.items(): - key_var = ctx["vars"].get(key, None) - if not key_var: - continue - ctx["vars"][key] = { - **key_var, - "name": ref["param"], - "description": ref.get("description", None), + "name": ref[name_param], + "description": ( + f"* Contains data for the {ref['abbr']} package. Data can be " + f"stored in a dictionary containing data for the {ref['abbr']} " + "package with variable names as keys and package data as " + f"values. Data just for the {ref['val']} variable is also " + f"acceptable. See {ref['abbr']} package documentation for more " + "information" + ), "ref": ref, "default": None, + "children": None, } return ctx @@ -373,14 +331,9 @@ def _transform_context(o): ctx_name = ctx["name"] ctx_base = ctx_name.base if ctx_base == "MFSimulationBase": - return _replace_refs_sim(ctx) - elif ctx_base == "MFModel": - return _replace_refs_mdl(ctx) - elif ctx_base == "MFPackage": - if ctx_name.l == "exg": - return _replace_refs_exg(ctx) - else: - return _replace_refs_pkg(ctx) + return _replace_refs(ctx, name_param="param") + else: + return _replace_refs(ctx) SHIM = { From 9e721c3cbc2d33283fcd1f804564e0a0e535cbcb Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 31 Oct 2024 10:34:08 -0400 Subject: [PATCH 46/75] trim shim --- flopy/mf6/utils/codegen/shim.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index 20d9c0fcb5..7b64306fdc 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -337,7 +337,7 @@ def _transform_context(o): SHIM = { - "keep_none": ["default", "block", "metadata"], + "keep_none": ["default"], "quote_str": ["default"], "set_pairs": [ ( From 6b0577915ea9a80edc7fdde505c061a279fc6e3d Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 31 Oct 2024 11:11:53 -0400 Subject: [PATCH 47/75] add Dfn.load_all() --- flopy/mf6/utils/codegen/__init__.py | 34 +--- flopy/mf6/utils/codegen/context.py | 8 +- flopy/mf6/utils/codegen/dfn.py | 233 +++++++++++++++------------- flopy/mf6/utils/codegen/shim.py | 8 +- 4 files changed, 134 insertions(+), 149 deletions(-) diff --git a/flopy/mf6/utils/codegen/__init__.py b/flopy/mf6/utils/codegen/__init__.py index 78ba16e234..80d00964d7 100644 --- a/flopy/mf6/utils/codegen/__init__.py +++ b/flopy/mf6/utils/codegen/__init__.py @@ -34,41 +34,13 @@ def make_all(dfndir: Path, outdir: Path, verbose: bool = False): """Generate Python source files from the DFN files in the given location.""" from flopy.mf6.utils.codegen.context import Context - from flopy.mf6.utils.codegen.dfn import Dfn, Dfns, Ref, Refs + from flopy.mf6.utils.codegen.dfn import Dfn if not jinja: raise RuntimeError("Jinja2 not installed, can't make targets") - # find definition files - paths = [ - p for p in dfndir.glob("*.dfn") if p.stem not in ["common", "flopy"] - ] - - # try to load common variables - common_path = dfndir / "common.dfn" - if not common_path.is_file: - common = None - else: - with open(common_path, "r") as f: - common, _ = Dfn._load(f) - - # load subpackage references first - refs: Refs = {} - for path in paths: - name = Dfn.Name(*path.stem.split("-")) - with open(path) as f: - dfn = Dfn.load(f, name=name, common=common) - ref = Ref.from_dfn(dfn) - if ref: - refs[ref.key] = ref - - # load all the input definitions - dfns: Dfns = {} - for path in paths: - name = Dfn.Name(*path.stem.split("-")) - with open(path) as f: - dfn = Dfn.load(f, name=name, refs=refs, common=common) - dfns[name] = dfn + # load dfns + dfns = Dfn.load_all(dfndir) # make target files for dfn in dfns.values(): diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py index 92d2f17778..58d05b7f56 100644 --- a/flopy/mf6/utils/codegen/context.py +++ b/flopy/mf6/utils/codegen/context.py @@ -164,7 +164,6 @@ def from_dfn(dfn: Dfn) -> List["Context.Name"]: name: Name vars: Vars - base: Optional[type] = None meta: Optional[Dict[str, Any]] = None @classmethod @@ -179,9 +178,4 @@ def from_dfn(cls, dfn: Dfn) -> Iterator["Context"]: if ref: meta["ref"] = ref for name in Context.Name.from_dfn(dfn): - yield Context( - name=name, - vars=dfn.data, - base=name.base, - meta=meta, - ) + yield Context(name=name, vars=dfn.data, meta=meta) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index 7c0c84d26b..ef98ab9983 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -28,8 +28,8 @@ Vars = Dict[str, "Var"] -Dfns = Dict[str, "Dfn"] Refs = Dict[str, "Ref"] +Dfns = Dict[str, "Dfn"] @dataclass @@ -57,6 +57,96 @@ class Kind(Enum): meta: Optional[Dict[str, Any]] = None +@dataclass +class Ref: + """ + A foreign-key-like reference between a file input variable + and another input definition. This allows an input context + to refer to another input context, by including a filepath + variable whose name acts as a foreign key for a different + input context. The referring context's `__init__` method + is modified such that the variable named `val` replaces + the `key` variable. + + This class is used to represent subpackage references. + """ + + key: str + val: str + abbr: str + param: str + parent: str + description: Optional[str] + + @classmethod + def from_dfn(cls, dfn: "Dfn") -> Optional["Ref"]: + """ + Try to load a reference from the definition. + Returns `None` if the definition cannot be + referenced by other contexts. + """ + + # TODO: all this won't be necessary once we + # structure DFN format; we can then support + # subpackage references directly instead of + # by making assumptions about `dfn.meta` + + if not dfn.meta or "dfn" not in dfn.meta: + return None + + _, meta = dfn.meta["dfn"] + + lines = { + "subpkg": next( + iter( + m + for m in meta + if isinstance(m, str) and m.startswith("subpac") + ), + None, + ), + "parent": next( + iter( + m + for m in meta + if isinstance(m, str) and m.startswith("parent") + ), + None, + ), + } + + def _subpkg(): + line = lines["subpkg"] + _, key, abbr, param, val = line.split() + matches = [v for v in dfn.values() if v.name == val] + if not any(matches): + descr = None + else: + if len(matches) > 1: + warn(f"Multiple matches for referenced variable {val}") + match = matches[0] + descr = match.description + + return { + "key": key, + "val": val, + "abbr": abbr, + "param": param, + "description": descr, + } + + def _parent(): + line = lines["parent"] + split = line.split() + return split[1] + + return ( + cls(**_subpkg(), parent=_parent()) + if all(v for v in lines.values()) + else None + ) + + class Dfn(UserDict): """ MODFLOW 6 input definition. An input definition @@ -431,109 +521,38 @@ def _is_implicit_scalar_record(): }, ) - -@dataclass -class Ref: - """ - A foreign-key-like reference between a file input variable - and another input definition. This allows an input context - to refer to another input context, by including a filepath - variable whose name acts as a foreign key for a different - input context. The referring context's `__init__` method - is modified such that the variable named `val` replaces - the `key` variable. - - Notes - ----- - This class is used to represent subpackage references. - - Parameters - ---------- - key : str - The name of the foreign key file input variable. - val : str - The name of the data variable in the referenced context. - abbr : str - An abbreviation of the referenced context's name. - param : str - The referenced parameter name. - parents : List[str] - The referenced context's supported parents. - description : Optional[str] - The reference's description. - """ - - key: str - val: str - abbr: str - param: str - parent: str - description: Optional[str] - - @classmethod - def from_dfn(cls, dfn: Dfn) -> Optional["Ref"]: - """ - Try to load a reference from the definition. - Returns `None` if the definition cannot be - referenced by other contexts. - """ - - # TODO: all this won't be necessary once we - # structure DFN format; we can then support - # subpackage references directly instead of - # by making assumptions about `dfn.meta` - - if not dfn.meta or "dfn" not in dfn.meta: - return None - - _, meta = dfn.meta["dfn"] - - lines = { - "subpkg": next( - iter( - m - for m in meta - if isinstance(m, str) and m.startswith("subpac") - ), - None, - ), - "parent": next( - iter( - m - for m in meta - if isinstance(m, str) and m.startswith("parent") - ), - None, - ), - } - - def _subpkg(): - line = lines["subpkg"] - _, key, abbr, param, val = line.split() - matches = [v for v in dfn.values() if v.name == val] - if not any(matches): - descr = None - else: - if len(matches) > 1: - warn(f"Multiple matches for referenced variable {val}") - match = matches[0] - descr = match.description - - return { - "key": key, - "val": val, - "abbr": abbr, - "param": param, - "description": descr, - } - - def _parent(): - line = lines["parent"] - split = line.split() - return split[1] - - return ( - cls(**_subpkg(), parent=_parent()) - if all(v for v in lines.values()) - else None - ) + @staticmethod + def load_all(dfndir: PathLike) -> Dict[str, "Dfn"]: + """Load all input definitions from the given directory.""" + # find definition files + paths = [ + p for p in dfndir.glob("*.dfn") if p.stem not in ["common", "flopy"] + ] + + # try to load common variables + common_path = dfndir / "common.dfn" + if not common_path.is_file: + common = None + else: + with open(common_path, "r") as f: + common, _ = Dfn._load(f) + + # load subpackage references first + refs: Refs = {} + for path in paths: + name = Dfn.Name(*path.stem.split("-")) + with open(path) as f: + dfn = Dfn.load(f, name=name, common=common) + ref = Ref.from_dfn(dfn) + if ref: + refs[ref.key] = ref + + # load all the input definitions + dfns: Dfns = {} + for path in paths: + name = Dfn.Name(*path.stem.split("-")) + with open(path) as f: + dfn = Dfn.load(f, name=name, refs=refs, common=common) + dfns[name] = dfn + + return dfns diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index 7b64306fdc..ed724e4b74 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -101,9 +101,9 @@ def _var(var: dict) -> List[str]: attrs = list(filter(None, [_attr(v) for v in ctx["vars"].values()])) - if ctx["base"] == "MFModel": + if ctx_name.base == "MFModel": attrs.append(f"model_type = {ctx_name.l}") - elif ctx["base"] == "MFPackage": + elif ctx_name.base == "MFPackage": attrs.extend( [ f"package_abbr = '{ctx_name.r}'" @@ -122,7 +122,7 @@ def _var(var: dict) -> List[str]: def _init_body(ctx: dict) -> List[str]: def _statements() -> Optional[List[str]]: - base = ctx["base"] + base = ctx["name"].base if base == "MFSimulationBase": def _should_set(var: dict) -> bool: @@ -281,7 +281,7 @@ def _init_skip(ctx: dict) -> List[str]: def _is_context(o) -> bool: d = dict(o) - return "name" in d and "base" in d + return "name" in d and "vars" in d def _parent(ctx: dict) -> str: From ddd65633bf9fd4efcbb068b9345094a9d6e0a00f Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 31 Oct 2024 11:17:04 -0400 Subject: [PATCH 48/75] dataclasses -> typeddicts --- flopy/mf6/utils/codegen/context.py | 1 + flopy/mf6/utils/codegen/dfn.py | 39 +++++++++++++++++------------- flopy/mf6/utils/codegen/shim.py | 4 ++- 3 files changed, 26 insertions(+), 18 deletions(-) diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py index 58d05b7f56..34c3201237 100644 --- a/flopy/mf6/utils/codegen/context.py +++ b/flopy/mf6/utils/codegen/context.py @@ -156,6 +156,7 @@ def from_dfn(dfn: Dfn) -> List["Context.Name"]: ("gwf", "gnc"), ("gwt", "mvt"), ]: + # TODO: remove special cases, deduplicate mfmvr.py/mfgwfmvr.py etc return [ Context.Name(*dfn.name), Context.Name(None, dfn.name.r), diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index ef98ab9983..fc606fa4cb 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -11,6 +11,7 @@ NamedTuple, Optional, Tuple, + TypedDict, Union, ) from warnings import warn @@ -32,8 +33,7 @@ Dfns = Dict[str, "Dfn"] -@dataclass -class Var: +class Var(TypedDict): """MODFLOW 6 input variable specification.""" class Kind(Enum): @@ -57,8 +57,7 @@ class Kind(Enum): meta: Optional[Dict[str, Any]] = None -@dataclass -class Ref: +class Ref(TypedDict): """ A foreign-key-like reference between a file input variable and another input definition. This allows an input context @@ -118,14 +117,14 @@ def from_dfn(cls, dfn: "Dfn") -> Optional["Ref"]: def _subpkg(): line = lines["subpkg"] _, key, abbr, param, val = line.split() - matches = [v for v in dfn.values() if v.name == val] + matches = [v for v in dfn.values() if v["name"] == val] if not any(matches): descr = None else: if len(matches) > 1: warn(f"Multiple matches for referenced variable {val}") match = matches[0] - descr = match.description + descr = match["description"] return { "key": key, @@ -356,7 +355,7 @@ def _fields(record_name: str) -> Vars: # set the type n = list(fields.keys())[0] path_field = fields[n] - path_field._type = Union[str, PathLike] + path_field["kind"] = Var.Kind.Scalar fields[n] = path_field # if tagged, remove the leading keyword @@ -412,7 +411,7 @@ def _is_implicit_scalar_record(): children=fields, description=description, meta={ - "type": f"[{', '.join([f.meta['type'] for f in fields.values()])}]" + "type": f"[{', '.join([f['meta']['type'] for f in fields.values()])}]" }, ) } @@ -426,21 +425,23 @@ def _is_implicit_scalar_record(): } first = list(fields.values())[0] single = len(fields) == 1 - name_ = first.name if single else _name + name_ = first["name"] if single else _name children = { name_: Var( name=name_, kind=Var.Kind.Record, block=block, - children=first.children if single else fields, + children=first["children"] if single else fields, description=description, meta={ - "type": f"[{', '.join([v.meta['type'] for v in fields.values()])}]" + "type": f"[{', '.join([v['meta']['type'] for v in fields.values()])}]" }, ) } kind = Var.Kind.List - type_ = f"[{', '.join([v.name for v in children.values()])}]" + type_ = ( + f"[{', '.join([v['name'] for v in children.values()])}]" + ) # union (product), children are choices elif _type.startswith("keystring"): @@ -451,13 +452,15 @@ def _is_implicit_scalar_record(): if v["name"] in names and v.get("in_record", False) } kind = Var.Kind.Union - type_ = f"[{', '.join([v.name for v in children.values()])}]" + type_ = ( + f"[{', '.join([v['name'] for v in children.values()])}]" + ) # record (sum), children are fields elif _type.startswith("record"): children = _fields(_name) kind = Var.Kind.Record - type_ = f"[{', '.join([v.meta['type'] for v in children.values()])}]" + type_ = f"[{', '.join([v['meta']['type'] for v in children.values()])}]" # at this point, if it has a shape, it's an array elif shape is not None: @@ -510,7 +513,7 @@ def _is_implicit_scalar_record(): # reset the var name. we may have altered # it when converting the variable e.g. to # avoid collision with a reserved keyword - flat = {v.name: v for v in flat.values()} + flat = {v["name"]: v for v in flat.values()} return cls( flat, @@ -526,7 +529,9 @@ def load_all(dfndir: PathLike) -> Dict[str, "Dfn"]: """Load all input definitions from the given directory.""" # find definition files paths = [ - p for p in dfndir.glob("*.dfn") if p.stem not in ["common", "flopy"] + p + for p in dfndir.glob("*.dfn") + if p.stem not in ["common", "flopy"] ] # try to load common variables @@ -545,7 +550,7 @@ def load_all(dfndir: PathLike) -> Dict[str, "Dfn"]: dfn = Dfn.load(f, name=name, common=common) ref = Ref.from_dfn(dfn) if ref: - refs[ref.key] = ref + refs[ref["key"]] = ref # load all the input definitions dfns: Dfns = {} diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index ed724e4b74..e1c3013a7d 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -9,13 +9,15 @@ from pprint import pformat from typing import List, Optional +from flopy.mf6.utils.codegen.utils import try_get_enum_value + def _cls_attrs(ctx: dict) -> List[str]: ctx_name = ctx["name"] def _attr(var: dict) -> Optional[str]: var_name = var["name"] - var_kind = var.get("kind", None) + var_kind = try_get_enum_value(var.get("kind", None)) var_block = var.get("block", None) var_ref = var.get("meta", dict()).get("ref", None) From 48e83dfd1178e49af0b281e619d20aaf0f4f12e2 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 31 Oct 2024 11:30:25 -0400 Subject: [PATCH 49/75] remove unneeded imports --- flopy/mf6/utils/codegen/dfn.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index fc606fa4cb..bb9ecaa9b8 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -1,6 +1,5 @@ from ast import literal_eval from collections import UserDict -from dataclasses import dataclass from enum import Enum from keyword import kwlist from os import PathLike @@ -12,7 +11,6 @@ Optional, Tuple, TypedDict, - Union, ) from warnings import warn From 3b0440deb8067ecd31eac2b869db0dce603a229c Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 31 Oct 2024 12:48:49 -0400 Subject: [PATCH 50/75] add params macro --- .../utils/codegen/templates/exchange.py.jinja | 5 ++--- .../mf6/utils/codegen/templates/macros.jinja | 6 ++++++ .../utils/codegen/templates/model.py.jinja | 5 ++--- .../utils/codegen/templates/package.py.jinja | 21 ++++++------------- .../codegen/templates/simulation.py.jinja | 5 ++--- 5 files changed, 18 insertions(+), 24 deletions(-) diff --git a/flopy/mf6/utils/codegen/templates/exchange.py.jinja b/flopy/mf6/utils/codegen/templates/exchange.py.jinja index a5d4a1e98f..4b76026d52 100644 --- a/flopy/mf6/utils/codegen/templates/exchange.py.jinja +++ b/flopy/mf6/utils/codegen/templates/exchange.py.jinja @@ -1,4 +1,5 @@ {% import 'macros.jinja' as macros %} +# autogenerated file, do not modify from os import PathLike, curdir from typing import Union @@ -26,9 +27,7 @@ class Modflow{{ name.title.title() }}(MFPackage): exgtype="{{ name.r[:3].upper() }}6-{{ name.r[3:].upper() }}6", exgmnamea=None, exgmnameb=None, - {%- for n, var in vars.items() if n not in init_skip %} - {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, - {%- endfor %} + {{ macros.vars_params(vars, init_skip, start_indent=8) }} filename=None, pname=None, **kwargs, diff --git a/flopy/mf6/utils/codegen/templates/macros.jinja b/flopy/mf6/utils/codegen/templates/macros.jinja index be5b1ab1a9..fe5ceb3fb6 100644 --- a/flopy/mf6/utils/codegen/templates/macros.jinja +++ b/flopy/mf6/utils/codegen/templates/macros.jinja @@ -1,3 +1,9 @@ +{% macro vars_params(vars, init_skip=none, start_indent=0) %} +{%- for n, var in vars.items() if n not in init_skip %} +{{ ""|indent(start_indent, first=true) }}{{ var.name }}{%- if var.default is defined %}={{ var.default }}{% endif -%}, +{%- endfor %} +{% endmacro %} + {% macro vars_docs(vars, start_indent=0) %} {%- for v in vars.values() recursive %} {{ ""|indent(start_indent, first=true) }}{% if loop.depth > 1 %}* {% endif %}{% if v.name|last == "_" %}{{ v.name.replace("_", "\\\_") }}{% else %}{{ v.name }}{% endif %}{% if v.meta is defined and v.meta.type is defined %} : {{ v.meta.type }}{% endif %} diff --git a/flopy/mf6/utils/codegen/templates/model.py.jinja b/flopy/mf6/utils/codegen/templates/model.py.jinja index a922bad9e0..3754124af0 100644 --- a/flopy/mf6/utils/codegen/templates/model.py.jinja +++ b/flopy/mf6/utils/codegen/templates/model.py.jinja @@ -1,4 +1,5 @@ {% import 'macros.jinja' as macros %} +# autogenerated file, do not modify from os import PathLike, curdir from typing import Union @@ -33,9 +34,7 @@ class Modflow{{ name.title.title() }}(MFModel): version="mf6", exe_name="mf6", model_rel_path=".", - {%- for n, var in vars.items() if n not in init_skip %} - {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, - {%- endfor %} + {{ macros.vars_params(vars, init_skip, start_indent=8) }} **kwargs, ): """ diff --git a/flopy/mf6/utils/codegen/templates/package.py.jinja b/flopy/mf6/utils/codegen/templates/package.py.jinja index 576ba79a78..e3d2375a2f 100644 --- a/flopy/mf6/utils/codegen/templates/package.py.jinja +++ b/flopy/mf6/utils/codegen/templates/package.py.jinja @@ -1,4 +1,5 @@ {% import 'macros.jinja' as macros %} +# autogenerated file, do not modify from os import PathLike, curdir from typing import Union @@ -24,9 +25,7 @@ class Modflow{{ name.title.title() }}(MFPackage): self, {{ parent }}, loading_package=False, - {%- for n, var in vars.items() if n not in init_skip %} - {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, - {%- endfor %} + {{ macros.vars_params(vars, init_skip, start_indent=8) }} filename=None, pname=None, **kwargs, @@ -95,17 +94,13 @@ class {{ name.title.title() }}Packages(MFChildPackages): def initialize( self, - {%- for n, var in vars.items() if n not in init_skip %} - {{ n }}{%- if var.default is defined %}={{ var.default }}{% endif -%}, - {%- endfor %} + {{ macros.vars_params(vars, init_skip, start_indent=8) }} filename=None, pname=None, ): new_package = Modflow{{ name.title.title() }}( self._cpparent, - {%- for n, var in vars.items() if n not in init_skip %} - {{ n }}={{ n }}, - {%- endfor %} + {{ macros.vars_params(vars, init_skip, start_indent=12) }} filename=filename, pname=pname, child_builder_call=True, @@ -115,17 +110,13 @@ class {{ name.title.title() }}Packages(MFChildPackages): {% if name.r != "obs" %} def append_package( self, - {%- for n, var in vars.items() if n not in init_skip %} - {{ n }}{%- if var.default is defined %}={{ var.default }}{% endif -%}, - {%- endfor %} + {{ macros.vars_params(vars, init_skip, start_indent=8) }} filename=None, pname=None, ): new_package = Modflow{{ name.title.title() }}( self._cpparent, - {%- for n, var in vars.items() if n not in init_skip %} - {{ n }}={{ n }}, - {%- endfor %} + {{ macros.vars_params(vars, init_skip, start_indent=12) }} filename=filename, pname=pname, child_builder_call=True, diff --git a/flopy/mf6/utils/codegen/templates/simulation.py.jinja b/flopy/mf6/utils/codegen/templates/simulation.py.jinja index 39f807574e..eb4b98f18f 100644 --- a/flopy/mf6/utils/codegen/templates/simulation.py.jinja +++ b/flopy/mf6/utils/codegen/templates/simulation.py.jinja @@ -1,4 +1,5 @@ {% import 'macros.jinja' as macros %} +# autogenerated file, do not modify from os import PathLike, curdir from typing import Union @@ -34,9 +35,7 @@ class MF{{ name.title.title() }}(MFSimulationBase): write_headers: bool = True, use_pandas: bool = True, lazy_io: bool = False, - {%- for n, var in vars.items() if n not in init_skip %} - {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, - {%- endfor %} + {{ macros.vars_params(vars, init_skip, start_indent=8) }} ): """ {{ name.description }} From ec2b1789b2dbcfda7ced8d0654ec3f0db21e9640 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 31 Oct 2024 14:10:28 -0400 Subject: [PATCH 51/75] initial toml support --- .gitignore | 5 +++- flopy/mf6/utils/codegen/__init__.py | 24 +++++++--------- flopy/mf6/utils/codegen/dfn.py | 2 ++ flopy/mf6/utils/codegen/dfn2toml.py | 40 +++++++++++++++++++++++++++ flopy/mf6/utils/codegen/renderable.py | 8 ++++-- 5 files changed, 62 insertions(+), 17 deletions(-) create mode 100644 flopy/mf6/utils/codegen/dfn2toml.py diff --git a/.gitignore b/.gitignore index 5b7873e8dc..6dc1a714c7 100644 --- a/.gitignore +++ b/.gitignore @@ -103,4 +103,7 @@ app **.DS_Store # DFN backups -flopy/mf6/data/dfn_backup/ \ No newline at end of file +flopy/mf6/data/dfn_backup/ + +# DFN TOML dir +flopy/mf6/data/toml/ \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/__init__.py b/flopy/mf6/utils/codegen/__init__.py index 80d00964d7..9f4264c746 100644 --- a/flopy/mf6/utils/codegen/__init__.py +++ b/flopy/mf6/utils/codegen/__init__.py @@ -1,29 +1,25 @@ from pathlib import Path -from warnings import warn from flopy.utils import import_optional_dependency __all__ = ["make_targets", "make_all"] - -jinja = import_optional_dependency("jinja2", errors="ignore") -if jinja: - _TEMPLATES_PATH = "mf6/utils/codegen/templates/" - _TEMPLATE_LOADER = jinja.PackageLoader("flopy", _TEMPLATES_PATH) - _TEMPLATE_ENV = jinja.Environment(loader=_TEMPLATE_LOADER) +__jinja = import_optional_dependency("jinja2", errors="ignore") def make_targets(dfn, outdir: Path, verbose: bool = False): """Generate Python source file(s) from the given input definition.""" - from flopy.mf6.utils.codegen.context import Context - - if not jinja: + if not __jinja: raise RuntimeError("Jinja2 not installed, can't make targets") + from flopy.mf6.utils.codegen.context import Context + + loader = __jinja.PackageLoader("flopy", "mf6/utils/codegen/templates/") + env = __jinja.Environment(loader=loader) for context in Context.from_dfn(dfn): name = context.name target = outdir / name.target - template = _TEMPLATE_ENV.get_template(name.template) + template = env.get_template(name.template) with open(target, "w") as f: f.write(template.render(**context.render())) if verbose: @@ -33,12 +29,12 @@ def make_targets(dfn, outdir: Path, verbose: bool = False): def make_all(dfndir: Path, outdir: Path, verbose: bool = False): """Generate Python source files from the DFN files in the given location.""" + if not __jinja: + raise RuntimeError("Jinja2 not installed, can't make targets") + from flopy.mf6.utils.codegen.context import Context from flopy.mf6.utils.codegen.dfn import Dfn - if not jinja: - raise RuntimeError("Jinja2 not installed, can't make targets") - # load dfns dfns = Dfn.load_all(dfndir) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index bb9ecaa9b8..e3b508d96a 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -16,6 +16,7 @@ from boltons.dictutils import OMD +from flopy.mf6.utils.codegen.renderable import renderable from flopy.mf6.utils.codegen.utils import try_literal_eval, try_parse_bool _SCALARS = { @@ -144,6 +145,7 @@ def _parent(): ) +@renderable class Dfn(UserDict): """ MODFLOW 6 input definition. An input definition diff --git a/flopy/mf6/utils/codegen/dfn2toml.py b/flopy/mf6/utils/codegen/dfn2toml.py new file mode 100644 index 0000000000..1a9d6673b2 --- /dev/null +++ b/flopy/mf6/utils/codegen/dfn2toml.py @@ -0,0 +1,40 @@ +import argparse +from pathlib import Path + +from flopy.utils import import_optional_dependency + +_MF6_PATH = Path(__file__).parents[2] +_DFN_PATH = _MF6_PATH / "data" / "dfn" +_TOML_PATH = _MF6_PATH / "data" / "toml" + +__tomlkit = import_optional_dependency("tomlkit") + + +if __name__ == "__main__": + """Convert DFN files to TOML.""" + + if not __tomlkit: + raise RuntimeError("tomlkit not installed, can't convert DFNs to TOML") + + from flopy.mf6.utils.codegen.dfn import Dfn + + parser = argparse.ArgumentParser(description="Convert DFN files to TOML.") + parser.add_argument( + "--dfndir", + type=str, + default=_DFN_PATH, + help="Directory containing DFN files.", + ) + parser.add_argument( + "--outdir", + default=_TOML_PATH, + help="Output directory.", + ) + + args = parser.parse_args() + dfndir = Path(args.dfndir) + outdir = Path(args.outdir) + outdir.mkdir(exist_ok=True, parents=True) + for dfn in Dfn.load_all(dfndir).values(): + with open(Path(outdir) / f"{'-'.join(dfn.name)}.toml", "w") as f: + __tomlkit.dump(dfn.render(), f) diff --git a/flopy/mf6/utils/codegen/renderable.py b/flopy/mf6/utils/codegen/renderable.py index 3aeaafd6cd..e8fdf2e21c 100644 --- a/flopy/mf6/utils/codegen/renderable.py +++ b/flopy/mf6/utils/codegen/renderable.py @@ -20,7 +20,7 @@ can also be implemented with transformations. """ -from dataclasses import asdict +from dataclasses import asdict, is_dataclass from typing import Any, Callable, Dict, Iterable, Optional, Tuple from flopy.mf6.utils.codegen.utils import try_get_enum_value @@ -128,7 +128,11 @@ def _dict_factory(o): def render(self) -> dict: """Recursively render the dataclass instance.""" - return _render(asdict(self, dict_factory=_dict_factory)) + return _render( + asdict(self, dict_factory=_dict_factory) + if is_dataclass(self) + else self + ) setattr(cls, "render", render) return cls From 002982a0d99fc453752d89fb604b0c25ef22d8be Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 31 Oct 2024 14:35:15 -0400 Subject: [PATCH 52/75] Revert "add params macro" This reverts commit 44f5db44988aa55db86f94741bc39aeb42e880e9. --- .../utils/codegen/templates/exchange.py.jinja | 5 +++-- .../mf6/utils/codegen/templates/macros.jinja | 6 ------ .../utils/codegen/templates/model.py.jinja | 5 +++-- .../utils/codegen/templates/package.py.jinja | 21 +++++++++++++------ .../codegen/templates/simulation.py.jinja | 5 +++-- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/flopy/mf6/utils/codegen/templates/exchange.py.jinja b/flopy/mf6/utils/codegen/templates/exchange.py.jinja index 4b76026d52..a5d4a1e98f 100644 --- a/flopy/mf6/utils/codegen/templates/exchange.py.jinja +++ b/flopy/mf6/utils/codegen/templates/exchange.py.jinja @@ -1,5 +1,4 @@ {% import 'macros.jinja' as macros %} -# autogenerated file, do not modify from os import PathLike, curdir from typing import Union @@ -27,7 +26,9 @@ class Modflow{{ name.title.title() }}(MFPackage): exgtype="{{ name.r[:3].upper() }}6-{{ name.r[3:].upper() }}6", exgmnamea=None, exgmnameb=None, - {{ macros.vars_params(vars, init_skip, start_indent=8) }} + {%- for n, var in vars.items() if n not in init_skip %} + {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, + {%- endfor %} filename=None, pname=None, **kwargs, diff --git a/flopy/mf6/utils/codegen/templates/macros.jinja b/flopy/mf6/utils/codegen/templates/macros.jinja index fe5ceb3fb6..be5b1ab1a9 100644 --- a/flopy/mf6/utils/codegen/templates/macros.jinja +++ b/flopy/mf6/utils/codegen/templates/macros.jinja @@ -1,9 +1,3 @@ -{% macro vars_params(vars, init_skip=none, start_indent=0) %} -{%- for n, var in vars.items() if n not in init_skip %} -{{ ""|indent(start_indent, first=true) }}{{ var.name }}{%- if var.default is defined %}={{ var.default }}{% endif -%}, -{%- endfor %} -{% endmacro %} - {% macro vars_docs(vars, start_indent=0) %} {%- for v in vars.values() recursive %} {{ ""|indent(start_indent, first=true) }}{% if loop.depth > 1 %}* {% endif %}{% if v.name|last == "_" %}{{ v.name.replace("_", "\\\_") }}{% else %}{{ v.name }}{% endif %}{% if v.meta is defined and v.meta.type is defined %} : {{ v.meta.type }}{% endif %} diff --git a/flopy/mf6/utils/codegen/templates/model.py.jinja b/flopy/mf6/utils/codegen/templates/model.py.jinja index 3754124af0..a922bad9e0 100644 --- a/flopy/mf6/utils/codegen/templates/model.py.jinja +++ b/flopy/mf6/utils/codegen/templates/model.py.jinja @@ -1,5 +1,4 @@ {% import 'macros.jinja' as macros %} -# autogenerated file, do not modify from os import PathLike, curdir from typing import Union @@ -34,7 +33,9 @@ class Modflow{{ name.title.title() }}(MFModel): version="mf6", exe_name="mf6", model_rel_path=".", - {{ macros.vars_params(vars, init_skip, start_indent=8) }} + {%- for n, var in vars.items() if n not in init_skip %} + {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, + {%- endfor %} **kwargs, ): """ diff --git a/flopy/mf6/utils/codegen/templates/package.py.jinja b/flopy/mf6/utils/codegen/templates/package.py.jinja index e3d2375a2f..576ba79a78 100644 --- a/flopy/mf6/utils/codegen/templates/package.py.jinja +++ b/flopy/mf6/utils/codegen/templates/package.py.jinja @@ -1,5 +1,4 @@ {% import 'macros.jinja' as macros %} -# autogenerated file, do not modify from os import PathLike, curdir from typing import Union @@ -25,7 +24,9 @@ class Modflow{{ name.title.title() }}(MFPackage): self, {{ parent }}, loading_package=False, - {{ macros.vars_params(vars, init_skip, start_indent=8) }} + {%- for n, var in vars.items() if n not in init_skip %} + {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, + {%- endfor %} filename=None, pname=None, **kwargs, @@ -94,13 +95,17 @@ class {{ name.title.title() }}Packages(MFChildPackages): def initialize( self, - {{ macros.vars_params(vars, init_skip, start_indent=8) }} + {%- for n, var in vars.items() if n not in init_skip %} + {{ n }}{%- if var.default is defined %}={{ var.default }}{% endif -%}, + {%- endfor %} filename=None, pname=None, ): new_package = Modflow{{ name.title.title() }}( self._cpparent, - {{ macros.vars_params(vars, init_skip, start_indent=12) }} + {%- for n, var in vars.items() if n not in init_skip %} + {{ n }}={{ n }}, + {%- endfor %} filename=filename, pname=pname, child_builder_call=True, @@ -110,13 +115,17 @@ class {{ name.title.title() }}Packages(MFChildPackages): {% if name.r != "obs" %} def append_package( self, - {{ macros.vars_params(vars, init_skip, start_indent=8) }} + {%- for n, var in vars.items() if n not in init_skip %} + {{ n }}{%- if var.default is defined %}={{ var.default }}{% endif -%}, + {%- endfor %} filename=None, pname=None, ): new_package = Modflow{{ name.title.title() }}( self._cpparent, - {{ macros.vars_params(vars, init_skip, start_indent=12) }} + {%- for n, var in vars.items() if n not in init_skip %} + {{ n }}={{ n }}, + {%- endfor %} filename=filename, pname=pname, child_builder_call=True, diff --git a/flopy/mf6/utils/codegen/templates/simulation.py.jinja b/flopy/mf6/utils/codegen/templates/simulation.py.jinja index eb4b98f18f..39f807574e 100644 --- a/flopy/mf6/utils/codegen/templates/simulation.py.jinja +++ b/flopy/mf6/utils/codegen/templates/simulation.py.jinja @@ -1,5 +1,4 @@ {% import 'macros.jinja' as macros %} -# autogenerated file, do not modify from os import PathLike, curdir from typing import Union @@ -35,7 +34,9 @@ class MF{{ name.title.title() }}(MFSimulationBase): write_headers: bool = True, use_pandas: bool = True, lazy_io: bool = False, - {{ macros.vars_params(vars, init_skip, start_indent=8) }} + {%- for n, var in vars.items() if n not in init_skip %} + {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, + {%- endfor %} ): """ {{ name.description }} From b29591cc013740558ee5763cb2ed5177a950d1e4 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 31 Oct 2024 15:27:35 -0400 Subject: [PATCH 53/75] fix renderable --- flopy/mf6/utils/codegen/renderable.py | 2 +- flopy/mf6/utils/codegen/shim.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/flopy/mf6/utils/codegen/renderable.py b/flopy/mf6/utils/codegen/renderable.py index e8fdf2e21c..2ebd6abf84 100644 --- a/flopy/mf6/utils/codegen/renderable.py +++ b/flopy/mf6/utils/codegen/renderable.py @@ -103,7 +103,7 @@ def _render_val(k, v): return { k: _render_val(k, v) for k, v in d.items() - if (k in keep_none or v is not None) + if (k in keep_none or (v and not isinstance(v, bool))) } def _dict(o): diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index e1c3013a7d..41024fb2a2 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -339,7 +339,7 @@ def _transform_context(o): SHIM = { - "keep_none": ["default"], + "keep_none": ["default", "vars"], "quote_str": ["default"], "set_pairs": [ ( From f0db6ae87b5cafdf8a994c40909031fd50e3d9b4 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 31 Oct 2024 15:36:36 -0400 Subject: [PATCH 54/75] support var type directly (as str for now) --- flopy/mf6/utils/codegen/dfn.py | 17 ++++++++--------- flopy/mf6/utils/codegen/renderable.py | 10 +++++++++- flopy/mf6/utils/codegen/templates/macros.jinja | 2 +- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index e3b508d96a..6d51433354 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -49,11 +49,11 @@ class Kind(Enum): name: str kind: Kind + type: Optional[str] = None block: Optional[str] = None default: Optional[Any] = None children: Optional[Vars] = None description: Optional[str] = None - meta: Optional[Dict[str, Any]] = None class Ref(TypedDict): @@ -410,9 +410,7 @@ def _is_implicit_scalar_record(): block=block, children=fields, description=description, - meta={ - "type": f"[{', '.join([f['meta']['type'] for f in fields.values()])}]" - }, + type=f"[{', '.join([f['type'] for f in fields.values()])}]", ) } kind = Var.Kind.List @@ -433,9 +431,7 @@ def _is_implicit_scalar_record(): block=block, children=first["children"] if single else fields, description=description, - meta={ - "type": f"[{', '.join([v['meta']['type'] for v in fields.values()])}]" - }, + type=f"[{', '.join([v['type'] for v in fields.values()])}]", ) } kind = Var.Kind.List @@ -460,7 +456,9 @@ def _is_implicit_scalar_record(): elif _type.startswith("record"): children = _fields(_name) kind = Var.Kind.Record - type_ = f"[{', '.join([v['meta']['type'] for v in children.values()])}]" + type_ = ( + f"[{', '.join([v['type'] for v in children.values()])}]" + ) # at this point, if it has a shape, it's an array elif shape is not None: @@ -485,6 +483,7 @@ def _is_implicit_scalar_record(): "-", "_" ), kind=kind, + type=type_, block=block, description=description, default=( @@ -494,7 +493,7 @@ def _is_implicit_scalar_record(): # type is a string for now, when # introducing type hints make it # a proper type... - meta={"ref": ref, "type": type_}, + meta={"ref": ref}, ) # pass the original DFN representation as diff --git a/flopy/mf6/utils/codegen/renderable.py b/flopy/mf6/utils/codegen/renderable.py index 2ebd6abf84..6d29c0991b 100644 --- a/flopy/mf6/utils/codegen/renderable.py +++ b/flopy/mf6/utils/codegen/renderable.py @@ -35,6 +35,7 @@ def renderable( maybe_cls=None, *, keep_none: Optional[Iterable[str]] = None, + drop_keys: Optional[Iterable[str]] = None, quote_str: Optional[Iterable[str]] = None, set_pairs: Optional[Iterable[Tuple[Predicate, Pairs]]] = None, transform: Optional[Iterable[Tuple[Predicate, Transform]]] = None, @@ -83,6 +84,7 @@ def renderable( quote_str = quote_str or list() keep_none = keep_none or list() + drop_keys = drop_keys or list() set_pairs = set_pairs or list() transform = transform or list() @@ -100,10 +102,16 @@ def _render_val(k, v): v = _render(v) return v + def _keep(k, v): + return k in keep_none or (v and not isinstance(v, bool)) + + def _drop(k, v): + return k in drop_keys + return { k: _render_val(k, v) for k, v in d.items() - if (k in keep_none or (v and not isinstance(v, bool))) + if (_keep(k, v) and not _drop(k, v)) } def _dict(o): diff --git a/flopy/mf6/utils/codegen/templates/macros.jinja b/flopy/mf6/utils/codegen/templates/macros.jinja index be5b1ab1a9..153ba15d9e 100644 --- a/flopy/mf6/utils/codegen/templates/macros.jinja +++ b/flopy/mf6/utils/codegen/templates/macros.jinja @@ -1,6 +1,6 @@ {% macro vars_docs(vars, start_indent=0) %} {%- for v in vars.values() recursive %} -{{ ""|indent(start_indent, first=true) }}{% if loop.depth > 1 %}* {% endif %}{% if v.name|last == "_" %}{{ v.name.replace("_", "\\\_") }}{% else %}{{ v.name }}{% endif %}{% if v.meta is defined and v.meta.type is defined %} : {{ v.meta.type }}{% endif %} +{{ ""|indent(start_indent, first=true) }}{% if loop.depth > 1 %}* {% endif %}{% if v.name|last == "_" %}{{ v.name.replace("_", "\\\_") }}{% else %}{{ v.name }}{% endif %}{% if v.type is defined %} : {{ v.type }}{% endif %} {%- if v.description is defined and v.description is not none %} {{ v.description|wordwrap|indent(start_indent + (loop.depth * 4), first=true) }} {%- endif %} From a85104468a31a473e009fcba509394a14daf26a8 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Mon, 4 Nov 2024 15:02:31 -0500 Subject: [PATCH 55/75] revisions --- flopy/mf6/utils/codegen/__init__.py | 64 ++-- flopy/mf6/utils/codegen/dfn.py | 23 ++ flopy/mf6/utils/codegen/dfn2toml.py | 9 +- flopy/mf6/utils/codegen/jinja.py | 297 +++++++++++++++++ flopy/mf6/utils/codegen/renderable.py | 22 -- flopy/mf6/utils/codegen/shim.py | 303 ------------------ .../utils/codegen/templates/__init__.py.jinja | 3 + .../utils/codegen/templates/exchange.py.jinja | 6 +- .../mf6/utils/codegen/templates/macros.jinja | 6 + .../utils/codegen/templates/model.py.jinja | 4 +- .../utils/codegen/templates/package.py.jinja | 22 +- .../codegen/templates/simulation.py.jinja | 4 +- flopy/mf6/utils/codegen/utils.py | 30 -- pyproject.toml | 3 +- 14 files changed, 385 insertions(+), 411 deletions(-) create mode 100644 flopy/mf6/utils/codegen/jinja.py create mode 100644 flopy/mf6/utils/codegen/templates/__init__.py.jinja diff --git a/flopy/mf6/utils/codegen/__init__.py b/flopy/mf6/utils/codegen/__init__.py index 9f4264c746..0439f68c0b 100644 --- a/flopy/mf6/utils/codegen/__init__.py +++ b/flopy/mf6/utils/codegen/__init__.py @@ -1,21 +1,53 @@ +from os import PathLike from pathlib import Path from flopy.utils import import_optional_dependency __all__ = ["make_targets", "make_all"] -__jinja = import_optional_dependency("jinja2", errors="ignore") -def make_targets(dfn, outdir: Path, verbose: bool = False): - """Generate Python source file(s) from the given input definition.""" +def _get_template_env(): + from flopy.mf6.utils.codegen.jinja import Filters + + jinja = import_optional_dependency("jinja2") + loader = jinja.PackageLoader("flopy", "mf6/utils/codegen/templates/") + env = jinja.Environment(loader=loader) + env.filters["parent"] = Filters.parent + env.filters["prefix"] = Filters.prefix + env.filters["skip"] = Filters.skip + return env - if not __jinja: - raise RuntimeError("Jinja2 not installed, can't make targets") + +def make_init(dfns: dict, outdir: PathLike, verbose: bool = False): + """Generate a Python __init__.py file for the given input definitions.""" from flopy.mf6.utils.codegen.context import Context - loader = __jinja.PackageLoader("flopy", "mf6/utils/codegen/templates/") - env = __jinja.Environment(loader=loader) + env = _get_template_env() + outdir = Path(outdir).expanduser() + contexts = [ + c + for cc in [ + [ctx for ctx in Context.from_dfn(dfn)] for dfn in dfns.values() + ] + for c in cc + ] # ugly, but it's the fastest way to flatten the list + target_name = "__init__.py" + target = outdir / target_name + template = env.get_template(f"{target_name}.jinja") + with open(target, "w") as f: + f.write(template.render(contexts=contexts)) + if verbose: + print(f"Wrote {target}") + + +def make_targets(dfn, outdir: PathLike, verbose: bool = False): + """Generate Python source file(s) from the given input definition.""" + + from flopy.mf6.utils.codegen.context import Context + + env = _get_template_env() + outdir = Path(outdir).expanduser() for context in Context.from_dfn(dfn): name = context.name target = outdir / name.target @@ -29,25 +61,9 @@ def make_targets(dfn, outdir: Path, verbose: bool = False): def make_all(dfndir: Path, outdir: Path, verbose: bool = False): """Generate Python source files from the DFN files in the given location.""" - if not __jinja: - raise RuntimeError("Jinja2 not installed, can't make targets") - - from flopy.mf6.utils.codegen.context import Context from flopy.mf6.utils.codegen.dfn import Dfn - # load dfns dfns = Dfn.load_all(dfndir) - - # make target files + make_init(dfns, outdir, verbose) for dfn in dfns.values(): make_targets(dfn, outdir, verbose) - - # make __init__.py file - init_path = outdir / "__init__.py" - with open(init_path, "w") as f: - for dfn in dfns.values(): - for name in Context.Name.from_dfn(dfn): - prefix = "MF" if name.base == "MFSimulationBase" else "Modflow" - f.write( - f"from .mf{name.title} import {prefix}{name.title.title()}\n" - ) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index 6d51433354..cf6aac5df0 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -27,6 +27,29 @@ } +def try_literal_eval(value: str) -> Any: + """ + Try to parse a string as a literal. If this fails, + return the value unaltered. + """ + try: + return literal_eval(value) + except (SyntaxError, ValueError): + return value + + +def try_parse_bool(value: Any) -> Any: + """ + Try to parse a boolean from a string as represented + in a DFN file, otherwise return the value unaltered. + """ + if isinstance(value, str): + value = value.lower() + if value in ["true", "false"]: + return value == "true" + return value + + Vars = Dict[str, "Var"] Refs = Dict[str, "Ref"] Dfns = Dict[str, "Dfn"] diff --git a/flopy/mf6/utils/codegen/dfn2toml.py b/flopy/mf6/utils/codegen/dfn2toml.py index 1a9d6673b2..771be3509e 100644 --- a/flopy/mf6/utils/codegen/dfn2toml.py +++ b/flopy/mf6/utils/codegen/dfn2toml.py @@ -7,17 +7,13 @@ _DFN_PATH = _MF6_PATH / "data" / "dfn" _TOML_PATH = _MF6_PATH / "data" / "toml" -__tomlkit = import_optional_dependency("tomlkit") - if __name__ == "__main__": """Convert DFN files to TOML.""" - if not __tomlkit: - raise RuntimeError("tomlkit not installed, can't convert DFNs to TOML") - from flopy.mf6.utils.codegen.dfn import Dfn + tomlkit = import_optional_dependency("tomlkit") parser = argparse.ArgumentParser(description="Convert DFN files to TOML.") parser.add_argument( "--dfndir", @@ -30,11 +26,10 @@ default=_TOML_PATH, help="Output directory.", ) - args = parser.parse_args() dfndir = Path(args.dfndir) outdir = Path(args.outdir) outdir.mkdir(exist_ok=True, parents=True) for dfn in Dfn.load_all(dfndir).values(): with open(Path(outdir) / f"{'-'.join(dfn.name)}.toml", "w") as f: - __tomlkit.dump(dfn.render(), f) + tomlkit.dump(dfn.render(), f) diff --git a/flopy/mf6/utils/codegen/jinja.py b/flopy/mf6/utils/codegen/jinja.py new file mode 100644 index 0000000000..58a57260ed --- /dev/null +++ b/flopy/mf6/utils/codegen/jinja.py @@ -0,0 +1,297 @@ +from enum import Enum +from keyword import kwlist +from pprint import pformat +from typing import Any, List, Optional +from jinja2 import pass_context + + +def try_get_enum_value(v: Any) -> Any: + """ + Get the enum's value if the object is an instance + of an enumeration, otherwise return it unaltered. + """ + return v.value if isinstance(v, Enum) else v + + +class Filters: + + @pass_context + def cls_attrs(ctx, ctx_name) -> List[str]: + def _attr(var: dict) -> Optional[str]: + var_name = var["name"] + var_kind = try_get_enum_value(var.get("kind", None)) + var_block = var.get("block", None) + var_ref = var.get("meta", dict()).get("ref", None) + + if ( + var_kind is None + or var_kind == "scalar" + or var_name in ["cvoptions", "output"] + or (ctx_name.r == "dis" and var_name == "packagedata") + or ( + var_name != "packages" + and (ctx_name.l is not None and ctx_name.r == "nam") + ) + ): + return None + + if var_kind in ["list", "record", "union", "array"]: + if not var_block: + raise ValueError("Need block") + + if var_kind != "array": + if var_ref: + # if the variable is a subpackage reference, use the original key + # (which has been replaced already with the referenced variable) + args = [ + f"'{ctx_name.r}'", + f"'{var_block}'", + f"'{var_ref['key']}'", + ] + if ctx_name.l is not None and ctx_name.l not in [ + "sim", + "sln", + "utl", + "exg", + ]: + args.insert(0, f"'{ctx_name.l}6'") + return f"{var_ref['key']} = ListTemplateGenerator(({', '.join(args)}))" + + def _args(): + args = [f"'{ctx_name.r}'", f"'{var_block}'", f"'{var_name}'"] + if ctx_name.l is not None and ctx_name.l not in [ + "sim", + "sln", + "utl", + "exg", + ]: + args.insert(0, f"'{ctx_name.l}6'") + return args + + kind = var_kind if var_kind == "array" else "list" + return f"{var_name} = {kind.title()}TemplateGenerator(({', '.join(_args())}))" + + return None + + def _dfn() -> List[List[str]]: + dfn, meta = ctx["meta"]["dfn"] + + def _meta(): + exclude = ["subpackage", "parent_name_type"] + return [v for v in meta if not any(p in v for p in exclude)] + + def _dfn(): + def _var(var: dict) -> List[str]: + exclude = ["longname", "description"] + name = var["name"] + var_ = ctx["vars"].get(name, None) + keys = [ + "construct_package", + "construct_data", + "parameter_name", + ] + if var_ and keys[0] in var_: + for k in keys: + var[k] = var_[k] + return [ + " ".join([k, v]).strip() + for k, v in var.items() + if k not in exclude + ] + + return [_var(var) for var in dfn] + + return [["header"] + _meta()] + _dfn() + + attrs = list(filter(None, [_attr(v) for v in ctx["vars"].values()])) + + if ctx_name.base == "MFModel": + attrs.append(f"model_type = {ctx_name.l}") + elif ctx_name.base == "MFPackage": + attrs.extend( + [ + f"package_abbr = '{ctx_name.r}'" + if ctx_name.l == "exg" + else f"package_abbr = '{'' if ctx_name.l in ['sln', 'sim', 'exg', None] else ctx_name.l}{ctx_name.r}'", + f"_package_type = '{ctx_name.r}'", + f"dfn_file_name = '{ctx_name.l}-{ctx_name.r}.dfn'" + if ctx_name.l == "exg" + else f"dfn_file_name = '{ctx_name.l or 'sim'}-{ctx_name.r}.dfn'", + f"dfn = {pformat(_dfn(), indent=10)}", + ] + ) + + return attrs + + @pass_context + def init_body(ctx, ctx_name) -> List[str]: + def _statements() -> Optional[List[str]]: + base = ctx_name.base + if base == "MFSimulationBase": + + def _should_set(var: dict) -> bool: + return var["name"] not in [ + "tdis6", + "models", + "exchanges", + "mxiter", + "solutiongroup", + "hpc_data", + ] + + stmts = [] + refs = {} + for var in ctx["vars"].values(): + ref = var.get("meta", dict()).get("ref", None) + if not var.get("kind", None): + continue + + name = var["name"] + if name in kwlist: + name = f"{name}_" + + if _should_set(var): + stmts.append(f"self.name_file.{name}.set_data({name})") + stmts.append(f"self.{name} = self.name_file.{name}") + if ref and ref["key"] not in refs: + refs[ref["key"]] = ref + stmts.append( + f"self.{ref['param']} = self._create_package('{ref['abbr']}', {ref['param']})" + ) + elif base == "MFModel": + + def _should_set(var: dict) -> bool: + return var["name"] not in [ + "export_netcdf", + "nc_filerecord", + "packages", + ] + + stmts = [] + refs = {} + for var in ctx["vars"].values(): + ref = var.get("meta", dict()).get("ref", None) + if not var.get("kind", None): + continue + + name = var["name"] + if name in kwlist: + name = f"{name}_" + + if _should_set(var): + stmts.append(f"self.name_file.{name}.set_data({name})") + stmts.append(f"self.{name} = self.name_file.{name}") + if ref and ref["key"] not in refs: + refs[ref["key"]] = ref + stmts.append( + f"self.{ref['param']} = self._create_package('{ref['abbr']}', {ref['param']})" + ) + elif base == "MFPackage": + + def _should_build(var: dict) -> bool: + if var.get("meta", dict()).get("ref", None) and ctx[ + "name" + ] != ( + None, + "nam", + ): + return False + name = var["name"] + if name in [ + "simulation", + "model", + "package", + "parent_model", + "parent_package", + "parent_model_or_package", + "parent_file", + "modelname", + "model_nam_file", + "export_netcdf", + "nc_filerecord", + "method", + "interpolation_method_single", + "sfac", + "output", + ]: + return False + return True + + stmts = [] + refs = {} + for var in ctx["vars"].values(): + name = var["name"] + ref = var.get("meta", dict()).get("ref", None) + if name in kwlist: + name = f"{name}_" + + if _should_build(var): + if ref and ctx["name"] == (None, "nam"): + stmts.append( + f"self.{'_' if ref else ''}{ref['key']} = self.build_mfdata('{ref['key']}', None)" + ) + else: + # hack... + _name = name[:-1] if name.endswith("_") else name + if _name == "steady_state": + _name = "steady-state" + stmts.append( + f"self.{'_' if ref else ''}{name} = self.build_mfdata('{_name}', {name if var.get('init_param', True) else 'None'})" + ) + + if ref and ref["key"] not in refs and ctx["name"].r != "nam": + refs[ref["key"]] = ref + stmts.append( + f"self._{ref['key']} = self.build_mfdata('{ref['key']}', None)" + ) + stmts.append( + f"self._{ref['abbr']}_package = self.build_child_package('{ref['abbr']}', {ref['val']}, '{ref['param']}', self._{ref['key']})" + ) + + return stmts + + return list(filter(None, _statements())) + + @pass_context + def parent(ctx, ctx_name): + ref = ctx["meta"].get("ref", None) + if ref: + return ref["parent"] + if ctx_name == ("sim", "nam"): + return None + elif ctx_name.l is None or ctx_name.r is None or ctx_name.l in ["sim", "exg", "sln"]: + return "simulation" + elif ref: + if ctx_name.l == "utl" and ctx_name.r == "hpc": + return "simulation" + return "package" + return "model" + + def prefix(ctx_name): + return "MF" if ctx_name.base == "MFSimulationBase" else "Modflow" + + @pass_context + def skip(ctx, ctx_name): + base = ctx_name.base + if base == "MFSimulationBase": + return [ + "tdis6", + "models", + "exchanges", + "mxiter", + "solutiongroup", + ] + elif base == "MFModel": + skip = ["packages", "export_netcdf", "nc_filerecord"] + refs = ctx.get("meta", dict()).get("refs", dict()) + if any(refs) and ctx["name"] != (None, "nam"): + for k in refs.keys(): + if ctx["vars"].get(k, None): + skip.append(k) + return skip + else: + if ctx_name.r == "nam": + return ["export_netcdf", "nc_filerecord"] + elif ctx_name == ("utl", "ts"): + return ["method", "interpolation_method_single", "sfac"] + return [] diff --git a/flopy/mf6/utils/codegen/renderable.py b/flopy/mf6/utils/codegen/renderable.py index 6d29c0991b..6e1384fd15 100644 --- a/flopy/mf6/utils/codegen/renderable.py +++ b/flopy/mf6/utils/codegen/renderable.py @@ -7,17 +7,6 @@ cleanly isolated from the reimplementation of which this code is a part, which aims for a more general approach. - -Jinja supports attribute- and dictionary- -based access on arbitrary objects but does -not support arbitrary expressions, and has -only a limited set of custom filters; this -can make it awkward to express some things, -which transformations can also remedy. - -Edge cases in the MF6 classes, e.g. the logic -determining the contents of generated classes, -can also be implemented with transformations. """ from dataclasses import asdict, is_dataclass @@ -37,7 +26,6 @@ def renderable( keep_none: Optional[Iterable[str]] = None, drop_keys: Optional[Iterable[str]] = None, quote_str: Optional[Iterable[str]] = None, - set_pairs: Optional[Iterable[Tuple[Predicate, Pairs]]] = None, transform: Optional[Iterable[Tuple[Predicate, Transform]]] = None, ): """ @@ -85,7 +73,6 @@ def renderable( quote_str = quote_str or list() keep_none = keep_none or list() drop_keys = drop_keys or list() - set_pairs = set_pairs or list() transform = transform or list() def __renderable(cls): @@ -120,15 +107,6 @@ def _dict(o): if p(o): d = t(o) break - - for p, e in set_pairs: - if not (p(d) and e): - continue - for k, v in e: - if callable(v): - v = v(d) - d[k] = v - return d def _dict_factory(o): diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py index 41024fb2a2..f94c55576d 100644 --- a/flopy/mf6/utils/codegen/shim.py +++ b/flopy/mf6/utils/codegen/shim.py @@ -5,304 +5,12 @@ generation infrastructure. """ -from keyword import kwlist -from pprint import pformat -from typing import List, Optional - -from flopy.mf6.utils.codegen.utils import try_get_enum_value - - -def _cls_attrs(ctx: dict) -> List[str]: - ctx_name = ctx["name"] - - def _attr(var: dict) -> Optional[str]: - var_name = var["name"] - var_kind = try_get_enum_value(var.get("kind", None)) - var_block = var.get("block", None) - var_ref = var.get("meta", dict()).get("ref", None) - - if ( - var_kind is None - or var_kind == "scalar" - or var_name in ["cvoptions", "output"] - or (ctx_name.r == "dis" and var_name == "packagedata") - or ( - var_name != "packages" - and (ctx_name.l is not None and ctx_name.r == "nam") - ) - ): - return None - - if var_kind in ["list", "record", "union", "array"]: - if not var_block: - raise ValueError("Need block") - - if var_kind != "array": - if var_ref: - # if the variable is a subpackage reference, use the original key - # (which has been replaced already with the referenced variable) - args = [ - f"'{ctx_name.r}'", - f"'{var_block}'", - f"'{var_ref['key']}'", - ] - if ctx_name.l is not None and ctx_name.l not in [ - "sim", - "sln", - "utl", - "exg", - ]: - args.insert(0, f"'{ctx_name.l}6'") - return f"{var_ref['key']} = ListTemplateGenerator(({', '.join(args)}))" - - def _args(): - args = [f"'{ctx_name.r}'", f"'{var_block}'", f"'{var_name}'"] - if ctx_name.l is not None and ctx_name.l not in [ - "sim", - "sln", - "utl", - "exg", - ]: - args.insert(0, f"'{ctx_name.l}6'") - return args - - kind = var_kind if var_kind == "array" else "list" - return f"{var_name} = {kind.title()}TemplateGenerator(({', '.join(_args())}))" - - return None - - def _dfn() -> List[List[str]]: - dfn, meta = ctx["meta"]["dfn"] - - def _meta(): - exclude = ["subpackage", "parent_name_type"] - return [v for v in meta if not any(p in v for p in exclude)] - - def _dfn(): - def _var(var: dict) -> List[str]: - exclude = ["longname", "description"] - name = var["name"] - var_ = ctx["vars"].get(name, None) - keys = [ - "construct_package", - "construct_data", - "parameter_name", - ] - if var_ and keys[0] in var_: - for k in keys: - var[k] = var_[k] - return [ - " ".join([k, v]).strip() - for k, v in var.items() - if k not in exclude - ] - - return [_var(var) for var in dfn] - - return [["header"] + _meta()] + _dfn() - - attrs = list(filter(None, [_attr(v) for v in ctx["vars"].values()])) - - if ctx_name.base == "MFModel": - attrs.append(f"model_type = {ctx_name.l}") - elif ctx_name.base == "MFPackage": - attrs.extend( - [ - f"package_abbr = '{ctx_name.r}'" - if ctx_name.l == "exg" - else f"package_abbr = '{'' if ctx_name.l in ['sln', 'sim', 'exg', None] else ctx_name.l}{ctx_name.r}'", - f"_package_type = '{ctx_name.r}'", - f"dfn_file_name = '{ctx_name.l}-{ctx_name.r}.dfn'" - if ctx_name.l == "exg" - else f"dfn_file_name = '{ctx_name.l or 'sim'}-{ctx_name.r}.dfn'", - f"dfn = {pformat(_dfn(), indent=10)}", - ] - ) - - return attrs - - -def _init_body(ctx: dict) -> List[str]: - def _statements() -> Optional[List[str]]: - base = ctx["name"].base - if base == "MFSimulationBase": - - def _should_set(var: dict) -> bool: - return var["name"] not in [ - "tdis6", - "models", - "exchanges", - "mxiter", - "solutiongroup", - "hpc_data", - ] - - stmts = [] - refs = {} - for var in ctx["vars"].values(): - ref = var.get("meta", dict()).get("ref", None) - if not var.get("kind", None): - continue - - name = var["name"] - if name in kwlist: - name = f"{name}_" - - if _should_set(var): - stmts.append(f"self.name_file.{name}.set_data({name})") - stmts.append(f"self.{name} = self.name_file.{name}") - if ref and ref["key"] not in refs: - refs[ref["key"]] = ref - stmts.append( - f"self.{ref['param']} = self._create_package('{ref['abbr']}', {ref['param']})" - ) - elif base == "MFModel": - - def _should_set(var: dict) -> bool: - return var["name"] not in [ - "export_netcdf", - "nc_filerecord", - "packages", - ] - - stmts = [] - refs = {} - for var in ctx["vars"].values(): - ref = var.get("meta", dict()).get("ref", None) - if not var.get("kind", None): - continue - - name = var["name"] - if name in kwlist: - name = f"{name}_" - - if _should_set(var): - stmts.append(f"self.name_file.{name}.set_data({name})") - stmts.append(f"self.{name} = self.name_file.{name}") - if ref and ref["key"] not in refs: - refs[ref["key"]] = ref - stmts.append( - f"self.{ref['param']} = self._create_package('{ref['abbr']}', {ref['param']})" - ) - elif base == "MFPackage": - - def _should_build(var: dict) -> bool: - if var.get("meta", dict()).get("ref", None) and ctx[ - "name" - ] != ( - None, - "nam", - ): - return False - name = var["name"] - if name in [ - "simulation", - "model", - "package", - "parent_model", - "parent_package", - "parent_model_or_package", - "parent_file", - "modelname", - "model_nam_file", - "export_netcdf", - "nc_filerecord", - "method", - "interpolation_method_single", - "sfac", - "output", - ]: - return False - return True - - stmts = [] - refs = {} - for var in ctx["vars"].values(): - name = var["name"] - ref = var.get("meta", dict()).get("ref", None) - if name in kwlist: - name = f"{name}_" - - if _should_build(var): - if ref and ctx["name"] == (None, "nam"): - stmts.append( - f"self.{'_' if ref else ''}{ref['key']} = self.build_mfdata('{ref['key']}', None)" - ) - else: - # hack... - _name = name[:-1] if name.endswith("_") else name - if _name == "steady_state": - _name = "steady-state" - stmts.append( - f"self.{'_' if ref else ''}{name} = self.build_mfdata('{_name}', {name if var.get('init_param', True) else 'None'})" - ) - - if ref and ref["key"] not in refs and ctx["name"].r != "nam": - refs[ref["key"]] = ref - stmts.append( - f"self._{ref['key']} = self.build_mfdata('{ref['key']}', None)" - ) - stmts.append( - f"self._{ref['abbr']}_package = self.build_child_package('{ref['abbr']}', {ref['val']}, '{ref['param']}', self._{ref['key']})" - ) - - return stmts - - return list(filter(None, _statements())) - - -def _init_skip(ctx: dict) -> List[str]: - name = ctx["name"] - base = name.base - if base == "MFSimulationBase": - skip = [ - "tdis6", - "models", - "exchanges", - "mxiter", - "solutiongroup", - ] - refs = ctx.get("meta", dict()).get("refs", dict()) - return skip - elif base == "MFModel": - skip = ["packages", "export_netcdf", "nc_filerecord"] - refs = ctx.get("meta", dict()).get("refs", dict()) - if any(refs) and ctx["name"] != (None, "nam"): - for key in refs.keys(): - if ctx["vars"].get(key, None): - skip.append(key) - return skip - elif base == "MFPackage": - if name.r == "nam": - return ["export_netcdf", "nc_filerecord"] - elif name == ("utl", "ts"): - return ["method", "interpolation_method_single", "sfac"] - else: - return [] - def _is_context(o) -> bool: d = dict(o) return "name" in d and "vars" in d -def _parent(ctx: dict) -> str: - ref = ctx["meta"].get("ref", None) - if ref: - return ref["parent"] - name = ctx["name"] - ref = ctx["meta"].get("ref", None) - if name == ("sim", "nam"): - return None - elif name.l is None or name.r is None or name.l in ["sim", "exg", "sln"]: - return "simulation" - elif ref: - if name.l == "utl" and name.r == "hpc": - return "simulation" - return "package" - return "model" - - def _replace_refs(ctx: dict, name_param: str = "val") -> dict: refs = ctx.get("meta", dict()).get("refs", dict()) if any(refs): @@ -341,16 +49,5 @@ def _transform_context(o): SHIM = { "keep_none": ["default", "vars"], "quote_str": ["default"], - "set_pairs": [ - ( - _is_context, - [ - ("cls_attrs", _cls_attrs), - ("init_skip", _init_skip), - ("init_body", _init_body), - ("parent", _parent), - ], - ), - ], "transform": [(_is_context, _transform_context)], } diff --git a/flopy/mf6/utils/codegen/templates/__init__.py.jinja b/flopy/mf6/utils/codegen/templates/__init__.py.jinja new file mode 100644 index 0000000000..1db6d034d6 --- /dev/null +++ b/flopy/mf6/utils/codegen/templates/__init__.py.jinja @@ -0,0 +1,3 @@ +{% for ctx in contexts %} +from .mf{{ ctx.name.title }} import {{ ctx.name|prefix }}{{ ctx.name.title.title() }} +{% endfor %} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/exchange.py.jinja b/flopy/mf6/utils/codegen/templates/exchange.py.jinja index a5d4a1e98f..fb52a2d498 100644 --- a/flopy/mf6/utils/codegen/templates/exchange.py.jinja +++ b/flopy/mf6/utils/codegen/templates/exchange.py.jinja @@ -26,9 +26,7 @@ class Modflow{{ name.title.title() }}(MFPackage): exgtype="{{ name.r[:3].upper() }}6-{{ name.r[3:].upper() }}6", exgmnamea=None, exgmnameb=None, - {%- for n, var in vars.items() if n not in init_skip %} - {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, - {%- endfor %} + {{ macros.init_vars(vars, skip=name|skip) }} filename=None, pname=None, **kwargs, @@ -69,7 +67,7 @@ class Modflow{{ name.title.title() }}(MFPackage): """ super().__init__( - {{ parent }}, + {{ name|parent }}, "{{ name.r }}", filename, pname, diff --git a/flopy/mf6/utils/codegen/templates/macros.jinja b/flopy/mf6/utils/codegen/templates/macros.jinja index 153ba15d9e..3befc515b1 100644 --- a/flopy/mf6/utils/codegen/templates/macros.jinja +++ b/flopy/mf6/utils/codegen/templates/macros.jinja @@ -1,3 +1,9 @@ +{% macro init_vars(vars, alias=false, skip=none) %} +{%- for n, var in vars.items() if n not in skip %} +{% if alias %}{{ n }}{% else %}{{ var.name }}{% endif %}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, +{%- endfor %} +{% endmacro %} + {% macro vars_docs(vars, start_indent=0) %} {%- for v in vars.values() recursive %} {{ ""|indent(start_indent, first=true) }}{% if loop.depth > 1 %}* {% endif %}{% if v.name|last == "_" %}{{ v.name.replace("_", "\\\_") }}{% else %}{{ v.name }}{% endif %}{% if v.type is defined %} : {{ v.type }}{% endif %} diff --git a/flopy/mf6/utils/codegen/templates/model.py.jinja b/flopy/mf6/utils/codegen/templates/model.py.jinja index a922bad9e0..079485debb 100644 --- a/flopy/mf6/utils/codegen/templates/model.py.jinja +++ b/flopy/mf6/utils/codegen/templates/model.py.jinja @@ -33,9 +33,7 @@ class Modflow{{ name.title.title() }}(MFModel): version="mf6", exe_name="mf6", model_rel_path=".", - {%- for n, var in vars.items() if n not in init_skip %} - {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, - {%- endfor %} + {{ macros.init_vars(vars, skip=name|skip) }} **kwargs, ): """ diff --git a/flopy/mf6/utils/codegen/templates/package.py.jinja b/flopy/mf6/utils/codegen/templates/package.py.jinja index 576ba79a78..cbc86319e5 100644 --- a/flopy/mf6/utils/codegen/templates/package.py.jinja +++ b/flopy/mf6/utils/codegen/templates/package.py.jinja @@ -22,11 +22,9 @@ class Modflow{{ name.title.title() }}(MFPackage): def __init__( self, - {{ parent }}, + {{ name|parent }}, loading_package=False, - {%- for n, var in vars.items() if n not in init_skip %} - {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, - {%- endfor %} + {{ macros.init_vars(vars, skip=name|skip) }} filename=None, pname=None, **kwargs, @@ -36,7 +34,7 @@ class Modflow{{ name.title.title() }}(MFPackage): Parameters ---------- - {{ parent }} : {{ name.parent }} + {{ name|parent }} Parent_package that this package is a part of. Package is automatically added to parent_package when it is initialized. @@ -59,7 +57,7 @@ class Modflow{{ name.title.title() }}(MFPackage): """ super().__init__( - {{ parent }}, + {{ name|parent }}, "{{ name.r }}", filename, pname, @@ -95,15 +93,13 @@ class {{ name.title.title() }}Packages(MFChildPackages): def initialize( self, - {%- for n, var in vars.items() if n not in init_skip %} - {{ n }}{%- if var.default is defined %}={{ var.default }}{% endif -%}, - {%- endfor %} + {{ macros.init_vars(vars, alias=true, skip=name|skip) }} filename=None, pname=None, ): new_package = Modflow{{ name.title.title() }}( self._cpparent, - {%- for n, var in vars.items() if n not in init_skip %} + {%- for n, var in vars.items() if n not in name|skip %} {{ n }}={{ n }}, {%- endfor %} filename=filename, @@ -115,15 +111,13 @@ class {{ name.title.title() }}Packages(MFChildPackages): {% if name.r != "obs" %} def append_package( self, - {%- for n, var in vars.items() if n not in init_skip %} - {{ n }}{%- if var.default is defined %}={{ var.default }}{% endif -%}, - {%- endfor %} + {{ macros.init_vars(vars, alias=true, skip=name|skip) }} filename=None, pname=None, ): new_package = Modflow{{ name.title.title() }}( self._cpparent, - {%- for n, var in vars.items() if n not in init_skip %} + {%- for n, var in vars.items() if n not in name|skip %} {{ n }}={{ n }}, {%- endfor %} filename=filename, diff --git a/flopy/mf6/utils/codegen/templates/simulation.py.jinja b/flopy/mf6/utils/codegen/templates/simulation.py.jinja index 39f807574e..76c15057bc 100644 --- a/flopy/mf6/utils/codegen/templates/simulation.py.jinja +++ b/flopy/mf6/utils/codegen/templates/simulation.py.jinja @@ -34,9 +34,7 @@ class MF{{ name.title.title() }}(MFSimulationBase): write_headers: bool = True, use_pandas: bool = True, lazy_io: bool = False, - {%- for n, var in vars.items() if n not in init_skip %} - {{ var.name }}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, - {%- endfor %} + {{ macros.init_vars(vars, skip=name|skip) }} ): """ {{ name.description }} diff --git a/flopy/mf6/utils/codegen/utils.py b/flopy/mf6/utils/codegen/utils.py index 556f96b399..8fa83b2e63 100644 --- a/flopy/mf6/utils/codegen/utils.py +++ b/flopy/mf6/utils/codegen/utils.py @@ -2,33 +2,3 @@ from enum import Enum from typing import Any - -def try_get_enum_value(v: Any) -> Any: - """ - Get the enum's value if the object is an instance - of an enumeration, otherwise return it unaltered. - """ - return v.value if isinstance(v, Enum) else v - - -def try_literal_eval(value: str) -> Any: - """ - Try to parse a string as a literal. If this fails, - return the value unaltered. - """ - try: - return literal_eval(value) - except (SyntaxError, ValueError): - return value - - -def try_parse_bool(value: Any) -> Any: - """ - Try to parse a boolean from a string as represented - in a DFN file, otherwise return the value unaltered. - """ - if isinstance(value, str): - value = value.lower() - if value in ["true", "false"]: - return value == "true" - return value diff --git a/pyproject.toml b/pyproject.toml index 1c023e39d9..db4ccfcc3a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,8 @@ dev = ["flopy[gen,lint,test,optional,doc]", "tach"] codegen = [ "boltons>=1.0", "Jinja2>=3.0", - "modflow-devtools" + "modflow-devtools", + "tomlkit", ] lint = ["cffconvert", "codespell[toml] >=2.2.2", "ruff"] test = [ From 700b9606cace0712b1faaf1ede7ba2405194d483 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Mon, 4 Nov 2024 15:32:48 -0500 Subject: [PATCH 56/75] ruff --- flopy/mf6/utils/codegen/jinja.py | 20 ++++++++++++++++---- flopy/mf6/utils/codegen/utils.py | 1 - 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/flopy/mf6/utils/codegen/jinja.py b/flopy/mf6/utils/codegen/jinja.py index 58a57260ed..a6ac4ad363 100644 --- a/flopy/mf6/utils/codegen/jinja.py +++ b/flopy/mf6/utils/codegen/jinja.py @@ -2,6 +2,7 @@ from keyword import kwlist from pprint import pformat from typing import Any, List, Optional + from jinja2 import pass_context @@ -14,7 +15,6 @@ def try_get_enum_value(v: Any) -> Any: class Filters: - @pass_context def cls_attrs(ctx, ctx_name) -> List[str]: def _attr(var: dict) -> Optional[str]: @@ -58,7 +58,11 @@ def _attr(var: dict) -> Optional[str]: return f"{var_ref['key']} = ListTemplateGenerator(({', '.join(args)}))" def _args(): - args = [f"'{ctx_name.r}'", f"'{var_block}'", f"'{var_name}'"] + args = [ + f"'{ctx_name.r}'", + f"'{var_block}'", + f"'{var_name}'", + ] if ctx_name.l is not None and ctx_name.l not in [ "sim", "sln", @@ -239,7 +243,11 @@ def _should_build(var: dict) -> bool: f"self.{'_' if ref else ''}{name} = self.build_mfdata('{_name}', {name if var.get('init_param', True) else 'None'})" ) - if ref and ref["key"] not in refs and ctx["name"].r != "nam": + if ( + ref + and ref["key"] not in refs + and ctx["name"].r != "nam" + ): refs[ref["key"]] = ref stmts.append( f"self._{ref['key']} = self.build_mfdata('{ref['key']}', None)" @@ -259,7 +267,11 @@ def parent(ctx, ctx_name): return ref["parent"] if ctx_name == ("sim", "nam"): return None - elif ctx_name.l is None or ctx_name.r is None or ctx_name.l in ["sim", "exg", "sln"]: + elif ( + ctx_name.l is None + or ctx_name.r is None + or ctx_name.l in ["sim", "exg", "sln"] + ): return "simulation" elif ref: if ctx_name.l == "utl" and ctx_name.r == "hpc": diff --git a/flopy/mf6/utils/codegen/utils.py b/flopy/mf6/utils/codegen/utils.py index 8fa83b2e63..e663835be6 100644 --- a/flopy/mf6/utils/codegen/utils.py +++ b/flopy/mf6/utils/codegen/utils.py @@ -1,4 +1,3 @@ from ast import literal_eval from enum import Enum from typing import Any - From 72829155fd6e063c0b0e11369739f4b06757386a Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Mon, 4 Nov 2024 21:51:08 -0500 Subject: [PATCH 57/75] remove shim and renderable, much cleanup --- autotest/test_codegen.py | 3 +- flopy/mf6/utils/codegen/__init__.py | 82 ++- flopy/mf6/utils/codegen/context.py | 67 -- flopy/mf6/utils/codegen/dfn.py | 268 ++++---- flopy/mf6/utils/codegen/dfn2toml.py | 13 +- flopy/mf6/utils/codegen/jinja.py | 619 ++++++++++-------- flopy/mf6/utils/codegen/renderable.py | 129 ---- flopy/mf6/utils/codegen/shim.py | 53 -- .../utils/codegen/templates/__init__.py.jinja | 2 +- .../utils/codegen/templates/exchange.py.jinja | 17 +- .../mf6/utils/codegen/templates/macros.jinja | 14 +- .../utils/codegen/templates/model.py.jinja | 21 +- .../utils/codegen/templates/package.py.jinja | 37 +- .../codegen/templates/simulation.py.jinja | 15 +- flopy/mf6/utils/codegen/utils.py | 3 - 15 files changed, 602 insertions(+), 741 deletions(-) delete mode 100644 flopy/mf6/utils/codegen/renderable.py delete mode 100644 flopy/mf6/utils/codegen/shim.py delete mode 100644 flopy/mf6/utils/codegen/utils.py diff --git a/autotest/test_codegen.py b/autotest/test_codegen.py index 7d5b58df7e..34ab9d814c 100644 --- a/autotest/test_codegen.py +++ b/autotest/test_codegen.py @@ -4,6 +4,7 @@ from flopy.mf6.utils.codegen import make_all, make_targets from flopy.mf6.utils.codegen.context import Context from flopy.mf6.utils.codegen.dfn import Dfn +from flopy.mf6.utils.codegen.jinja import Filters PROJ_ROOT = get_project_root_path() MF6_PATH = PROJ_ROOT / "flopy" / "mf6" @@ -38,7 +39,7 @@ def test_make_targets(dfn_name, function_tmpdir): make_targets(dfn, function_tmpdir, verbose=True) assert all( - (function_tmpdir / name.target).is_file() + (function_tmpdir / f"mf{Filters.Cls.title(name)}.py").is_file() for name in Context.Name.from_dfn(dfn) ) diff --git a/flopy/mf6/utils/codegen/__init__.py b/flopy/mf6/utils/codegen/__init__.py index 0439f68c0b..f90df170fd 100644 --- a/flopy/mf6/utils/codegen/__init__.py +++ b/flopy/mf6/utils/codegen/__init__.py @@ -1,61 +1,93 @@ +from dataclasses import asdict +from itertools import chain from os import PathLike from pathlib import Path from flopy.utils import import_optional_dependency -__all__ = ["make_targets", "make_all"] +__all__ = ["make_init", "make_targets", "make_all"] def _get_template_env(): - from flopy.mf6.utils.codegen.jinja import Filters - + # import here instead of module so we don't + # expect optional deps at module init time jinja = import_optional_dependency("jinja2") loader = jinja.PackageLoader("flopy", "mf6/utils/codegen/templates/") env = jinja.Environment(loader=loader) - env.filters["parent"] = Filters.parent - env.filters["prefix"] = Filters.prefix - env.filters["skip"] = Filters.skip + + from flopy.mf6.utils.codegen.jinja import Filters + + env.filters["base"] = Filters.Cls.base + env.filters["title"] = Filters.Cls.title + env.filters["description"] = Filters.Cls.description + env.filters["prefix"] = Filters.Cls.prefix + env.filters["parent"] = Filters.Cls.parent + env.filters["skip"] = Filters.Cls.skip + + env.filters["attrs"] = Filters.Vars.attrs + env.filters["init"] = Filters.Vars.init + + env.filters["type"] = Filters.Var.type + + env.filters["nokw"] = Filters.nokw + env.filters["escape_trailing"] = Filters.escape_trailing + env.filters["value"] = Filters.value + return env def make_init(dfns: dict, outdir: PathLike, verbose: bool = False): """Generate a Python __init__.py file for the given input definitions.""" - from flopy.mf6.utils.codegen.context import Context - env = _get_template_env() outdir = Path(outdir).expanduser() - contexts = [ - c - for cc in [ - [ctx for ctx in Context.from_dfn(dfn)] for dfn in dfns.values() - ] - for c in cc - ] # ugly, but it's the fastest way to flatten the list + + from flopy.mf6.utils.codegen.context import Context + + contexts = list( + chain( + *[[ctx for ctx in Context.from_dfn(dfn)] for dfn in dfns.values()] + ) + ) target_name = "__init__.py" - target = outdir / target_name + target_path = outdir / target_name template = env.get_template(f"{target_name}.jinja") - with open(target, "w") as f: + with open(target_path, "w") as f: f.write(template.render(contexts=contexts)) if verbose: - print(f"Wrote {target}") + print(f"Wrote {target_path}") def make_targets(dfn, outdir: PathLike, verbose: bool = False): """Generate Python source file(s) from the given input definition.""" - from flopy.mf6.utils.codegen.context import Context - env = _get_template_env() outdir = Path(outdir).expanduser() + + from flopy.mf6.utils.codegen.context import Context + from flopy.mf6.utils.codegen.jinja import Filters + + def _get_template_name(ctx_name) -> str: + """The template file to use.""" + base = Filters.Cls.base(ctx_name) + if base == "MFSimulationBase": + return "simulation.py.jinja" + elif base == "MFModel": + return "model.py.jinja" + elif base == "MFPackage": + if ctx_name.l == "exg": + return "exchange.py.jinja" + return "package.py.jinja" + for context in Context.from_dfn(dfn): name = context.name - target = outdir / name.target - template = env.get_template(name.template) - with open(target, "w") as f: - f.write(template.render(**context.render())) + target_path = outdir / f"mf{Filters.Cls.title(name)}.py" + template_name = _get_template_name(name) + template = env.get_template(template_name) + with open(target_path, "w") as f: + f.write(template.render(**asdict(context))) if verbose: - print(f"Wrote {target}") + print(f"Wrote {target_path}") def make_all(dfndir: Path, outdir: Path, verbose: bool = False): diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py index 34c3201237..0c30cecb00 100644 --- a/flopy/mf6/utils/codegen/context.py +++ b/flopy/mf6/utils/codegen/context.py @@ -9,11 +9,8 @@ ) from flopy.mf6.utils.codegen.dfn import Dfn, Ref, Vars -from flopy.mf6.utils.codegen.renderable import renderable -from flopy.mf6.utils.codegen.shim import SHIM -@renderable(**SHIM) @dataclass class Context: """ @@ -62,70 +59,6 @@ class Name(NamedTuple): l: str r: Optional[str] - @property - def title(self) -> str: - """ - The input context's unique title. This is not - identical to `f"{l}{r}` in some cases, but it - remains unique. The title is substituted into - the file name and class name. - """ - l, r = self - if self == ("sim", "nam"): - return "simulation" - if l is None: - return r - if r is None: - return l - if l == "sim": - return r - if l in ["sln", "exg"]: - return r - return l + r - - @property - def base(self) -> str: - """Base class from which the input context should inherit.""" - _, r = self - if self == ("sim", "nam"): - return "MFSimulationBase" - if r is None: - return "MFModel" - return "MFPackage" - - @property - def target(self) -> str: - """The source file name to generate.""" - return f"mf{self.title}.py" - - @property - def template(self) -> str: - """The template file to use.""" - if self.base == "MFSimulationBase": - return "simulation.py.jinja" - elif self.base == "MFModel": - return "model.py.jinja" - elif self.base == "MFPackage": - if self.l == "exg": - return "exchange.py.jinja" - return "package.py.jinja" - - @property - def description(self) -> str: - """A description of the input context.""" - l, r = self - title = self.title.title() - if self.base == "MFPackage": - return f"Modflow{title} defines a {r.upper()} package." - elif self.base == "MFModel": - return f"Modflow{title} defines a {l.upper()} model." - elif self.base == "MFSimulationBase": - return ( - "MFSimulation is used to load, build, and/or save a MODFLOW 6 simulation." - " A MFSimulation object must be created before creating any of the MODFLOW" - " 6 model objects." - ) - @staticmethod def from_dfn(dfn: Dfn) -> List["Context.Name"]: """ diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index cf6aac5df0..577ab4d91d 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -16,9 +16,6 @@ from boltons.dictutils import OMD -from flopy.mf6.utils.codegen.renderable import renderable -from flopy.mf6.utils.codegen.utils import try_literal_eval, try_parse_bool - _SCALARS = { "keyword", "integer", @@ -27,7 +24,7 @@ } -def try_literal_eval(value: str) -> Any: +def _try_literal_eval(value: str) -> Any: """ Try to parse a string as a literal. If this fails, return the value unaltered. @@ -38,7 +35,7 @@ def try_literal_eval(value: str) -> Any: return value -def try_parse_bool(value: Any) -> Any: +def _try_parse_bool(value: Any) -> Any: """ Try to parse a boolean from a string as represented in a DFN file, otherwise return the value unaltered. @@ -58,22 +55,11 @@ def try_parse_bool(value: Any) -> Any: class Var(TypedDict): """MODFLOW 6 input variable specification.""" - class Kind(Enum): - """ - An input variable's kind. This is an enumeration - of the general shapes of data MODFLOW 6 accepts. - """ - - Array = "array" - Scalar = "scalar" - Record = "record" - Union = "union" - List = "list" - name: str - kind: Kind type: Optional[str] = None + shape: Optional[Any] = None block: Optional[str] = None + fkey: Optional["Ref"] = None default: Optional[Any] = None children: Optional[Vars] = None description: Optional[str] = None @@ -107,11 +93,6 @@ def from_dfn(cls, dfn: "Dfn") -> Optional["Ref"]: referenced by other contexts. """ - # TODO: all this won't be necessary once we - # structure DFN format; we can then support - # subpackage references directly instead of - # by making assumptions about `dfn.meta` - if not dfn.meta or "dfn" not in dfn.meta: return None @@ -168,7 +149,6 @@ def _parent(): ) -@renderable class Dfn(UserDict): """ MODFLOW 6 input definition. An input definition @@ -192,6 +172,9 @@ def parse(cls, v: str) -> "Dfn.Name": except: raise ValueError(f"Bad DFN name format: {v}") + def __str__(self) -> str: + return "-".join(self) + name: Optional[Name] meta: Optional[Dict[str, Any]] @@ -208,14 +191,10 @@ def __init__( @staticmethod def _load(f, common: Optional[dict] = None) -> Tuple[OMD, List[str]]: """ - Internal use only. Loads the DFN as an ordered multi-dictionary* and - a list of string metadata. This is later parsed into more structured - form. We also store the original representation for now so it can be - used by the shim. + Internal use only. Loads the DFN as a flat multi-dictionary* with a + list of string metadata, which are then parsed into structured form. - *The point of the OMD is to handle duplicate variable names; the only - case of this right now is 'auxiliary' which can appear in the options - block and again as a keyword in a record in a package data variable. + *The point of this is to losslessly handle duplicate variable names. """ var = dict() @@ -304,11 +283,18 @@ def load( refs: Optional[Dfns] = None, **kwargs, ) -> "Dfn": - """Load an input definition.""" + """ + Load an input definition from a DFN file. + + Notes + ----- + Loads the DFN as a flat multidict with `_load()` + then walks composite variables and builds a tree. + """ - refs = refs or dict() - referenced = dict() flat, meta = Dfn._load(f, **kwargs) + refs = refs or dict() + fkeys = dict() def _map(spec: Dict[str, Any]) -> Var: """ @@ -317,16 +303,11 @@ def _map(spec: Dict[str, Any]) -> Var: Notes ----- - This involves expanding nested type hierarchies, mapping - types to roughly equivalent Python primitives/composites. - The composite inflation step will not be necessary after - DFNs move to a structured format. - If a variable does not have a `default` attribute, it will default to `False` if it is a keyword, otherwise to `None`. - Any filepath variable whose name functions as a foreign key - for another context will be given a pointer to the context. + A filepath variable whose name functions as a foreign key + for a separate context will be given a reference to it. """ @@ -334,7 +315,7 @@ def _map(spec: Dict[str, Any]) -> Var: # stay a string except default values, which we'll # try to parse as arbitrary literals below, and at # some point types, once we introduce type hinting - spec = {k: try_parse_bool(v) for k, v in spec.items()} + spec = {k: _try_parse_bool(v) for k, v in spec.items()} _name = spec["name"] _type = spec.get("type", None) @@ -342,18 +323,29 @@ def _map(spec: Dict[str, Any]) -> Var: shape = spec.get("shape", None) shape = None if shape == "" else shape default = spec.get("default", None) + default = ( + _try_literal_eval(default) if _type != "string" else default + ) description = spec.get("description", "") + tagged = spec.get("tagged", False) children = dict() - # if var is a foreign key, register the reference - ref = refs.get(_name, None) - if ref: - referenced[_name] = ref + # if var is a foreign key, register it + fkey = refs.get(_name, None) + if fkey: + fkeys[_name] = fkey + + def _choices() -> Vars: + """Load a union's children (choices).""" + names = _type.split()[1:] + return { + v["name"]: _map(v) + for v in flat.values(multi=True) + if v["name"] in names and v.get("in_record", False) + } - def _fields(record_name: str) -> Vars: - """Recursively load/convert a record's fields.""" - record = next(iter(flat.getlist(record_name)), None) - assert record + def _fields() -> Vars: + """Load a record's children (fields).""" names = _type.split()[1:] fields = { v["name"]: _map(v) @@ -375,14 +367,8 @@ def _fields(record_name: str) -> Vars: if keyword: fields.pop(keyword) - # set the type - n = list(fields.keys())[0] - path_field = fields[n] - path_field["kind"] = Var.Kind.Scalar - fields[n] = path_field - # if tagged, remove the leading keyword - elif record.get("tagged", False): + elif tagged: keyword = next(iter(fields), None) if keyword: fields.pop(keyword) @@ -391,52 +377,48 @@ def _fields(record_name: str) -> Vars: # list, child is the item type if _type.startswith("recarray"): - # make sure columns are defined names = _type.split()[1:] + types = [ + v["type"] + for v in flat.values(multi=True) + if v["name"] in names and v.get("in_record", False) + ] + n_names = len(names) if n_names < 1: raise ValueError(f"Missing recarray definition: {_type}") - # list input can have records or unions as rows. - # lists which have a consistent record type are - # regular, inconsistent record types irregular. - - # regular tabular/columnar data (1 record type) can be - # defined with a nested record (i.e. explicit) or with - # fields directly inside the recarray (implicit). list - # data for unions/keystrings necessarily comes nested. - - is_explicit_record = n_names == 1 and flat[names[0]][ - "type" - ].startswith("record") - - def _is_implicit_scalar_record(): - # if the record is defined implicitly and it has - # only scalar fields - types = [ - v["type"] - for v in flat.values(multi=True) - if v["name"] in names and v.get("in_record", False) - ] - return all(t in _SCALARS for t in types) - - if is_explicit_record: - record = next(iter(flat.getlist(names[0])), None) - children = {names[0]: _map(record)} - kind = Var.Kind.List - elif _is_implicit_scalar_record(): - fields = _fields(_name) + # list input can have records or unions as rows. lists + # that have a consistent item type can be considered + # tabular. lists that can possess multiple item types + # (unions) are considered irregular. regular lists can + # be defined with a nested record (explicit) or with a + # set of fields directly in the recarray (implicit). an + # irregular list is always defined with a nested union. + is_explicit_composite = n_names == 1 and ( + types[0].startswith("record") + or types[0].startswith("keystring") + ) + is_implicit_scalar_record = all(t in _SCALARS for t in types) + + if is_explicit_composite: + child = next(iter(flat.getlist(names[0]))) + children = {names[0]: _map(child)} + _type = "list" + elif is_implicit_scalar_record: + fields = _fields() children = { _name: Var( name=_name, - kind=Var.Kind.Record, + type="record", block=block, children=fields, - description=description, - type=f"[{', '.join([f['type'] for f in fields.values()])}]", + description=description.replace( + "is the list of", "is the record of" + ), ) } - kind = Var.Kind.List + _type = "list" else: # implicit complex record (i.e. some fields are records or unions) fields = { @@ -447,75 +429,67 @@ def _is_implicit_scalar_record(): first = list(fields.values())[0] single = len(fields) == 1 name_ = first["name"] if single else _name + child_type = ( + "union" + if single and "keystring" in first["type"] + else "record" + ) children = { name_: Var( name=name_, - kind=Var.Kind.Record, + type=child_type, block=block, children=first["children"] if single else fields, - description=description, - type=f"[{', '.join([v['type'] for v in fields.values()])}]", + description=description.replace( + "is the list of", f"is the {child_type} of" + ), ) } - kind = Var.Kind.List - type_ = ( - f"[{', '.join([v['name'] for v in children.values()])}]" - ) + _type = "list" - # union (product), children are choices + # union (product) type elif _type.startswith("keystring"): - names = _type.split()[1:] - children = { - v["name"]: _map(v) - for v in flat.values(multi=True) - if v["name"] in names and v.get("in_record", False) - } - kind = Var.Kind.Union - type_ = ( - f"[{', '.join([v['name'] for v in children.values()])}]" - ) + children = _choices() + _type = "union" - # record (sum), children are fields + # record (sum) type elif _type.startswith("record"): - children = _fields(_name) - kind = Var.Kind.Record - type_ = ( - f"[{', '.join([v['type'] for v in children.values()])}]" - ) + children = _fields() + _type = "record" - # at this point, if it has a shape, it's an array + # at this point, if it has a shape, it's an array. check its type elif shape is not None: if _type not in _SCALARS: raise TypeError(f"Unsupported array type: {_type}") - elif _type == "string": - kind = Var.Kind.List - else: - kind = Var.Kind.Array - type_ = f"[{_type}]" - - # finally scalars - else: - kind = Var.Kind.Scalar - type_ = _type - # create var + # if the var is a foreign key, swap in the referenced variable + ref = refs.get(_name, None) + if not ref: + return Var( + name=_name, + type=_type, + shape=shape, + block=block, + description=description, + default=default, + children=children, + meta={"ref": fkey}, + ) return Var( - # if name is a reserved keyword, add a trailing underscore to it. - # convert dashes to underscores since it may become a class attr. - name=(f"{_name}_" if _name in kwlist else _name).replace( - "-", "_" - ), - kind=kind, - type=type_, + name=ref["param" if name == ("sim", "nam") else "val"], + type=_type, + shape=shape, block=block, - description=description, - default=( - try_literal_eval(default) if _type != "string" else default + description=( + f"* Contains data for the {ref['abbr']} package. Data can be " + f"stored in a dictionary containing data for the {ref['abbr']} " + "package with variable names as keys and package data as " + f"values. Data just for the {ref['val']} variable is also " + f"acceptable. See {ref['abbr']} package documentation for more " + "information" ), - children=children, - # type is a string for now, when - # introducing type hints make it - # a proper type... + default=None, + children=None, meta={"ref": ref}, ) @@ -526,29 +500,25 @@ def _is_implicit_scalar_record(): # convert input variable specs to # structured form, descending into # composites recursively as needed - flat = { + vars_ = { var["name"]: _map(var) for var in flat.values(multi=True) if not var.get("in_record", False) } - # reset the var name. we may have altered - # it when converting the variable e.g. to - # avoid collision with a reserved keyword - flat = {v["name"]: v for v in flat.values()} - return cls( - flat, + vars_, name, { "dfn": (_vars, meta), - "refs": referenced, + "refs": fkeys, }, ) @staticmethod def load_all(dfndir: PathLike) -> Dict[str, "Dfn"]: """Load all input definitions from the given directory.""" + # find definition files paths = [ p diff --git a/flopy/mf6/utils/codegen/dfn2toml.py b/flopy/mf6/utils/codegen/dfn2toml.py index 771be3509e..198e61d9de 100644 --- a/flopy/mf6/utils/codegen/dfn2toml.py +++ b/flopy/mf6/utils/codegen/dfn2toml.py @@ -1,4 +1,5 @@ import argparse +from collections.abc import Mapping from pathlib import Path from flopy.utils import import_optional_dependency @@ -8,6 +9,14 @@ _TOML_PATH = _MF6_PATH / "data" / "toml" +def _drop_none(d: dict) -> dict: + return ( + {k: _drop_none(v) for k, v in d.items() if v is not None} + if isinstance(d, Mapping) + else d + ) + + if __name__ == "__main__": """Convert DFN files to TOML.""" @@ -31,5 +40,5 @@ outdir = Path(args.outdir) outdir.mkdir(exist_ok=True, parents=True) for dfn in Dfn.load_all(dfndir).values(): - with open(Path(outdir) / f"{'-'.join(dfn.name)}.toml", "w") as f: - tomlkit.dump(dfn.render(), f) + with open(Path(outdir) / f"{dfn.name}.toml", "w") as f: + tomlkit.dump(_drop_none(dfn), f) diff --git a/flopy/mf6/utils/codegen/jinja.py b/flopy/mf6/utils/codegen/jinja.py index a6ac4ad363..cb0b3f764f 100644 --- a/flopy/mf6/utils/codegen/jinja.py +++ b/flopy/mf6/utils/codegen/jinja.py @@ -5,6 +5,8 @@ from jinja2 import pass_context +from flopy.mf6.utils.codegen.dfn import _SCALARS + def try_get_enum_value(v: Any) -> Any: """ @@ -15,38 +17,178 @@ def try_get_enum_value(v: Any) -> Any: class Filters: - @pass_context - def cls_attrs(ctx, ctx_name) -> List[str]: - def _attr(var: dict) -> Optional[str]: - var_name = var["name"] - var_kind = try_get_enum_value(var.get("kind", None)) - var_block = var.get("block", None) - var_ref = var.get("meta", dict()).get("ref", None) - - if ( - var_kind is None - or var_kind == "scalar" - or var_name in ["cvoptions", "output"] - or (ctx_name.r == "dis" and var_name == "packagedata") - or ( - var_name != "packages" - and (ctx_name.l is not None and ctx_name.r == "nam") + class Cls: + def base(ctx_name) -> str: + """Base class from which the input context should inherit.""" + _, r = ctx_name + if ctx_name == ("sim", "nam"): + return "MFSimulationBase" + if r is None: + return "MFModel" + return "MFPackage" + + def title(ctx_name) -> str: + """ + The input context's unique title. This is not + identical to `f"{l}{r}` in some cases, but it + remains unique. The title is substituted into + the file name and class name. + """ + l, r = ctx_name + if (l, r) == ("sim", "nam"): + return "simulation" + if l is None: + return r + if r is None: + return l + if l == "sim": + return r + if l in ["sln", "exg"]: + return r + return l + r + + def description(ctx_name) -> str: + """A description of the input context.""" + l, r = ctx_name + base = Filters.Cls.base(ctx_name) + title = Filters.Cls.title(ctx_name).title() + if base == "MFPackage": + return f"Modflow{title} defines a {r.upper()} package." + elif base == "MFModel": + return f"Modflow{title} defines a {l.upper()} model." + elif base == "MFSimulationBase": + return ( + "MFSimulation is used to load, build, and/or save a MODFLOW 6 simulation." + " A MFSimulation object must be created before creating any of the MODFLOW" + " 6 model objects." ) - ): - return None - if var_kind in ["list", "record", "union", "array"]: - if not var_block: - raise ValueError("Need block") + def prefix(ctx_name) -> str: + base = Filters.Cls.base(ctx_name) + return "MF" if base == "MFSimulationBase" else "Modflow" - if var_kind != "array": - if var_ref: - # if the variable is a subpackage reference, use the original key - # (which has been replaced already with the referenced variable) + @pass_context + def parent(ctx, ctx_name) -> str: + ref = ctx["meta"].get("ref", None) + if ref: + return ref["parent"] + if ctx_name == ("sim", "nam"): + return None + elif ( + ctx_name.l is None + or ctx_name.r is None + or ctx_name.l in ["sim", "exg", "sln"] + ): + return "simulation" + elif ref: + if ctx_name.l == "utl" and ctx_name.r == "hpc": + return "simulation" + return "package" + return "model" + + @pass_context + def skip(ctx, ctx_name) -> List[str]: + base = Filters.Cls.base(ctx_name) + if base == "MFSimulationBase": + return [ + "tdis6", + "models", + "exchanges", + "mxiter", + "solutiongroup", + ] + elif base == "MFModel": + skip = ["packages", "export_netcdf", "nc_filerecord"] + refs = ctx.get("meta", dict()).get("refs", dict()) + if any(refs) and ctx["name"] != (None, "nam"): + for k in refs.keys(): + if ctx["vars"].get(k, None): + skip.append(k) + return skip + else: + if ctx_name.r == "nam": + return ["export_netcdf", "nc_filerecord"] + elif ctx_name == ("utl", "ts"): + return ["method", "interpolation_method_single", "sfac"] + return [] + + class Var: + def type(var: dict) -> str: + _type = var["type"] + shape = var.get("shape", None) + children = var.get("children", None) + if children: + if _type == "list": + children = ", ".join( + [v["name"] for v in children.values()] + ) + return f"[{children}]" + elif _type == "record": + children = ", ".join( + [v["name"] for v in children.values()] + ) + return f"({children})" + elif _type == "union": + return " | ".join([v["name"] for v in children.values()]) + if shape: + return f"[{_type}]" + return var["type"] + + class Vars: + @pass_context + def attrs(ctx, vars) -> List[str]: + ctx_name = ctx["name"] + base = Filters.Cls.base(ctx_name) + + def _attr(var: dict) -> Optional[str]: + var_name = var["name"] + var_type = var["type"] + var_shape = var.get("shape", None) + var_block = var.get("block", None) + var_ref = var.get("meta", dict()).get("ref", None) + + if ( + (var_type in _SCALARS and not var_shape) + or var_name in ["cvoptions", "output"] + or (ctx_name.r == "dis" and var_name == "packagedata") + or ( + var_name != "packages" + and (ctx_name.l is not None and ctx_name.r == "nam") + ) + ): + return None + + is_array = ( + var_type in ["integer", "double precision"] and var_shape + ) + is_composite = var_type in ["list", "record", "union"] + if is_array or is_composite: + if not var_block: + raise ValueError("Need block") + + if not is_array: + if var_ref: + # if the variable is a subpackage reference, use the original key + # (which has been replaced already with the referenced variable) + args = [ + f"'{ctx_name.r}'", + f"'{var_block}'", + f"'{var_ref['key']}'", + ] + if ctx_name.l is not None and ctx_name.l not in [ + "sim", + "sln", + "utl", + "exg", + ]: + args.insert(0, f"'{ctx_name.l}6'") + return f"{var_ref['key']} = ListTemplateGenerator(({', '.join(args)}))" + + def _args(): args = [ f"'{ctx_name.r}'", f"'{var_block}'", - f"'{var_ref['key']}'", + f"'{var_name}'", ] if ctx_name.l is not None and ctx_name.l not in [ "sim", @@ -55,255 +197,210 @@ def _attr(var: dict) -> Optional[str]: "exg", ]: args.insert(0, f"'{ctx_name.l}6'") - return f"{var_ref['key']} = ListTemplateGenerator(({', '.join(args)}))" + return args - def _args(): - args = [ - f"'{ctx_name.r}'", - f"'{var_block}'", - f"'{var_name}'", - ] - if ctx_name.l is not None and ctx_name.l not in [ - "sim", - "sln", - "utl", - "exg", - ]: - args.insert(0, f"'{ctx_name.l}6'") - return args - - kind = var_kind if var_kind == "array" else "list" - return f"{var_name} = {kind.title()}TemplateGenerator(({', '.join(_args())}))" - - return None - - def _dfn() -> List[List[str]]: - dfn, meta = ctx["meta"]["dfn"] - - def _meta(): - exclude = ["subpackage", "parent_name_type"] - return [v for v in meta if not any(p in v for p in exclude)] - - def _dfn(): - def _var(var: dict) -> List[str]: - exclude = ["longname", "description"] - name = var["name"] - var_ = ctx["vars"].get(name, None) - keys = [ - "construct_package", - "construct_data", - "parameter_name", - ] - if var_ and keys[0] in var_: - for k in keys: - var[k] = var_[k] + kind = "array" if is_array else "list" + return f"{var_name} = {kind.title()}TemplateGenerator(({', '.join(_args())}))" + + return None + + def _dfn() -> List[List[str]]: + dfn, meta = ctx["meta"]["dfn"] + + def _meta(): + exclude = ["subpackage", "parent_name_type"] return [ - " ".join([k, v]).strip() - for k, v in var.items() - if k not in exclude + v for v in meta if not any(p in v for p in exclude) ] - return [_var(var) for var in dfn] - - return [["header"] + _meta()] + _dfn() - - attrs = list(filter(None, [_attr(v) for v in ctx["vars"].values()])) - - if ctx_name.base == "MFModel": - attrs.append(f"model_type = {ctx_name.l}") - elif ctx_name.base == "MFPackage": - attrs.extend( - [ - f"package_abbr = '{ctx_name.r}'" - if ctx_name.l == "exg" - else f"package_abbr = '{'' if ctx_name.l in ['sln', 'sim', 'exg', None] else ctx_name.l}{ctx_name.r}'", - f"_package_type = '{ctx_name.r}'", - f"dfn_file_name = '{ctx_name.l}-{ctx_name.r}.dfn'" - if ctx_name.l == "exg" - else f"dfn_file_name = '{ctx_name.l or 'sim'}-{ctx_name.r}.dfn'", - f"dfn = {pformat(_dfn(), indent=10)}", - ] - ) + def _dfn(): + def _var(var: dict) -> List[str]: + exclude = ["longname", "description"] + name = var["name"] + var_ = vars.get(name, None) + keys = [ + "construct_package", + "construct_data", + "parameter_name", + ] + if var_ and keys[0] in var_: + for k in keys: + var[k] = var_[k] + return [ + " ".join([k, v]).strip() + for k, v in var.items() + if k not in exclude + ] - return attrs + return [_var(var) for var in dfn] - @pass_context - def init_body(ctx, ctx_name) -> List[str]: - def _statements() -> Optional[List[str]]: - base = ctx_name.base - if base == "MFSimulationBase": + return [["header"] + _meta()] + _dfn() + + attrs = list(filter(None, [_attr(v) for v in vars.values()])) - def _should_set(var: dict) -> bool: - return var["name"] not in [ - "tdis6", - "models", - "exchanges", - "mxiter", - "solutiongroup", - "hpc_data", + if base == "MFModel": + attrs.append(f"model_type = {ctx_name.l}") + elif base == "MFPackage": + attrs.extend( + [ + f"package_abbr = '{ctx_name.r}'" + if ctx_name.l == "exg" + else f"package_abbr = '{'' if ctx_name.l in ['sln', 'sim', 'exg', None] else ctx_name.l}{ctx_name.r}'", + f"_package_type = '{ctx_name.r}'", + f"dfn_file_name = '{ctx_name.l}-{ctx_name.r}.dfn'" + if ctx_name.l == "exg" + else f"dfn_file_name = '{ctx_name.l or 'sim'}-{ctx_name.r}.dfn'", + f"dfn = {pformat(_dfn(), indent=10)}", ] + ) - stmts = [] - refs = {} - for var in ctx["vars"].values(): - ref = var.get("meta", dict()).get("ref", None) - if not var.get("kind", None): - continue - - name = var["name"] - if name in kwlist: - name = f"{name}_" - - if _should_set(var): - stmts.append(f"self.name_file.{name}.set_data({name})") - stmts.append(f"self.{name} = self.name_file.{name}") - if ref and ref["key"] not in refs: - refs[ref["key"]] = ref - stmts.append( - f"self.{ref['param']} = self._create_package('{ref['abbr']}', {ref['param']})" - ) - elif base == "MFModel": + return attrs - def _should_set(var: dict) -> bool: - return var["name"] not in [ - "export_netcdf", - "nc_filerecord", - "packages", - ] + @pass_context + def init(ctx, vars) -> List[str]: + ctx_name = ctx["name"] + base = Filters.Cls.base(ctx_name) - stmts = [] - refs = {} - for var in ctx["vars"].values(): - ref = var.get("meta", dict()).get("ref", None) - if not var.get("kind", None): - continue - - name = var["name"] - if name in kwlist: - name = f"{name}_" - - if _should_set(var): - stmts.append(f"self.name_file.{name}.set_data({name})") - stmts.append(f"self.{name} = self.name_file.{name}") - if ref and ref["key"] not in refs: - refs[ref["key"]] = ref - stmts.append( - f"self.{ref['param']} = self._create_package('{ref['abbr']}', {ref['param']})" - ) - elif base == "MFPackage": + def _statements() -> Optional[List[str]]: + if base == "MFSimulationBase": + + def _should_set(var: dict) -> bool: + return var["name"] not in [ + "tdis6", + "models", + "exchanges", + "mxiter", + "solutiongroup", + "hpc_data", + ] + + stmts = [] + refs = {} + for var in vars.values(): + ref = var.get("meta", dict()).get("ref", None) + name = var["name"] + if name in kwlist: + name = f"{name}_" - def _should_build(var: dict) -> bool: - if var.get("meta", dict()).get("ref", None) and ctx[ - "name" - ] != ( - None, - "nam", - ): - return False - name = var["name"] - if name in [ - "simulation", - "model", - "package", - "parent_model", - "parent_package", - "parent_model_or_package", - "parent_file", - "modelname", - "model_nam_file", - "export_netcdf", - "nc_filerecord", - "method", - "interpolation_method_single", - "sfac", - "output", - ]: - return False - return True - - stmts = [] - refs = {} - for var in ctx["vars"].values(): - name = var["name"] - ref = var.get("meta", dict()).get("ref", None) - if name in kwlist: - name = f"{name}_" - - if _should_build(var): - if ref and ctx["name"] == (None, "nam"): + if _should_set(var): stmts.append( - f"self.{'_' if ref else ''}{ref['key']} = self.build_mfdata('{ref['key']}', None)" + f"self.name_file.{name}.set_data({name})" ) - else: - # hack... - _name = name[:-1] if name.endswith("_") else name - if _name == "steady_state": - _name = "steady-state" stmts.append( - f"self.{'_' if ref else ''}{name} = self.build_mfdata('{_name}', {name if var.get('init_param', True) else 'None'})" + f"self.{name} = self.name_file.{name}" ) + if ref and ref["key"] not in refs: + refs[ref["key"]] = ref + stmts.append( + f"self.{ref['param']} = self._create_package('{ref['abbr']}', {ref['param']})" + ) + elif base == "MFModel": - if ( - ref - and ref["key"] not in refs - and ctx["name"].r != "nam" - ): - refs[ref["key"]] = ref - stmts.append( - f"self._{ref['key']} = self.build_mfdata('{ref['key']}', None)" - ) - stmts.append( - f"self._{ref['abbr']}_package = self.build_child_package('{ref['abbr']}', {ref['val']}, '{ref['param']}', self._{ref['key']})" - ) - - return stmts - - return list(filter(None, _statements())) - - @pass_context - def parent(ctx, ctx_name): - ref = ctx["meta"].get("ref", None) - if ref: - return ref["parent"] - if ctx_name == ("sim", "nam"): - return None - elif ( - ctx_name.l is None - or ctx_name.r is None - or ctx_name.l in ["sim", "exg", "sln"] - ): - return "simulation" - elif ref: - if ctx_name.l == "utl" and ctx_name.r == "hpc": - return "simulation" - return "package" - return "model" - - def prefix(ctx_name): - return "MF" if ctx_name.base == "MFSimulationBase" else "Modflow" - - @pass_context - def skip(ctx, ctx_name): - base = ctx_name.base - if base == "MFSimulationBase": - return [ - "tdis6", - "models", - "exchanges", - "mxiter", - "solutiongroup", - ] - elif base == "MFModel": - skip = ["packages", "export_netcdf", "nc_filerecord"] - refs = ctx.get("meta", dict()).get("refs", dict()) - if any(refs) and ctx["name"] != (None, "nam"): - for k in refs.keys(): - if ctx["vars"].get(k, None): - skip.append(k) - return skip - else: - if ctx_name.r == "nam": - return ["export_netcdf", "nc_filerecord"] - elif ctx_name == ("utl", "ts"): - return ["method", "interpolation_method_single", "sfac"] - return [] + def _should_set(var: dict) -> bool: + return var["name"] not in [ + "export_netcdf", + "nc_filerecord", + "packages", + ] + + stmts = [] + refs = {} + for var in vars.values(): + ref = var.get("meta", dict()).get("ref", None) + name = var["name"] + if name in kwlist: + name = f"{name}_" + + if _should_set(var): + stmts.append( + f"self.name_file.{name}.set_data({name})" + ) + stmts.append( + f"self.{name} = self.name_file.{name}" + ) + if ref and ref["key"] not in refs: + refs[ref["key"]] = ref + stmts.append( + f"self.{ref['param']} = self._create_package('{ref['abbr']}', {ref['param']})" + ) + elif base == "MFPackage": + + def _should_build(var: dict) -> bool: + if var.get("meta", dict()).get( + "ref", None + ) and ctx_name != ( + None, + "nam", + ): + return False + name = var["name"] + if name in [ + "simulation", + "model", + "package", + "parent_model", + "parent_package", + "parent_model_or_package", + "parent_file", + "modelname", + "model_nam_file", + "export_netcdf", + "nc_filerecord", + "method", + "interpolation_method_single", + "sfac", + "output", + ]: + return False + return True + + stmts = [] + refs = {} + for var in vars.values(): + name = var["name"] + ref = var.get("meta", dict()).get("ref", None) + if name in kwlist: + name = f"{name}_" + + if _should_build(var): + if ref and ctx["name"] == (None, "nam"): + stmts.append( + f"self.{'_' if ref else ''}{ref['key']} = self.build_mfdata('{ref['key']}', None)" + ) + else: + _name = ( + name[:-1] if name.endswith("_") else name + ) + name = name.replace("-", "_") + stmts.append( + f"self.{'_' if ref else ''}{name} = self.build_mfdata('{_name}', {name})" + ) + + if ( + ref + and ref["key"] not in refs + and ctx["name"].r != "nam" + ): + refs[ref["key"]] = ref + stmts.append( + f"self._{ref['key']} = self.build_mfdata('{ref['key']}', None)" + ) + stmts.append( + f"self._{ref['abbr']}_package = self.build_child_package('{ref['abbr']}', {ref['val']}, '{ref['param']}', self._{ref['key']})" + ) + + return stmts + + return list(filter(None, _statements())) + + def nokw(v: str) -> str: + return (f"{v}_" if v in kwlist else v).replace("-", "_") + + def escape_trailing(v: str) -> str: + return f"{v[:-1]}\\\\_" if v.endswith("_") else v + + def value(v: Any) -> str: + v = try_get_enum_value(v) + if isinstance(v, str) and v[0] not in ["'", '"']: + v = f"'{v}'" + return v diff --git a/flopy/mf6/utils/codegen/renderable.py b/flopy/mf6/utils/codegen/renderable.py deleted file mode 100644 index 6e1384fd15..0000000000 --- a/flopy/mf6/utils/codegen/renderable.py +++ /dev/null @@ -1,129 +0,0 @@ -""" -This module contains a decorator intended to -allow modifying dataclass instances to make -them more palatable for templates. It also -keeps implementation details incidental to -the current design of MF6 input framework -cleanly isolated from the reimplementation -of which this code is a part, which aims -for a more general approach. -""" - -from dataclasses import asdict, is_dataclass -from typing import Any, Callable, Dict, Iterable, Optional, Tuple - -from flopy.mf6.utils.codegen.utils import try_get_enum_value - -Predicate = Callable[[Any], bool] -Transform = Callable[[Any], Dict[str, str]] -Pair = Tuple[str, Any] -Pairs = Iterable[Pair] - - -def renderable( - maybe_cls=None, - *, - keep_none: Optional[Iterable[str]] = None, - drop_keys: Optional[Iterable[str]] = None, - quote_str: Optional[Iterable[str]] = None, - transform: Optional[Iterable[Tuple[Predicate, Transform]]] = None, -): - """ - Decorator for dataclasses which are meant - to be passed into a Jinja template. The - decorator adds a `.render()` method to - the decorated class, which recursively - converts the instance to a dictionary - with (by default) the `asdict()` builtin - `dataclasses` module function, plus a - few modifications to make the instance - easier to work with from the template. - - By default, attributes with value `None` - are dropped before conversion to a `dict`. - To specify that a given attribute should - remain even with a `None` value, use the - `keep_none` parameter. - - When a string value is to become the RHS - of an assignment or an argument-passing - expression, it needs to be wrapped with - quotation marks before insertion into - the template. To indicate an attribute's - value should be wrapped with quotation - marks, use the `quote_str` parameter. - - Arbitrary transformations can be configured - via the `transform` parameter, which accepts - an iterable of predicate / function tuples. - Each of these specifies a condition in which - an instance of a context should be modified, - and a function to make the alteration. - - Notes - ----- - Because a transformation function accepts an - instance of a dataclass and converts it to a - dictionary, only one transformation function - (of the first matching predicate) is applied. - - This was inspired by `attrs` class decorators. - """ - - quote_str = quote_str or list() - keep_none = keep_none or list() - drop_keys = drop_keys or list() - transform = transform or list() - - def __renderable(cls): - def _render(d: dict) -> dict: - def _render_val(k, v): - v = try_get_enum_value(v) - if ( - k in quote_str - and isinstance(v, str) - and v[0] not in ["'", '"'] - ): - v = f"'{v}'" - elif isinstance(v, dict): - v = _render(v) - return v - - def _keep(k, v): - return k in keep_none or (v and not isinstance(v, bool)) - - def _drop(k, v): - return k in drop_keys - - return { - k: _render_val(k, v) - for k, v in d.items() - if (_keep(k, v) and not _drop(k, v)) - } - - def _dict(o): - d = dict(o) - for p, t in transform: - if p(o): - d = t(o) - break - return d - - def _dict_factory(o): - return _render(_dict(o)) - - def render(self) -> dict: - """Recursively render the dataclass instance.""" - return _render( - asdict(self, dict_factory=_dict_factory) - if is_dataclass(self) - else self - ) - - setattr(cls, "render", render) - return cls - - # first arg value depends on the decorator usage: - # class if `@renderable`, `None` if `@renderable()`. - # referenced from https://github.com/python-attrs/attrs/blob/a59c5d7292228dfec5480388b5f6a14ecdf0626c/src/attr/_next_gen.py#L405C4-L406C65 - return __renderable if maybe_cls is None else __renderable(maybe_cls) diff --git a/flopy/mf6/utils/codegen/shim.py b/flopy/mf6/utils/codegen/shim.py deleted file mode 100644 index f94c55576d..0000000000 --- a/flopy/mf6/utils/codegen/shim.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -The purpose of this module is to keep special handling -necessary to support the current `flopy.mf6` generated -classes separate from more general templating and code -generation infrastructure. -""" - - -def _is_context(o) -> bool: - d = dict(o) - return "name" in d and "vars" in d - - -def _replace_refs(ctx: dict, name_param: str = "val") -> dict: - refs = ctx.get("meta", dict()).get("refs", dict()) - if any(refs): - for key, ref in refs.items(): - key_var = ctx["vars"].get(key, None) - if not key_var: - continue - ctx["vars"][key] = { - **key_var, - "name": ref[name_param], - "description": ( - f"* Contains data for the {ref['abbr']} package. Data can be " - f"stored in a dictionary containing data for the {ref['abbr']} " - "package with variable names as keys and package data as " - f"values. Data just for the {ref['val']} variable is also " - f"acceptable. See {ref['abbr']} package documentation for more " - "information" - ), - "ref": ref, - "default": None, - "children": None, - } - return ctx - - -def _transform_context(o): - ctx = dict(o) - ctx_name = ctx["name"] - ctx_base = ctx_name.base - if ctx_base == "MFSimulationBase": - return _replace_refs(ctx, name_param="param") - else: - return _replace_refs(ctx) - - -SHIM = { - "keep_none": ["default", "vars"], - "quote_str": ["default"], - "transform": [(_is_context, _transform_context)], -} diff --git a/flopy/mf6/utils/codegen/templates/__init__.py.jinja b/flopy/mf6/utils/codegen/templates/__init__.py.jinja index 1db6d034d6..2eb7d436c9 100644 --- a/flopy/mf6/utils/codegen/templates/__init__.py.jinja +++ b/flopy/mf6/utils/codegen/templates/__init__.py.jinja @@ -1,3 +1,3 @@ {% for ctx in contexts %} -from .mf{{ ctx.name.title }} import {{ ctx.name|prefix }}{{ ctx.name.title.title() }} +from .mf{{ ctx.name|title }} import {{ ctx.name|prefix }}{{ (ctx.name|title).title() }} {% endfor %} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/exchange.py.jinja b/flopy/mf6/utils/codegen/templates/exchange.py.jinja index fb52a2d498..bf052f1a12 100644 --- a/flopy/mf6/utils/codegen/templates/exchange.py.jinja +++ b/flopy/mf6/utils/codegen/templates/exchange.py.jinja @@ -1,4 +1,5 @@ {% import 'macros.jinja' as macros %} +{% set title = (name|title).title() %} from os import PathLike, curdir from typing import Union @@ -6,16 +7,16 @@ from typing import Union from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator from flopy.mf6.mfpackage import MFPackage -class Modflow{{ name.title.title() }}(MFPackage): +class Modflow{{ title }}(MFPackage): """ - {{ name.description }} + {{ name|description }} Parameters ---------- - {{ macros.vars_docs(vars, start_indent=4) }} + {{ macros.vars_docs(vars, indent=4) }} """ - {% for attr in cls_attrs %} + {% for attr in vars|attrs %} {{ attr }} {%- endfor %} @@ -26,13 +27,13 @@ class Modflow{{ name.title.title() }}(MFPackage): exgtype="{{ name.r[:3].upper() }}6-{{ name.r[3:].upper() }}6", exgmnamea=None, exgmnameb=None, - {{ macros.init_vars(vars, skip=name|skip) }} + {{ macros.init_vars(vars, indent=8, skip=name|skip) }} filename=None, pname=None, **kwargs, ): """ - {{ name.description }} + {{ name|description }} simulation : MFSimulation Simulation that this package is a part of. Package is automatically @@ -63,7 +64,7 @@ class Modflow{{ name.title.title() }}(MFPackage): GWE Model with the name exgmnameb must correspond to the GWF Model with the name gwfmodelname2. - {{ macros.vars_docs(vars, start_indent=8) }} + {{ macros.vars_docs(vars, indent=8) }} """ super().__init__( @@ -80,7 +81,7 @@ class Modflow{{ name.title.title() }}(MFPackage): self.exgmnameb = exgmnameb simulation.register_exchange_file(self) - {% for statement in init_body %} + {% for statement in vars|init %} {{ statement }} {%- endfor %} diff --git a/flopy/mf6/utils/codegen/templates/macros.jinja b/flopy/mf6/utils/codegen/templates/macros.jinja index 3befc515b1..ee7a038c96 100644 --- a/flopy/mf6/utils/codegen/templates/macros.jinja +++ b/flopy/mf6/utils/codegen/templates/macros.jinja @@ -1,17 +1,17 @@ -{% macro init_vars(vars, alias=false, skip=none) %} -{%- for n, var in vars.items() if n not in skip %} -{% if alias %}{{ n }}{% else %}{{ var.name }}{% endif %}{%- if var.default is defined %}={{ var.default }}{%- endif -%}, +{% macro init_vars(vars, alias=false, indent=0, skip=none) %} +{%- for n, v in vars.items() if n not in skip %} +{{ ""|indent(indent, first=true) }}{% if alias %}{{ n|nokw }}{% else %}{{ v.name|nokw }}{% endif %}{%- if v.default is defined %}={{ v.default|value }}{%- endif -%}, {%- endfor %} {% endmacro %} -{% macro vars_docs(vars, start_indent=0) %} +{% macro vars_docs(vars, indent=0) %} {%- for v in vars.values() recursive %} -{{ ""|indent(start_indent, first=true) }}{% if loop.depth > 1 %}* {% endif %}{% if v.name|last == "_" %}{{ v.name.replace("_", "\\\_") }}{% else %}{{ v.name }}{% endif %}{% if v.type is defined %} : {{ v.type }}{% endif %} +{{ ""|indent(indent, first=true) }}{% if loop.depth > 1 %}* {% endif %}{{ v.name|nokw|escape_trailing }} : {{ v|type }} {%- if v.description is defined and v.description is not none %} -{{ v.description|wordwrap|indent(start_indent + (loop.depth * 4), first=true) }} +{{ v.description|wordwrap|indent(indent + (loop.depth * 4), first=true) }} {%- endif %} {%- if v.children is defined and v.children is not none -%} -{{ loop(v.children.values())|indent(start_indent, first=true) }} +{{ loop(v.children.values())|indent(indent, first=true) }} {%- endif %} {% endfor -%} {% endmacro %} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/model.py.jinja b/flopy/mf6/utils/codegen/templates/model.py.jinja index 079485debb..651dcf6a61 100644 --- a/flopy/mf6/utils/codegen/templates/model.py.jinja +++ b/flopy/mf6/utils/codegen/templates/model.py.jinja @@ -1,4 +1,5 @@ {% import 'macros.jinja' as macros %} +{% set title = (name|title).title() %} from os import PathLike, curdir from typing import Union @@ -7,13 +8,13 @@ from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenera from flopy.mf6.mfmodel import MFModel -class Modflow{{ name.title.title() }}(MFModel): +class Modflow{{ title }}(MFModel): """ - {{ name.description }} + {{ name|description }} Parameters ---------- - {{ macros.vars_docs(vars, start_indent=4) }} + {{ macros.vars_docs(vars, indent=4) }} Methods ------- @@ -23,7 +24,7 @@ class Modflow{{ name.title.title() }}(MFModel): a class method that loads a model from files """ - model_type = "{{ name.title }}" + model_type = "{{ title.lower() }}" def __init__( self, @@ -33,11 +34,11 @@ class Modflow{{ name.title.title() }}(MFModel): version="mf6", exe_name="mf6", model_rel_path=".", - {{ macros.init_vars(vars, skip=name|skip) }} + {{ macros.init_vars(vars, indent=8, skip=name|skip) }} **kwargs, ): """ - {{ name.description }} + {{ name|description }} Parameters ---------- @@ -60,12 +61,12 @@ class Modflow{{ name.title.title() }}(MFModel): Simulation that this model is a part of. Model is automatically added to simulation when it is initialized. - {{ macros.vars_docs(vars, start_indent=8) }} + {{ macros.vars_docs(vars, indent=8) }} """ super().__init__( simulation, - model_type="{{ name.title }}6", + model_type="{{ title.lower() }}6", modelname=modelname, model_nam_file=model_nam_file, version=version, @@ -74,7 +75,7 @@ class Modflow{{ name.title.title() }}(MFModel): **kwargs, ) - {% for statement in init_body %} + {% for statement in vars|init %} {{ statement }} {%- endfor %} @@ -97,7 +98,7 @@ class Modflow{{ name.title.title() }}(MFModel): structure, modelname, model_nam_file, - "{{ name.title }}6", + "{{ title.lower() }}6", version, exe_name, strict, diff --git a/flopy/mf6/utils/codegen/templates/package.py.jinja b/flopy/mf6/utils/codegen/templates/package.py.jinja index cbc86319e5..a763e81eb0 100644 --- a/flopy/mf6/utils/codegen/templates/package.py.jinja +++ b/flopy/mf6/utils/codegen/templates/package.py.jinja @@ -1,4 +1,5 @@ {% import 'macros.jinja' as macros %} +{% set title = (name|title).title() %} from os import PathLike, curdir from typing import Union @@ -7,16 +8,16 @@ from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenera from flopy.mf6.mfpackage import MFPackage, MFChildPackages -class Modflow{{ name.title.title() }}(MFPackage): +class Modflow{{ title }}(MFPackage): """ - {{ name.description }} + {{ name|description }} Parameters ---------- - {{ macros.vars_docs(vars, start_indent=4) }} + {{ macros.vars_docs(vars, indent=4) }} """ - {% for attr in cls_attrs %} + {% for attr in vars|attrs %} {{ attr }} {%- endfor %} @@ -24,13 +25,13 @@ class Modflow{{ name.title.title() }}(MFPackage): self, {{ name|parent }}, loading_package=False, - {{ macros.init_vars(vars, skip=name|skip) }} + {{ macros.init_vars(vars, indent=8, skip=name|skip) }} filename=None, pname=None, **kwargs, ): """ - {{ name.description }} + {{ name|description }} Parameters ---------- @@ -42,7 +43,7 @@ class Modflow{{ name.title.title() }}(MFPackage): Do not set this parameter. It is intended for debugging and internal processing purposes only. - {{ macros.vars_docs(vars, start_indent=8) }} + {{ macros.vars_docs(vars, indent=8) }} filename : str File name for this package. @@ -65,39 +66,39 @@ class Modflow{{ name.title.title() }}(MFPackage): **kwargs ) - {% for statement in init_body %} + {% for statement in vars|init %} {{ statement }} {%- endfor %} self._init_complete = True {% if "ref" in meta and name.r != "hpc" %} -class {{ name.title.title() }}Packages(MFChildPackages): +class {{ title }}Packages(MFChildPackages): """ - {{ name.title.title() }}Packages is a container class for the Modflow{{ name.title.title() }} class. + {{ title }}Packages is a container class for the Modflow{{ title }} class. Methods ------- initialize - Initializes a new Modflow{{ name.title.title() }} package removing any sibling child - packages attached to the same parent package. See Modflow{{ name.title.title() }} init + Initializes a new Modflow{{ title }} package removing any sibling child + packages attached to the same parent package. See Modflow{{ title }} init documentation for definition of parameters. append_package - Adds a new Modflow{{ name.title.title() }} package to the container. See Modflow{{ name.title.title() }} + Adds a new Modflow{{ title }} package to the container. See Modflow{{ title }} init documentation for definition of parameters. """ - package_abbr = "{{ name.title.lower() }}packages" + package_abbr = "{{ title.lower() }}packages" def initialize( self, - {{ macros.init_vars(vars, alias=true, skip=name|skip) }} + {{ macros.init_vars(vars, alias=true, indent=8, skip=name|skip) }} filename=None, pname=None, ): - new_package = Modflow{{ name.title.title() }}( + new_package = Modflow{{ title }}( self._cpparent, {%- for n, var in vars.items() if n not in name|skip %} {{ n }}={{ n }}, @@ -111,11 +112,11 @@ class {{ name.title.title() }}Packages(MFChildPackages): {% if name.r != "obs" %} def append_package( self, - {{ macros.init_vars(vars, alias=true, skip=name|skip) }} + {{ macros.init_vars(vars, alias=true, indent=8, skip=name|skip) }} filename=None, pname=None, ): - new_package = Modflow{{ name.title.title() }}( + new_package = Modflow{{ title }}( self._cpparent, {%- for n, var in vars.items() if n not in name|skip %} {{ n }}={{ n }}, diff --git a/flopy/mf6/utils/codegen/templates/simulation.py.jinja b/flopy/mf6/utils/codegen/templates/simulation.py.jinja index 76c15057bc..bdfbf0547f 100644 --- a/flopy/mf6/utils/codegen/templates/simulation.py.jinja +++ b/flopy/mf6/utils/codegen/templates/simulation.py.jinja @@ -1,4 +1,5 @@ {% import 'macros.jinja' as macros %} +{% set title = (name|title).title() %} from os import PathLike, curdir from typing import Union @@ -6,13 +7,13 @@ from typing import Union from flopy.mf6.data.mfdatautil import ArrayTemplateGenerator, ListTemplateGenerator from flopy.mf6.mfsimbase import MFSimulationBase -class MF{{ name.title.title() }}(MFSimulationBase): +class MF{{ title }}(MFSimulationBase): """ - {{ name.description }} + {{ name|description }} Parameters ---------- - {{ macros.vars_docs(vars, start_indent=4) }} + {{ macros.vars_docs(vars, indent=4) }} Methods ------- @@ -34,10 +35,10 @@ class MF{{ name.title.title() }}(MFSimulationBase): write_headers: bool = True, use_pandas: bool = True, lazy_io: bool = False, - {{ macros.init_vars(vars, skip=name|skip) }} + {{ macros.init_vars(vars, indent=8, skip=name|skip) }} ): """ - {{ name.description }} + {{ name|description }} Parameters ---------- @@ -65,7 +66,7 @@ class MF{{ name.title.title() }}(MFSimulationBase): lazy_io Whether to use lazy IO - {{ macros.vars_docs(vars, start_indent=8) }} + {{ macros.vars_docs(vars, indent=8) }} """ super().__init__( @@ -79,7 +80,7 @@ class MF{{ name.title.title() }}(MFSimulationBase): use_pandas=use_pandas ) - {% for statement in init_body %} + {% for statement in vars|init %} {{ statement }} {%- endfor %} diff --git a/flopy/mf6/utils/codegen/utils.py b/flopy/mf6/utils/codegen/utils.py deleted file mode 100644 index e663835be6..0000000000 --- a/flopy/mf6/utils/codegen/utils.py +++ /dev/null @@ -1,3 +0,0 @@ -from ast import literal_eval -from enum import Enum -from typing import Any From 0a16989953d5148354c2a124d10fce43fad86c15 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 5 Nov 2024 10:27:10 -0500 Subject: [PATCH 58/75] cleanup --- flopy/mf6/utils/codegen/context.py | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py index 0c30cecb00..7c6288f782 100644 --- a/flopy/mf6/utils/codegen/context.py +++ b/flopy/mf6/utils/codegen/context.py @@ -36,24 +36,15 @@ class Context: class Name(NamedTuple): """ Uniquely identifies an input context. The name - consists of a left term and optional right term. + consists of a left (component) term and optional + right (subcomponent) term. Notes ----- A single definition may be associated with one or more contexts. For instance, a model DFN file will produce - both a namefile package class and a model class. - - From the context name several other things are derived: - - - a description of the context - - the input context class' name - - the template the context will populate - - the base class the context inherits from - - the name of the source file the context is in - - the name of the parent parameter in the context - class' `__init__` method, if it can have a parent - + both a namefile package class and a model class. These + share a single DFN name but have different context names. """ l: str @@ -63,15 +54,7 @@ class Name(NamedTuple): def from_dfn(dfn: Dfn) -> List["Context.Name"]: """ Returns a list of context names this definition produces. - - Notes - ----- An input definition may produce one or more input contexts. - - Model definition files produce both a model class context and - a model namefile package context. The same goes for simulation - definition files. All other definition files produce a single - context. """ if dfn.name.r == "nam": if dfn.name.l == "sim": From bdfd63bba32f1518b4fb0f511e776a1f62c8c45f Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 5 Nov 2024 10:55:56 -0500 Subject: [PATCH 59/75] update dev guide --- docs/mf6_dev_guide.md | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/docs/mf6_dev_guide.md b/docs/mf6_dev_guide.md index cb4e041ed2..fd1921d3dd 100644 --- a/docs/mf6_dev_guide.md +++ b/docs/mf6_dev_guide.md @@ -1,14 +1,32 @@ Introduction ----------------------------------------------- -This file provides an overview of how FloPy for MODFLOW 6 (FPMF6) works under the hood and is intended for anyone who wants to add a new package, new model type, or new features to this library. FloPy library files that support MODFLOW 6 can be found in the flopy/mf6 folder and sub-folders. +This file provides an overview of how FloPy's MODFLOW 6 module `flopy.mf6` works under the hood. It is intended for anyone who wants to add a new package, new model, or new features to this library. -Package Meta-Data and Package Files +Code generation ----------------------------------------------- -FPMF6 uses meta-data files located in flopy/mf6/data/dfn to define the model and package types supported by MODFLOW 6. When additional model and package types are added to MODFLOW 6, additional meta-data files can be added to this folder and flopy/mf6/utils/createpackages.py can be run to add new packages to the FloPy library. createpackages.py uses flopy/mf6/data/mfstructure.py to read meta-data files (*.dfn) and use that meta-data to create the package files found in flopy/mf6/modflow (do not directly modify any of the files in this folder, they are all automatically generated). The automatically generated package files contain an interface for accessing package data and data documentation generated from the meta-data files. Additionally, meta-data describing package data types and shapes is stored in the dfn attribute. flopy/mf6/data/mfstructure.py can load structure information using the dfn attribute (instead of loading it from the meta-data files). This allows for flopy to be installed without the dfn files. +MODFLOW 6 describes its input specification with definition files. These are currently a custom text-based format. Definition files have suffix `.dfn` by convention. -All meta-data can be accessed from the flopy.mf6.data.mfstructure.MFStructure class. This is a singleton class, meaning only one instance of this class can be created. The class contains a sim_struct attribute (which is a flopy.mf6.data.mfstructure.MFSimulationStructure object) which contains all of the meta-data for all package files. Meta-data is stored in a structured format. MFSimulationStructure contains MFModelStructure and MFInputFileStructure objects, which contain the meta-data for each model type and each "simulation-level" package (tdis, ims, ...). MFModelStructure contains model specific meta-data and a MFInputFileStructure object for each package in that model. MFInputFileStructure contains package specific meta-data and a MFBlockStructure object for each block contained in the package file. MFBlockStructure contains block specific meta-data and a MFDataStructure object for each data structure defined in the block, and MFDataStructure contains data structure specific meta-data and a MFDataItemStructure object for each data item contained in the data structure. Data structures define the structure of data that is naturally grouped together, for example, the data in a numpy recarray. Data item structures define the structure of specific pieces of data, for example, a single column of a numpy recarray. The meta-data defined in these classes provides all the information FloPy needs to read and write MODFLOW 6 package and name files, create the Flopy interface, and check the data for various constraints. +Definition files describe components (e.g. simulations, models, packages) supported by MODFLOW 6, and are used to generate both source code and documentation. + +FloPy has two scripts that can be used to generate a MODFLOW 6 compatibility layer: + +- `flopy/mf6/utils/createpackages.py`: assumes definition files are in `flopy/mf6/data/dfn` +- `flopy/mf6/utils/generate_classes.py`: downloads DFNs then runs `createpackages.py` + +The latter is typically used with e.g. `python -m flopy.mf6.utils.generate_classes --ref develop`. + +Generated files are created in `flopy/mf6/modflow/` and contain interface classes, one file/class per input component. These can be used to initialize and access model/package data as well as the input specification itself. + +**Note**: Code generation scripts previously used `flopy/mf6/data/mfstructure.py` to read and represent definition files, and wrote Python by hand. They now use the `flopy.mf6.utils.codegen` module, which uses Jinja2. + +**Note**: Each component's input definition is currently reproduced almost verbatim in the `dfn` class attribute. Currently, `flopy/mf6/data/mfstructure.py` is used to introspect the input specification using the `dfn` attribute. This can eventually be removed in favor of direct introspection. + +Input specification +------------------- + +The `flopy.mf6.data.mfstructure.MFStructure` class represents an input specification. The class is a singleton, meaning only one instance of this class can be created. The class contains a sim_struct attribute (which is a flopy.mf6.data.mfstructure.MFSimulationStructure object) which contains all of the meta-data for all package files. Meta-data is stored in a structured format. MFSimulationStructure contains MFModelStructure and MFInputFileStructure objects, which contain the meta-data for each model type and each "simulation-level" package (tdis, ims, ...). MFModelStructure contains model specific meta-data and a MFInputFileStructure object for each package in that model. MFInputFileStructure contains package specific meta-data and a MFBlockStructure object for each block contained in the package file. MFBlockStructure contains block specific meta-data and a MFDataStructure object for each data structure defined in the block, and MFDataStructure contains data structure specific meta-data and a MFDataItemStructure object for each data item contained in the data structure. Data structures define the structure of data that is naturally grouped together, for example, the data in a numpy recarray. Data item structures define the structure of specific pieces of data, for example, a single column of a numpy recarray. The meta-data defined in these classes provides all the information FloPy needs to read and write MODFLOW 6 package and name files, create the Flopy interface, and check the data for various constraints. ```mermaid classDiagram @@ -22,12 +40,8 @@ classDiagram Figure 1: Generic data structure hierarchy. Connections show composition relationships. -Package and Data Base Classes ------------------------------------------------ - The package and data classes are related as shown below in figure 2. On the top of the figure 2 is the MFPackage class, which is the base class for all packages. MFPackage contains generic methods for building data objects and reading and writing the package to a file. MFPackage contains a MFInputFileStructure object that defines how the data is structured in the package file. MFPackage also contains a dictionary of blocks (MFBlock). The MFBlock class is a generic class used to represent a block within a package. MFBlock contains a MFBlockStructure object that defines how the data in the block is structured. MFBlock also contains a dictionary of data objects (subclasses of MFData) contained in the block and a list of block headers (MFBlockHeader) for that block. Block headers contain the block's name and optionally data items (eg. iprn). - ```mermaid classDiagram From 6af0b9562cd1e6e98897aef966affa1edf5cab44 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 5 Nov 2024 13:55:05 -0500 Subject: [PATCH 60/75] file record squashing as a filter --- autotest/test_codegen.py | 2 +- flopy/mf6/utils/codegen/__init__.py | 9 +++-- flopy/mf6/utils/codegen/dfn.py | 36 ++++--------------- flopy/mf6/utils/codegen/dfn2toml.py | 9 ++--- .../utils/codegen/{jinja.py => filters.py} | 32 ++++++++++++++++- .../mf6/utils/codegen/templates/macros.jinja | 12 ++++--- 6 files changed, 55 insertions(+), 45 deletions(-) rename flopy/mf6/utils/codegen/{jinja.py => filters.py} (94%) diff --git a/autotest/test_codegen.py b/autotest/test_codegen.py index 34ab9d814c..0d7b3bec7b 100644 --- a/autotest/test_codegen.py +++ b/autotest/test_codegen.py @@ -4,7 +4,7 @@ from flopy.mf6.utils.codegen import make_all, make_targets from flopy.mf6.utils.codegen.context import Context from flopy.mf6.utils.codegen.dfn import Dfn -from flopy.mf6.utils.codegen.jinja import Filters +from flopy.mf6.utils.codegen.filters import Filters PROJ_ROOT = get_project_root_path() MF6_PATH = PROJ_ROOT / "flopy" / "mf6" diff --git a/flopy/mf6/utils/codegen/__init__.py b/flopy/mf6/utils/codegen/__init__.py index f90df170fd..51319b0853 100644 --- a/flopy/mf6/utils/codegen/__init__.py +++ b/flopy/mf6/utils/codegen/__init__.py @@ -15,7 +15,7 @@ def _get_template_env(): loader = jinja.PackageLoader("flopy", "mf6/utils/codegen/templates/") env = jinja.Environment(loader=loader) - from flopy.mf6.utils.codegen.jinja import Filters + from flopy.mf6.utils.codegen.filters import Filters env.filters["base"] = Filters.Cls.base env.filters["title"] = Filters.Cls.title @@ -27,10 +27,13 @@ def _get_template_env(): env.filters["attrs"] = Filters.Vars.attrs env.filters["init"] = Filters.Vars.init + env.filters["maybe_file"] = Filters.Var.maybe_file env.filters["type"] = Filters.Var.type env.filters["nokw"] = Filters.nokw - env.filters["escape_trailing"] = Filters.escape_trailing + env.filters["escape_trailing_underscore"] = ( + Filters.escape_trailing_underscore + ) env.filters["value"] = Filters.value return env @@ -65,7 +68,7 @@ def make_targets(dfn, outdir: PathLike, verbose: bool = False): outdir = Path(outdir).expanduser() from flopy.mf6.utils.codegen.context import Context - from flopy.mf6.utils.codegen.jinja import Filters + from flopy.mf6.utils.codegen.filters import Filters def _get_template_name(ctx_name) -> str: """The template file to use.""" diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index 577ab4d91d..927cc1f4c7 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -1,7 +1,5 @@ from ast import literal_eval from collections import UserDict -from enum import Enum -from keyword import kwlist from os import PathLike from typing import ( Any, @@ -53,13 +51,12 @@ def _try_parse_bool(value: Any) -> Any: class Var(TypedDict): - """MODFLOW 6 input variable specification.""" + """An input variable specification.""" name: str - type: Optional[str] = None + type: str shape: Optional[Any] = None block: Optional[str] = None - fkey: Optional["Ref"] = None default: Optional[Any] = None children: Optional[Vars] = None description: Optional[str] = None @@ -184,7 +181,7 @@ def __init__( name: Optional[Name] = None, meta: Optional[Dict[str, Any]] = None, ): - self.data = OMD(data) + self.data = data or dict() self.name = name self.meta = meta @@ -327,7 +324,6 @@ def _map(spec: Dict[str, Any]) -> Var: _try_literal_eval(default) if _type != "string" else default ) description = spec.get("description", "") - tagged = spec.get("tagged", False) children = dict() # if var is a foreign key, register it @@ -345,36 +341,16 @@ def _choices() -> Vars: } def _fields() -> Vars: - """Load a record's children (fields).""" + """Load a record's scalar children (fields).""" names = _type.split()[1:] - fields = { + return { v["name"]: _map(v) for v in flat.values(multi=True) if v["name"] in names - and not v["type"].startswith("record") and v.get("in_record", False) + and not v["type"].startswith("record") } - # if the record represents a file... - if "file" in _name: - # remove filein/fileout - for term in ["filein", "fileout"]: - if term in names: - fields.pop(term) - - # remove leading keyword - keyword = next(iter(fields), None) - if keyword: - fields.pop(keyword) - - # if tagged, remove the leading keyword - elif tagged: - keyword = next(iter(fields), None) - if keyword: - fields.pop(keyword) - - return fields - # list, child is the item type if _type.startswith("recarray"): names = _type.split()[1:] diff --git a/flopy/mf6/utils/codegen/dfn2toml.py b/flopy/mf6/utils/codegen/dfn2toml.py index 198e61d9de..98e835a4be 100644 --- a/flopy/mf6/utils/codegen/dfn2toml.py +++ b/flopy/mf6/utils/codegen/dfn2toml.py @@ -1,7 +1,8 @@ import argparse -from collections.abc import Mapping from pathlib import Path +from boltons.iterutils import remap + from flopy.utils import import_optional_dependency _MF6_PATH = Path(__file__).parents[2] @@ -10,11 +11,7 @@ def _drop_none(d: dict) -> dict: - return ( - {k: _drop_none(v) for k, v in d.items() if v is not None} - if isinstance(d, Mapping) - else d - ) + return remap(d, lambda p, k, v: isinstance(v, bool) or bool(v)) if __name__ == "__main__": diff --git a/flopy/mf6/utils/codegen/jinja.py b/flopy/mf6/utils/codegen/filters.py similarity index 94% rename from flopy/mf6/utils/codegen/jinja.py rename to flopy/mf6/utils/codegen/filters.py index cb0b3f764f..d29a317719 100644 --- a/flopy/mf6/utils/codegen/jinja.py +++ b/flopy/mf6/utils/codegen/filters.py @@ -113,6 +113,36 @@ def skip(ctx, ctx_name) -> List[str]: return [] class Var: + def maybe_file(var: dict) -> dict: + name = var["name"] + tagged = var.get("tagged", False) + fields = var.get("children", None) + + if not fields: + return var + + # if tagged, remove the leading keyword + elif tagged: + keyword = next(iter(fields), None) + if keyword: + fields.pop(keyword) + + # if the record represents a file... + elif "file" in name: + # remove filein/fileout + field_names = list(fields.keys()) + for term in ["filein", "fileout"]: + if term in field_names: + fields.pop(term) + + # remove leading keyword + keyword = next(iter(fields), None) + if keyword: + fields.pop(keyword) + + var["children"] = fields + return var + def type(var: dict) -> str: _type = var["type"] shape = var.get("shape", None) @@ -396,7 +426,7 @@ def _should_build(var: dict) -> bool: def nokw(v: str) -> str: return (f"{v}_" if v in kwlist else v).replace("-", "_") - def escape_trailing(v: str) -> str: + def escape_trailing_underscore(v: str) -> str: return f"{v[:-1]}\\\\_" if v.endswith("_") else v def value(v: Any) -> str: diff --git a/flopy/mf6/utils/codegen/templates/macros.jinja b/flopy/mf6/utils/codegen/templates/macros.jinja index ee7a038c96..2feb951415 100644 --- a/flopy/mf6/utils/codegen/templates/macros.jinja +++ b/flopy/mf6/utils/codegen/templates/macros.jinja @@ -1,12 +1,16 @@ {% macro init_vars(vars, alias=false, indent=0, skip=none) %} -{%- for n, v in vars.items() if n not in skip %} -{{ ""|indent(indent, first=true) }}{% if alias %}{{ n|nokw }}{% else %}{{ v.name|nokw }}{% endif %}{%- if v.default is defined %}={{ v.default|value }}{%- endif -%}, +{%- for name, var in vars.items() if name not in skip %} +{% set v = var|maybe_file %} +{% set n = (name if alias else v.name)|nokw %} +{{ ""|indent(indent, first=true) }}{{ n }}{%- if v.default is defined %}={{ v.default|value }}{%- endif -%}, {%- endfor %} {% endmacro %} {% macro vars_docs(vars, indent=0) %} -{%- for v in vars.values() recursive %} -{{ ""|indent(indent, first=true) }}{% if loop.depth > 1 %}* {% endif %}{{ v.name|nokw|escape_trailing }} : {{ v|type }} +{%- for var in vars.values() recursive %} +{% set v = var|maybe_file %} +{% set n = v.name|nokw|escape_trailing_underscore %} +{{ ""|indent(indent, first=true) }}{% if loop.depth > 1 %}* {% endif %}{{ n }} : {{ v|type }} {%- if v.description is defined and v.description is not none %} {{ v.description|wordwrap|indent(indent + (loop.depth * 4), first=true) }} {%- endif %} From 3fdc1d6d9d1d76c301d1420e5bada7d179ade618 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 5 Nov 2024 16:10:49 -0500 Subject: [PATCH 61/75] prep for version support --- autotest/test_codegen.py | 4 +- flopy/mf6/utils/codegen/dfn.py | 233 +++++++++--------- flopy/mf6/utils/codegen/filters.py | 51 ++-- .../mf6/utils/codegen/templates/macros.jinja | 8 +- 4 files changed, 153 insertions(+), 143 deletions(-) diff --git a/autotest/test_codegen.py b/autotest/test_codegen.py index 0d7b3bec7b..7ee07bc72e 100644 --- a/autotest/test_codegen.py +++ b/autotest/test_codegen.py @@ -23,7 +23,7 @@ def test_dfn_load(dfn_name): open(DFN_PATH / f"{dfn_name}.dfn", "r") as dfn_file, ): name = Dfn.Name.parse(dfn_name) - common, _ = Dfn._load(common_file) + common, _ = Dfn._load_v1_flat(common_file) Dfn.load(dfn_file, name=name, common=common) @@ -34,7 +34,7 @@ def test_make_targets(dfn_name, function_tmpdir): open(DFN_PATH / f"{dfn_name}.dfn", "r") as dfn_file, ): name = Dfn.Name.parse(dfn_name) - common, _ = Dfn._load(common_file) + common, _ = Dfn._load_v1_flat(common_file) dfn = Dfn.load(dfn_file, name=name, common=common) make_targets(dfn, function_tmpdir, verbose=True) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index 927cc1f4c7..e6217e9155 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -5,6 +5,7 @@ Any, Dict, List, + Literal, NamedTuple, Optional, Tuple, @@ -14,6 +15,8 @@ from boltons.dictutils import OMD +from flopy.utils.utl_import import import_optional_dependency + _SCALARS = { "keyword", "integer", @@ -172,6 +175,8 @@ def parse(cls, v: str) -> "Dfn.Name": def __str__(self) -> str: return "-".join(self) + Version = Literal[1] + name: Optional[Name] meta: Optional[Dict[str, Any]] @@ -186,14 +191,9 @@ def __init__( self.meta = meta @staticmethod - def _load(f, common: Optional[dict] = None) -> Tuple[OMD, List[str]]: - """ - Internal use only. Loads the DFN as a flat multi-dictionary* with a - list of string metadata, which are then parsed into structured form. - - *The point of this is to losslessly handle duplicate variable names. - - """ + def _load_v1_flat( + f, common: Optional[dict] = None, **kwargs + ) -> Tuple[OMD, List[str]]: var = dict() flat = list() meta = list() @@ -270,33 +270,22 @@ def _load(f, common: Optional[dict] = None) -> Tuple[OMD, List[str]]: if any(var): flat.append((var["name"], var)) + # the point of the OMD is to losslessly handle duplicate variable names return OMD(flat), meta @classmethod - def load( - cls, - f, - name: Optional[Name] = None, - refs: Optional[Dfns] = None, - **kwargs, - ) -> "Dfn": - """ - Load an input definition from a DFN file. - - Notes - ----- - Loads the DFN as a flat multidict with `_load()` - then walks composite variables and builds a tree. - """ - - flat, meta = Dfn._load(f, **kwargs) - refs = refs or dict() + def _load_v1(cls, f, name, **kwargs) -> "Dfn": + flat, meta = Dfn._load_v1_flat(f, **kwargs) + refs = kwargs.pop("refs", dict()) fkeys = dict() def _map(spec: Dict[str, Any]) -> Var: """ - Convert a variable specification from its representation - in an input definition file to a Pythonic form. + Convert an input variable specification from its shape + in a classic definition file to a Python-friendly form. + + This involves trimming unneeded attributes and setting + some others. Notes ----- @@ -316,50 +305,30 @@ def _map(spec: Dict[str, Any]) -> Var: _name = spec["name"] _type = spec.get("type", None) - block = spec.get("block", None) shape = spec.get("shape", None) shape = None if shape == "" else shape + block = spec.get("block", None) + children = dict() default = spec.get("default", None) default = ( _try_literal_eval(default) if _type != "string" else default ) description = spec.get("description", "") - children = dict() + fkey = refs.get(_name, None) # if var is a foreign key, register it - fkey = refs.get(_name, None) if fkey: fkeys[_name] = fkey - def _choices() -> Vars: - """Load a union's children (choices).""" - names = _type.split()[1:] - return { - v["name"]: _map(v) - for v in flat.values(multi=True) - if v["name"] in names and v.get("in_record", False) - } - - def _fields() -> Vars: - """Load a record's scalar children (fields).""" - names = _type.split()[1:] - return { - v["name"]: _map(v) - for v in flat.values(multi=True) - if v["name"] in names - and v.get("in_record", False) - and not v["type"].startswith("record") - } + def _items() -> Vars: + """Load a list's children (items: record or union of records).""" - # list, child is the item type - if _type.startswith("recarray"): names = _type.split()[1:] types = [ v["type"] for v in flat.values(multi=True) if v["name"] in names and v.get("in_record", False) ] - n_names = len(names) if n_names < 1: raise ValueError(f"Missing recarray definition: {_type}") @@ -371,19 +340,18 @@ def _fields() -> Vars: # be defined with a nested record (explicit) or with a # set of fields directly in the recarray (implicit). an # irregular list is always defined with a nested union. - is_explicit_composite = n_names == 1 and ( + is_explicit = n_names == 1 and ( types[0].startswith("record") or types[0].startswith("keystring") ) - is_implicit_scalar_record = all(t in _SCALARS for t in types) - if is_explicit_composite: + if is_explicit: child = next(iter(flat.getlist(names[0]))) - children = {names[0]: _map(child)} - _type = "list" - elif is_implicit_scalar_record: + return {names[0]: _map(child)} + elif all(t in _SCALARS for t in types): + # implicit simple record (all fields are scalars) fields = _fields() - children = { + return { _name: Var( name=_name, type="record", @@ -394,9 +362,8 @@ def _fields() -> Vars: ), ) } - _type = "list" else: - # implicit complex record (i.e. some fields are records or unions) + # implicit complex record (some fields are records or unions) fields = { v["name"]: _map(v) for v in flat.values(multi=True) @@ -410,7 +377,7 @@ def _fields() -> Vars: if single and "keystring" in first["type"] else "record" ) - children = { + return { name_: Var( name=name_, type=child_type, @@ -421,80 +388,115 @@ def _fields() -> Vars: ), ) } - _type = "list" - # union (product) type + def _choices() -> Vars: + """Load a union's children (choices).""" + names = _type.split()[1:] + return { + v["name"]: _map(v) + for v in flat.values(multi=True) + if v["name"] in names and v.get("in_record", False) + } + + def _fields() -> Vars: + """Load a record's children (fields).""" + names = _type.split()[1:] + return { + v["name"]: _map(v) + for v in flat.values(multi=True) + if v["name"] in names + and v.get("in_record", False) + and not v["type"].startswith("record") + } + + if _type.startswith("recarray"): + children = _items() + _type = "list" + elif _type.startswith("keystring"): children = _choices() _type = "union" - # record (sum) type elif _type.startswith("record"): children = _fields() _type = "record" - # at this point, if it has a shape, it's an array. check its type - elif shape is not None: - if _type not in _SCALARS: - raise TypeError(f"Unsupported array type: {_type}") + # for now, we can tell a var is an array if its type + # is scalar and it has a shape. once we have proper + # typing, this can be read off the type itself. + elif shape is not None and _type not in _SCALARS: + raise TypeError(f"Unsupported array type: {_type}") - # if the var is a foreign key, swap in the referenced variable - ref = refs.get(_name, None) - if not ref: + # if var is a foreign key, return subpkg var instead + if fkey: return Var( - name=_name, + name=fkey["param" if name == ("sim", "nam") else "val"], type=_type, shape=shape, block=block, - description=description, - default=default, - children=children, - meta={"ref": fkey}, + children=None, + description=( + f"* Contains data for the {fkey['abbr']} package. Data can be " + f"stored in a dictionary containing data for the {fkey['abbr']} " + "package with variable names as keys and package data as " + f"values. Data just for the {fkey['val']} variable is also " + f"acceptable. See {fkey['abbr']} package documentation for more " + "information" + ), + default=None, + fkey=fkey, ) + return Var( - name=ref["param" if name == ("sim", "nam") else "val"], + name=_name, type=_type, shape=shape, block=block, - description=( - f"* Contains data for the {ref['abbr']} package. Data can be " - f"stored in a dictionary containing data for the {ref['abbr']} " - "package with variable names as keys and package data as " - f"values. Data just for the {ref['val']} variable is also " - f"acceptable. See {ref['abbr']} package documentation for more " - "information" - ), - default=None, - children=None, - meta={"ref": ref}, + children=children, + description=description, + default=default, ) - # pass the original DFN representation as - # metadata so the shim can use it for now - _vars = list(flat.values(multi=True)) - - # convert input variable specs to - # structured form, descending into - # composites recursively as needed - vars_ = { - var["name"]: _map(var) - for var in flat.values(multi=True) - if not var.get("in_record", False) - } - return cls( - vars_, + { + var["name"]: _map(var) + for var in flat.values(multi=True) + if not var.get("in_record", False) + }, name, { - "dfn": (_vars, meta), - "refs": fkeys, + "dfn": ( + # pass the original DFN representation as + # metadata so templates can use it for now, + # eventually we can hopefully drop this + list(flat.values(multi=True)), + meta, + ), + "fkeys": fkeys, }, ) - @staticmethod - def load_all(dfndir: PathLike) -> Dict[str, "Dfn"]: - """Load all input definitions from the given directory.""" + @classmethod + def load( + cls, + f, + name: Optional[Name] = None, + version: Version = 1, + **kwargs, + ) -> "Dfn": + """ + Load an input definition from a DFN file. + """ + + if version == 1: + return cls._load_v1(f, name, **kwargs) + else: + raise ValueError( + f"Unsupported version, expected one of {version.__args__}" + ) + @staticmethod + def _load_all_v1(dfndir: PathLike) -> Dfns: # find definition files paths = [ p @@ -508,7 +510,7 @@ def load_all(dfndir: PathLike) -> Dict[str, "Dfn"]: common = None else: with open(common_path, "r") as f: - common, _ = Dfn._load(f) + common, _ = Dfn._load_v1_flat(f) # load subpackage references first refs: Refs = {} @@ -529,3 +531,14 @@ def load_all(dfndir: PathLike) -> Dict[str, "Dfn"]: dfns[name] = dfn return dfns + + @staticmethod + def load_all(dfndir: PathLike, version: Version = 1) -> Dfns: + """Load all input definitions from the given directory.""" + + if version == 1: + return Dfn._load_all_v1(dfndir) + else: + raise ValueError( + f"Unsupported version, expected one of {version.__args__}" + ) diff --git a/flopy/mf6/utils/codegen/filters.py b/flopy/mf6/utils/codegen/filters.py index d29a317719..d57496ec68 100644 --- a/flopy/mf6/utils/codegen/filters.py +++ b/flopy/mf6/utils/codegen/filters.py @@ -99,7 +99,7 @@ def skip(ctx, ctx_name) -> List[str]: ] elif base == "MFModel": skip = ["packages", "export_netcdf", "nc_filerecord"] - refs = ctx.get("meta", dict()).get("refs", dict()) + refs = ctx.get("meta", dict()).get("fkeys", dict()) if any(refs) and ctx["name"] != (None, "nam"): for k in refs.keys(): if ctx["vars"].get(k, None): @@ -175,7 +175,7 @@ def _attr(var: dict) -> Optional[str]: var_type = var["type"] var_shape = var.get("shape", None) var_block = var.get("block", None) - var_ref = var.get("meta", dict()).get("ref", None) + var_ref = var.get("fkey", None) if ( (var_type in _SCALARS and not var_shape) @@ -307,7 +307,6 @@ def _should_set(var: dict) -> bool: stmts = [] refs = {} for var in vars.values(): - ref = var.get("meta", dict()).get("ref", None) name = var["name"] if name in kwlist: name = f"{name}_" @@ -319,10 +318,12 @@ def _should_set(var: dict) -> bool: stmts.append( f"self.{name} = self.name_file.{name}" ) - if ref and ref["key"] not in refs: - refs[ref["key"]] = ref + + fkey = var.get("fkey", None) + if fkey and fkey["key"] not in refs: + refs[fkey["key"]] = fkey stmts.append( - f"self.{ref['param']} = self._create_package('{ref['abbr']}', {ref['param']})" + f"self.{fkey['param']} = self._create_package('{fkey['abbr']}', {fkey['param']})" ) elif base == "MFModel": @@ -336,7 +337,6 @@ def _should_set(var: dict) -> bool: stmts = [] refs = {} for var in vars.values(): - ref = var.get("meta", dict()).get("ref", None) name = var["name"] if name in kwlist: name = f"{name}_" @@ -348,23 +348,22 @@ def _should_set(var: dict) -> bool: stmts.append( f"self.{name} = self.name_file.{name}" ) - if ref and ref["key"] not in refs: - refs[ref["key"]] = ref + + fkey = var.get("fkey", None) + if fkey and fkey["key"] not in refs: + refs[fkey["key"]] = fkey stmts.append( - f"self.{ref['param']} = self._create_package('{ref['abbr']}', {ref['param']})" + f"self.{fkey['param']} = self._create_package('{fkey['abbr']}', {fkey['param']})" ) elif base == "MFPackage": def _should_build(var: dict) -> bool: - if var.get("meta", dict()).get( - "ref", None - ) and ctx_name != ( + if var.get("fkey", None) and ctx_name != ( None, "nam", ): return False - name = var["name"] - if name in [ + return var["name"] not in [ "simulation", "model", "package", @@ -380,22 +379,20 @@ def _should_build(var: dict) -> bool: "interpolation_method_single", "sfac", "output", - ]: - return False - return True + ] stmts = [] refs = {} for var in vars.values(): name = var["name"] - ref = var.get("meta", dict()).get("ref", None) if name in kwlist: name = f"{name}_" + fkey = var.get("fkey", None) if _should_build(var): - if ref and ctx["name"] == (None, "nam"): + if fkey and ctx["name"] == (None, "nam"): stmts.append( - f"self.{'_' if ref else ''}{ref['key']} = self.build_mfdata('{ref['key']}', None)" + f"self.{'_' if fkey else ''}{fkey['key']} = self.build_mfdata('{fkey['key']}', None)" ) else: _name = ( @@ -403,20 +400,20 @@ def _should_build(var: dict) -> bool: ) name = name.replace("-", "_") stmts.append( - f"self.{'_' if ref else ''}{name} = self.build_mfdata('{_name}', {name})" + f"self.{'_' if fkey else ''}{name} = self.build_mfdata('{_name}', {name})" ) if ( - ref - and ref["key"] not in refs + fkey + and fkey["key"] not in refs and ctx["name"].r != "nam" ): - refs[ref["key"]] = ref + refs[fkey["key"]] = fkey stmts.append( - f"self._{ref['key']} = self.build_mfdata('{ref['key']}', None)" + f"self._{fkey['key']} = self.build_mfdata('{fkey['key']}', None)" ) stmts.append( - f"self._{ref['abbr']}_package = self.build_child_package('{ref['abbr']}', {ref['val']}, '{ref['param']}', self._{ref['key']})" + f"self._{fkey['abbr']}_package = self.build_child_package('{fkey['abbr']}', {fkey['val']}, '{fkey['param']}', self._{fkey['key']})" ) return stmts diff --git a/flopy/mf6/utils/codegen/templates/macros.jinja b/flopy/mf6/utils/codegen/templates/macros.jinja index 2feb951415..eaaa63b521 100644 --- a/flopy/mf6/utils/codegen/templates/macros.jinja +++ b/flopy/mf6/utils/codegen/templates/macros.jinja @@ -1,15 +1,15 @@ {% macro init_vars(vars, alias=false, indent=0, skip=none) %} {%- for name, var in vars.items() if name not in skip %} -{% set v = var|maybe_file %} -{% set n = (name if alias else v.name)|nokw %} +{% set v = var|maybe_file -%} +{% set n = (name if alias else v.name)|nokw -%} {{ ""|indent(indent, first=true) }}{{ n }}{%- if v.default is defined %}={{ v.default|value }}{%- endif -%}, {%- endfor %} {% endmacro %} {% macro vars_docs(vars, indent=0) %} {%- for var in vars.values() recursive %} -{% set v = var|maybe_file %} -{% set n = v.name|nokw|escape_trailing_underscore %} +{% set v = var|maybe_file -%} +{% set n = v.name|nokw|escape_trailing_underscore -%} {{ ""|indent(indent, first=true) }}{% if loop.depth > 1 %}* {% endif %}{{ n }} : {{ v|type }} {%- if v.description is defined and v.description is not none %} {{ v.description|wordwrap|indent(indent + (loop.depth * 4), first=true) }} From 4f759d740d173d5426014ed7faa92c62453a10c5 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Tue, 5 Nov 2024 16:36:45 -0500 Subject: [PATCH 62/75] cleaner subpkg ref passing --- flopy/mf6/utils/codegen/__init__.py | 2 +- flopy/mf6/utils/codegen/context.py | 8 +- flopy/mf6/utils/codegen/dfn.py | 129 ++++++++---------- flopy/mf6/utils/codegen/filters.py | 10 +- .../mf6/utils/codegen/templates/macros.jinja | 4 +- .../utils/codegen/templates/package.py.jinja | 6 +- 6 files changed, 74 insertions(+), 85 deletions(-) diff --git a/flopy/mf6/utils/codegen/__init__.py b/flopy/mf6/utils/codegen/__init__.py index 51319b0853..649e15a09a 100644 --- a/flopy/mf6/utils/codegen/__init__.py +++ b/flopy/mf6/utils/codegen/__init__.py @@ -30,7 +30,7 @@ def _get_template_env(): env.filters["maybe_file"] = Filters.Var.maybe_file env.filters["type"] = Filters.Var.type - env.filters["nokw"] = Filters.nokw + env.filters["safe_str"] = Filters.safe_str env.filters["escape_trailing_underscore"] = ( Filters.escape_trailing_underscore ) diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py index 7c6288f782..1471044729 100644 --- a/flopy/mf6/utils/codegen/context.py +++ b/flopy/mf6/utils/codegen/context.py @@ -90,9 +90,7 @@ def from_dfn(cls, dfn: Dfn) -> Iterator["Context"]: These are structured representations of input context classes. Each input definition yields one or more input contexts. """ - meta = dfn.meta.copy() - ref = Ref.from_dfn(dfn) - if ref: - meta["ref"] = ref for name in Context.Name.from_dfn(dfn): - yield Context(name=name, vars=dfn.data, meta=meta) + yield Context( + name=name, vars=dfn.data.copy(), meta=dfn.meta.copy() + ) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index e6217e9155..59c3902a74 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -85,69 +85,6 @@ class Ref(TypedDict): parent: str description: Optional[str] - @classmethod - def from_dfn(cls, dfn: "Dfn") -> Optional["Ref"]: - """ - Try to load a reference from the definition. - Returns `None` if the definition cannot be - referenced by other contexts. - """ - - if not dfn.meta or "dfn" not in dfn.meta: - return None - - _, meta = dfn.meta["dfn"] - - lines = { - "subpkg": next( - iter( - m - for m in meta - if isinstance(m, str) and m.startswith("subpac") - ), - None, - ), - "parent": next( - iter( - m - for m in meta - if isinstance(m, str) and m.startswith("parent") - ), - None, - ), - } - - def _subpkg(): - line = lines["subpkg"] - _, key, abbr, param, val = line.split() - matches = [v for v in dfn.values() if v["name"] == val] - if not any(matches): - descr = None - else: - if len(matches) > 1: - warn(f"Multiple matches for referenced variable {val}") - match = matches[0] - descr = match["description"] - - return { - "key": key, - "val": val, - "abbr": abbr, - "param": param, - "description": descr, - } - - def _parent(): - line = lines["parent"] - split = line.split() - return split[1] - - return ( - cls(**_subpkg(), parent=_parent()) - if all(v for v in lines.values()) - else None - ) - class Dfn(UserDict): """ @@ -457,12 +394,65 @@ def _fields() -> Vars: default=default, ) + vars_ = { + var["name"]: _map(var) + for var in flat.values(multi=True) + if not var.get("in_record", False) + } + + def _subpkg() -> Optional["Ref"]: + lines = { + "subpkg": next( + iter( + m + for m in meta + if isinstance(m, str) and m.startswith("subpac") + ), + None, + ), + "parent": next( + iter( + m + for m in meta + if isinstance(m, str) and m.startswith("parent") + ), + None, + ), + } + + def __subpkg(): + line = lines["subpkg"] + _, key, abbr, param, val = line.split() + matches = [v for v in vars_.values() if v["name"] == val] + if not any(matches): + descr = None + else: + if len(matches) > 1: + warn(f"Multiple matches for referenced variable {val}") + match = matches[0] + descr = match["description"] + + return { + "key": key, + "val": val, + "abbr": abbr, + "param": param, + "description": descr, + } + + def _parent(): + line = lines["parent"] + split = line.split() + return split[1] + + return ( + Ref(**__subpkg(), parent=_parent()) + if all(v for v in lines.values()) + else None + ) + return cls( - { - var["name"]: _map(var) - for var in flat.values(multi=True) - if not var.get("in_record", False) - }, + vars_, name, { "dfn": ( @@ -473,6 +463,7 @@ def _fields() -> Vars: meta, ), "fkeys": fkeys, + "subpkg": _subpkg(), }, ) @@ -518,7 +509,7 @@ def _load_all_v1(dfndir: PathLike) -> Dfns: name = Dfn.Name(*path.stem.split("-")) with open(path) as f: dfn = Dfn.load(f, name=name, common=common) - ref = Ref.from_dfn(dfn) + ref = dfn.meta.get("subpkg", None) if ref: refs[ref["key"]] = ref diff --git a/flopy/mf6/utils/codegen/filters.py b/flopy/mf6/utils/codegen/filters.py index d57496ec68..5ca9fb9024 100644 --- a/flopy/mf6/utils/codegen/filters.py +++ b/flopy/mf6/utils/codegen/filters.py @@ -69,9 +69,9 @@ def prefix(ctx_name) -> str: @pass_context def parent(ctx, ctx_name) -> str: - ref = ctx["meta"].get("ref", None) - if ref: - return ref["parent"] + subpkg = ctx["meta"].get("subpkg", None) + if subpkg: + return subpkg["parent"] if ctx_name == ("sim", "nam"): return None elif ( @@ -80,7 +80,7 @@ def parent(ctx, ctx_name) -> str: or ctx_name.l in ["sim", "exg", "sln"] ): return "simulation" - elif ref: + elif subpkg: if ctx_name.l == "utl" and ctx_name.r == "hpc": return "simulation" return "package" @@ -420,7 +420,7 @@ def _should_build(var: dict) -> bool: return list(filter(None, _statements())) - def nokw(v: str) -> str: + def safe_str(v: str) -> str: return (f"{v}_" if v in kwlist else v).replace("-", "_") def escape_trailing_underscore(v: str) -> str: diff --git a/flopy/mf6/utils/codegen/templates/macros.jinja b/flopy/mf6/utils/codegen/templates/macros.jinja index eaaa63b521..990ee8c84e 100644 --- a/flopy/mf6/utils/codegen/templates/macros.jinja +++ b/flopy/mf6/utils/codegen/templates/macros.jinja @@ -1,7 +1,7 @@ {% macro init_vars(vars, alias=false, indent=0, skip=none) %} {%- for name, var in vars.items() if name not in skip %} {% set v = var|maybe_file -%} -{% set n = (name if alias else v.name)|nokw -%} +{% set n = (name if alias else v.name)|safe_str -%} {{ ""|indent(indent, first=true) }}{{ n }}{%- if v.default is defined %}={{ v.default|value }}{%- endif -%}, {%- endfor %} {% endmacro %} @@ -9,7 +9,7 @@ {% macro vars_docs(vars, indent=0) %} {%- for var in vars.values() recursive %} {% set v = var|maybe_file -%} -{% set n = v.name|nokw|escape_trailing_underscore -%} +{% set n = v.name|safe_str|escape_trailing_underscore -%} {{ ""|indent(indent, first=true) }}{% if loop.depth > 1 %}* {% endif %}{{ n }} : {{ v|type }} {%- if v.description is defined and v.description is not none %} {{ v.description|wordwrap|indent(indent + (loop.depth * 4), first=true) }} diff --git a/flopy/mf6/utils/codegen/templates/package.py.jinja b/flopy/mf6/utils/codegen/templates/package.py.jinja index a763e81eb0..f3776ad2cd 100644 --- a/flopy/mf6/utils/codegen/templates/package.py.jinja +++ b/flopy/mf6/utils/codegen/templates/package.py.jinja @@ -72,7 +72,7 @@ class Modflow{{ title }}(MFPackage): self._init_complete = True -{% if "ref" in meta and name.r != "hpc" %} +{% if "subpkg" in meta and name.r != "hpc" %} class {{ title }}Packages(MFChildPackages): """ {{ title }}Packages is a container class for the Modflow{{ title }} class. @@ -101,7 +101,7 @@ class {{ title }}Packages(MFChildPackages): new_package = Modflow{{ title }}( self._cpparent, {%- for n, var in vars.items() if n not in name|skip %} - {{ n }}={{ n }}, + {{ n|safe_str }}={{ n|safe_str }}, {%- endfor %} filename=filename, pname=pname, @@ -119,7 +119,7 @@ class {{ title }}Packages(MFChildPackages): new_package = Modflow{{ title }}( self._cpparent, {%- for n, var in vars.items() if n not in name|skip %} - {{ n }}={{ n }}, + {{ n|safe_str }}={{ n|safe_str }}, {%- endfor %} filename=filename, pname=pname, From bd08c554302142b1f35b85f1058ac72342275ac1 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 6 Nov 2024 12:43:27 -0500 Subject: [PATCH 63/75] add new deps to environment.yml --- etc/environment.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/etc/environment.yml b/etc/environment.yml index 1128785219..c8ca1a935e 100644 --- a/etc/environment.yml +++ b/etc/environment.yml @@ -10,6 +10,13 @@ dependencies: - matplotlib>=1.4.0 - pandas>=2.0.0 + # codegen + - boltons>=1.0 + - Jinja2>=3.0 + - pip: + - git+https://github.com/MODFLOW-USGS/modflow-devtools.git + - tomlkit + # lint - cffconvert - ruff @@ -21,8 +28,6 @@ dependencies: - jupyter - jupyter_client>=8.4.0 - jupytext - - pip: - - git+https://github.com/MODFLOW-USGS/modflow-devtools.git - pytest!=8.1.0 - pytest-benchmark - pytest-cov From ad46ed51ccb4c2862a00c2a3773006782e7856f1 Mon Sep 17 00:00:00 2001 From: w-bonelli Date: Wed, 6 Nov 2024 21:44:08 -0500 Subject: [PATCH 64/75] fix filein/fileout in toml conversion --- flopy/mf6/utils/codegen/dfn.py | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index 59c3902a74..9420e815b0 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -215,6 +215,7 @@ def _load_v1(cls, f, name, **kwargs) -> "Dfn": flat, meta = Dfn._load_v1_flat(f, **kwargs) refs = kwargs.pop("refs", dict()) fkeys = dict() + popped = dict() def _map(spec: Dict[str, Any]) -> Var: """ @@ -336,15 +337,31 @@ def _choices() -> Vars: } def _fields() -> Vars: - """Load a record's children (fields).""" + """ + Load a record's children (fields). + + Notes + ----- + This includes a hack to handle cases where `filein` or `fileout` + is defined just once in a DFN file, where in the new structured + format it is expected wherever it appears. + """ names = _type.split()[1:] - return { - v["name"]: _map(v) - for v in flat.values(multi=True) - if v["name"] in names - and v.get("in_record", False) - and not v["type"].startswith("record") - } + fields = dict() + for name in names: + v = popped.get(name, None) + if v: + fields[name] = v + continue + v = flat.get(name, None) + if ( + not v + or not v.get("in_record", False) + or v["type"].startswith("record") + ): + continue + fields[name] = v + return fields if _type.startswith("recarray"): children = _items() From 4ecb55013f3be9e6948587e9d9029fa4a5c307e5 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 7 Nov 2024 08:25:57 -0500 Subject: [PATCH 65/75] switch dfn dataclass -> typeddict, owns-a (not is-a) var dict, clearer member naming --- autotest/test_codegen.py | 21 +- flopy/mf6/utils/codegen/__init__.py | 5 +- flopy/mf6/utils/codegen/context.py | 42 +-- flopy/mf6/utils/codegen/dfn.py | 275 ++++++++++-------- flopy/mf6/utils/codegen/dfn2toml.py | 24 +- flopy/mf6/utils/codegen/filters.py | 99 ++++--- .../utils/codegen/templates/package.py.jinja | 2 +- 7 files changed, 248 insertions(+), 220 deletions(-) diff --git a/autotest/test_codegen.py b/autotest/test_codegen.py index 7ee07bc72e..453d78ba0f 100644 --- a/autotest/test_codegen.py +++ b/autotest/test_codegen.py @@ -1,10 +1,8 @@ import pytest from autotest.conftest import get_project_root_path -from flopy.mf6.utils.codegen import make_all, make_targets -from flopy.mf6.utils.codegen.context import Context +from flopy.mf6.utils.codegen import make_all from flopy.mf6.utils.codegen.dfn import Dfn -from flopy.mf6.utils.codegen.filters import Filters PROJ_ROOT = get_project_root_path() MF6_PATH = PROJ_ROOT / "flopy" / "mf6" @@ -27,23 +25,6 @@ def test_dfn_load(dfn_name): Dfn.load(dfn_file, name=name, common=common) -@pytest.mark.parametrize("dfn_name", DFN_NAMES) -def test_make_targets(dfn_name, function_tmpdir): - with ( - open(DFN_PATH / "common.dfn", "r") as common_file, - open(DFN_PATH / f"{dfn_name}.dfn", "r") as dfn_file, - ): - name = Dfn.Name.parse(dfn_name) - common, _ = Dfn._load_v1_flat(common_file) - dfn = Dfn.load(dfn_file, name=name, common=common) - - make_targets(dfn, function_tmpdir, verbose=True) - assert all( - (function_tmpdir / f"mf{Filters.Cls.title(name)}.py").is_file() - for name in Context.Name.from_dfn(dfn) - ) - - def test_make_all(function_tmpdir): make_all(DFN_PATH, function_tmpdir, verbose=True) assert any(function_tmpdir.glob("*.py")) diff --git a/flopy/mf6/utils/codegen/__init__.py b/flopy/mf6/utils/codegen/__init__.py index 649e15a09a..e586a652d6 100644 --- a/flopy/mf6/utils/codegen/__init__.py +++ b/flopy/mf6/utils/codegen/__init__.py @@ -1,4 +1,3 @@ -from dataclasses import asdict from itertools import chain from os import PathLike from pathlib import Path @@ -83,12 +82,12 @@ def _get_template_name(ctx_name) -> str: return "package.py.jinja" for context in Context.from_dfn(dfn): - name = context.name + name = context["name"] target_path = outdir / f"mf{Filters.Cls.title(name)}.py" template_name = _get_template_name(name) template = env.get_template(template_name) with open(target_path, "w") as f: - f.write(template.render(**asdict(context))) + f.write(template.render(**context)) if verbose: print(f"Wrote {target_path}") diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py index 1471044729..b597b94989 100644 --- a/flopy/mf6/utils/codegen/context.py +++ b/flopy/mf6/utils/codegen/context.py @@ -1,18 +1,15 @@ -from dataclasses import dataclass from typing import ( - Any, - Dict, Iterator, List, NamedTuple, Optional, + TypedDict, ) -from flopy.mf6.utils.codegen.dfn import Dfn, Ref, Vars +from flopy.mf6.utils.codegen.dfn import Dfn, Vars -@dataclass -class Context: +class Context(TypedDict): """ An input context. Each of these is specified by a definition file and becomes a generated class. A definition file may specify more @@ -56,32 +53,32 @@ def from_dfn(dfn: Dfn) -> List["Context.Name"]: Returns a list of context names this definition produces. An input definition may produce one or more input contexts. """ - if dfn.name.r == "nam": - if dfn.name.l == "sim": + name = dfn["name"] + if name.r == "nam": + if name.l == "sim": return [ - Context.Name(None, dfn.name.r), # nam pkg - Context.Name(*dfn.name), # simulation + Context.Name(None, name.r), # nam pkg + Context.Name(*name), # simulation ] else: return [ - Context.Name(*dfn.name), # nam pkg - Context.Name(dfn.name.l, None), # model + Context.Name(*name), # nam pkg + Context.Name(name.l, None), # model ] - elif dfn.name in [ + elif name in [ ("gwf", "mvr"), ("gwf", "gnc"), ("gwt", "mvt"), ]: # TODO: remove special cases, deduplicate mfmvr.py/mfgwfmvr.py etc return [ - Context.Name(*dfn.name), - Context.Name(None, dfn.name.r), + Context.Name(*name), + Context.Name(None, name.r), ] - return [Context.Name(*dfn.name)] + return [Context.Name(*name)] name: Name vars: Vars - meta: Optional[Dict[str, Any]] = None @classmethod def from_dfn(cls, dfn: Dfn) -> Iterator["Context"]: @@ -90,7 +87,12 @@ def from_dfn(cls, dfn: Dfn) -> Iterator["Context"]: These are structured representations of input context classes. Each input definition yields one or more input contexts. """ + + def _to_context(n, d): + d = d.copy() + d.pop("name", None) + vars_ = d.pop("vars", dict()) + return Context(name=n, vars=vars_, **d) + for name in Context.Name.from_dfn(dfn): - yield Context( - name=name, vars=dfn.data.copy(), meta=dfn.meta.copy() - ) + yield _to_context(name, dfn) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index 9420e815b0..42c6af00c0 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -1,5 +1,5 @@ from ast import literal_eval -from collections import UserDict +from collections.abc import Mapping from os import PathLike from typing import ( Any, @@ -15,8 +15,6 @@ from boltons.dictutils import OMD -from flopy.utils.utl_import import import_optional_dependency - _SCALARS = { "keyword", "integer", @@ -61,21 +59,19 @@ class Var(TypedDict): shape: Optional[Any] = None block: Optional[str] = None default: Optional[Any] = None - children: Optional[Vars] = None + children: Optional["Vars"] = None description: Optional[str] = None class Ref(TypedDict): """ - A foreign-key-like reference between a file input variable + This class is used to represent subpackage references: + a foreign-key-like reference between a file input variable and another input definition. This allows an input context - to refer to another input context, by including a filepath - variable whose name acts as a foreign key for a different - input context. The referring context's `__init__` method + to refer to another input context by including a filepath + variable as a foreign key. The former's `__init__` method is modified such that the variable named `val` replaces the `key` variable. - - This class is used to represent subpackage references. """ key: str @@ -86,7 +82,16 @@ class Ref(TypedDict): description: Optional[str] -class Dfn(UserDict): +class Sln(TypedDict): + abbr: str + pattern: str + + +DfnFmtVersion = Literal[1] +"""DFN format version number.""" + + +class Dfn(TypedDict): """ MODFLOW 6 input definition. An input definition file specifies a component of an MF6 simulation, @@ -112,25 +117,13 @@ def parse(cls, v: str) -> "Dfn.Name": def __str__(self) -> str: return "-".join(self) - Version = Literal[1] - - name: Optional[Name] - meta: Optional[Dict[str, Any]] - - def __init__( - self, - data: Optional[Vars] = None, - name: Optional[Name] = None, - meta: Optional[Dict[str, Any]] = None, - ): - self.data = data or dict() - self.name = name - self.meta = meta + name: Name + vars: Vars @staticmethod def _load_v1_flat( - f, common: Optional[dict] = None, **kwargs - ) -> Tuple[OMD, List[str]]: + f, common: Optional[dict] = None + ) -> Tuple[Mapping, List[str]]: var = dict() flat = list() meta = list() @@ -145,19 +138,16 @@ def _load_v1_flat( if line.startswith("#"): _, sep, tail = line.partition("flopy") if sep == "flopy": - tail = tail.strip() - if "solution_package" in tail: - tail = tail.split() - tail.pop(1) - meta.append(tail) - continue + if ( + "multi-package" in tail + or "solution_package" in tail + or "subpackage" in tail + or "parent" in tail + ): + meta.append(tail.strip()) _, sep, tail = line.partition("package-type") if sep == "package-type": - if meta is None: - meta = list - meta.append(f"{sep} {tail.strip()}") - continue - _, sep, tail = line.partition("solution_package") + meta.append(f"package-type {tail.strip()}") continue # if we hit a newline and the parameter dict @@ -212,15 +202,44 @@ def _load_v1_flat( @classmethod def _load_v1(cls, f, name, **kwargs) -> "Dfn": - flat, meta = Dfn._load_v1_flat(f, **kwargs) + """ + Temporary load routine for the v1 DFN format. + This can go away once we convert to v2 (TOML). + """ + + # if we have any subpackage references + # we need to watch for foreign key vars + # (file input vars) and register matches refs = kwargs.pop("refs", dict()) fkeys = dict() - popped = dict() - def _map(spec: Dict[str, Any]) -> Var: + # load dfn as flat multidict and str metadata + flat, meta = Dfn._load_v1_flat(f, **kwargs) + + # pass the original dfn representation on + # the dfn since it is reproduced verbatim + # in generated classes for now. drop this + # later when we figure out how to unravel + # mfstructure.py etc + def _meta(): + meta_ = list() + for m in meta: + if "multi" in m: + meta_.append(m) + elif "solution" in m: + s = m.split() + meta_.append([s[0], s[2]]) + elif "package-type" in m: + s = m.split() + meta_.append(" ".join(s)) + return meta_ + + dfn = list(flat.values(multi=True)), _meta() + + def _load_variable(var: Dict[str, Any]) -> Var: """ - Convert an input variable specification from its shape - in a classic definition file to a Python-friendly form. + Convert an input variable from its original representation + in a definition file to a structured, Python-friendly form. This involves trimming unneeded attributes and setting some others. @@ -232,31 +251,30 @@ def _map(spec: Dict[str, Any]) -> Var: A filepath variable whose name functions as a foreign key for a separate context will be given a reference to it. - """ # parse booleans from strings. everything else can # stay a string except default values, which we'll # try to parse as arbitrary literals below, and at # some point types, once we introduce type hinting - spec = {k: _try_parse_bool(v) for k, v in spec.items()} + var = {k: _try_parse_bool(v) for k, v in var.items()} - _name = spec["name"] - _type = spec.get("type", None) - shape = spec.get("shape", None) + _name = var["name"] + _type = var.get("type", None) + shape = var.get("shape", None) shape = None if shape == "" else shape - block = spec.get("block", None) + block = var.get("block", None) children = dict() - default = spec.get("default", None) + default = var.get("default", None) default = ( _try_literal_eval(default) if _type != "string" else default ) - description = spec.get("description", "") - fkey = refs.get(_name, None) + description = var.get("description", "") + ref = refs.get(_name, None) # if var is a foreign key, register it - if fkey: - fkeys[_name] = fkey + if ref: + fkeys[_name] = ref def _items() -> Vars: """Load a list's children (items: record or union of records).""" @@ -285,7 +303,7 @@ def _items() -> Vars: if is_explicit: child = next(iter(flat.getlist(names[0]))) - return {names[0]: _map(child)} + return {names[0]: _load_variable(child)} elif all(t in _SCALARS for t in types): # implicit simple record (all fields are scalars) fields = _fields() @@ -303,7 +321,7 @@ def _items() -> Vars: else: # implicit complex record (some fields are records or unions) fields = { - v["name"]: _map(v) + v["name"]: _load_variable(v) for v in flat.values(multi=True) if v["name"] in names and v.get("in_record", False) } @@ -331,28 +349,16 @@ def _choices() -> Vars: """Load a union's children (choices).""" names = _type.split()[1:] return { - v["name"]: _map(v) + v["name"]: _load_variable(v) for v in flat.values(multi=True) if v["name"] in names and v.get("in_record", False) } def _fields() -> Vars: - """ - Load a record's children (fields). - - Notes - ----- - This includes a hack to handle cases where `filein` or `fileout` - is defined just once in a DFN file, where in the new structured - format it is expected wherever it appears. - """ + """Load a record's children (fields).""" names = _type.split()[1:] fields = dict() for name in names: - v = popped.get(name, None) - if v: - fields[name] = v - continue v = flat.get(name, None) if ( not v @@ -382,23 +388,23 @@ def _fields() -> Vars: raise TypeError(f"Unsupported array type: {_type}") # if var is a foreign key, return subpkg var instead - if fkey: + if ref: return Var( - name=fkey["param" if name == ("sim", "nam") else "val"], + name=ref["param" if name == ("sim", "nam") else "val"], type=_type, shape=shape, block=block, children=None, description=( - f"* Contains data for the {fkey['abbr']} package. Data can be " - f"stored in a dictionary containing data for the {fkey['abbr']} " + f"* Contains data for the {ref['abbr']} package. Data can be " + f"stored in a dictionary containing data for the {ref['abbr']} " "package with variable names as keys and package data as " - f"values. Data just for the {fkey['val']} variable is also " - f"acceptable. See {fkey['abbr']} package documentation for more " + f"values. Data just for the {ref['val']} variable is also " + f"acceptable. See {ref['abbr']} package documentation for more " "information" ), default=None, - fkey=fkey, + subpackage=ref, ) return Var( @@ -411,34 +417,51 @@ def _fields() -> Vars: default=default, ) + # load top-level variables. any nested + # variables will be loaded recursively vars_ = { - var["name"]: _map(var) + var["name"]: _load_variable(var) for var in flat.values(multi=True) if not var.get("in_record", False) } - def _subpkg() -> Optional["Ref"]: - lines = { - "subpkg": next( + def _package_type() -> Optional[str]: + line = next( + iter( + m + for m in meta + if isinstance(m, str) and m.startswith("package-type") + ), + None, + ) + return line.split()[-1] if line else None + + def _subpackage() -> Optional["Ref"]: + def _parent(): + line = next( iter( m for m in meta - if isinstance(m, str) and m.startswith("subpac") + if isinstance(m, str) and m.startswith("parent") ), None, - ), - "parent": next( + ) + if not line: + return None + split = line.split() + return split[1] + + def _rest(): + line = next( iter( m for m in meta - if isinstance(m, str) and m.startswith("parent") + if isinstance(m, str) and m.startswith("subpac") ), None, - ), - } - - def __subpkg(): - line = lines["subpkg"] + ) + if not line: + return None _, key, abbr, param, val = line.split() matches = [v for v in vars_.values() if v["name"] == val] if not any(matches): @@ -457,31 +480,43 @@ def __subpkg(): "description": descr, } - def _parent(): - line = lines["parent"] - split = line.split() - return split[1] - - return ( - Ref(**__subpkg(), parent=_parent()) - if all(v for v in lines.values()) - else None + parent = _parent() + rest = _rest() + if parent and rest: + return Ref(parent=parent, **rest) + return None + + def _solution() -> Optional[Sln]: + sln = next( + iter( + m + for m in meta + if isinstance(m, str) and m.startswith("solution_package") + ), + None, ) + if sln: + abbr, pattern = sln.split()[1:] + return Sln(abbr=abbr, pattern=pattern) + return None + + def _multi() -> bool: + return any("multi-package" in m for m in meta) + + package_type = _package_type() + subpackage = _subpackage() + solution = _solution() + multi = _multi() return cls( - vars_, - name, - { - "dfn": ( - # pass the original DFN representation as - # metadata so templates can use it for now, - # eventually we can hopefully drop this - list(flat.values(multi=True)), - meta, - ), - "fkeys": fkeys, - "subpkg": _subpkg(), - }, + name=name, + vars=vars_, + dfn=dfn, + foreign_keys=fkeys, + package_type=package_type, + subpackage=subpackage, + solution=solution, + multi=multi, ) @classmethod @@ -489,7 +524,7 @@ def load( cls, f, name: Optional[Name] = None, - version: Version = 1, + version: DfnFmtVersion = 1, **kwargs, ) -> "Dfn": """ @@ -523,25 +558,25 @@ def _load_all_v1(dfndir: PathLike) -> Dfns: # load subpackage references first refs: Refs = {} for path in paths: - name = Dfn.Name(*path.stem.split("-")) with open(path) as f: + name = Dfn.Name.parse(path.stem) dfn = Dfn.load(f, name=name, common=common) - ref = dfn.meta.get("subpkg", None) - if ref: - refs[ref["key"]] = ref + subpkg = dfn.get("subpackage", None) + if subpkg: + refs[subpkg["key"]] = subpkg # load all the input definitions dfns: Dfns = {} for path in paths: - name = Dfn.Name(*path.stem.split("-")) with open(path) as f: - dfn = Dfn.load(f, name=name, refs=refs, common=common) + name = Dfn.Name.parse(path.stem) + dfn = Dfn.load(f, name=name, common=common, refs=refs) dfns[name] = dfn return dfns @staticmethod - def load_all(dfndir: PathLike, version: Version = 1) -> Dfns: + def load_all(dfndir: PathLike, version: DfnFmtVersion = 1) -> Dfns: """Load all input definitions from the given directory.""" if version == 1: diff --git a/flopy/mf6/utils/codegen/dfn2toml.py b/flopy/mf6/utils/codegen/dfn2toml.py index 98e835a4be..9f0caaebca 100644 --- a/flopy/mf6/utils/codegen/dfn2toml.py +++ b/flopy/mf6/utils/codegen/dfn2toml.py @@ -1,8 +1,7 @@ import argparse +from collections.abc import Mapping from pathlib import Path -from boltons.iterutils import remap - from flopy.utils import import_optional_dependency _MF6_PATH = Path(__file__).parents[2] @@ -10,8 +9,21 @@ _TOML_PATH = _MF6_PATH / "data" / "toml" -def _drop_none(d: dict) -> dict: - return remap(d, lambda p, k, v: isinstance(v, bool) or bool(v)) +def _drop_none(d): + if isinstance(d, Mapping): + return { + k: _drop_none(v) + for k, v in d.items() + if (v or isinstance(v, bool)) + } + else: + return d + + +def _shim(d): + del d["dfn"] + d["name"] = str(d["name"]) + return d if __name__ == "__main__": @@ -37,5 +49,5 @@ def _drop_none(d: dict) -> dict: outdir = Path(args.outdir) outdir.mkdir(exist_ok=True, parents=True) for dfn in Dfn.load_all(dfndir).values(): - with open(Path(outdir) / f"{dfn.name}.toml", "w") as f: - tomlkit.dump(_drop_none(dfn), f) + with open(Path(outdir) / f"{dfn['name']}.toml", "w") as f: + tomlkit.dump(_drop_none(_shim(dfn)), f) diff --git a/flopy/mf6/utils/codegen/filters.py b/flopy/mf6/utils/codegen/filters.py index 5ca9fb9024..d1a4b1f1e8 100644 --- a/flopy/mf6/utils/codegen/filters.py +++ b/flopy/mf6/utils/codegen/filters.py @@ -69,7 +69,7 @@ def prefix(ctx_name) -> str: @pass_context def parent(ctx, ctx_name) -> str: - subpkg = ctx["meta"].get("subpkg", None) + subpkg = ctx.get("subpackage", None) if subpkg: return subpkg["parent"] if ctx_name == ("sim", "nam"): @@ -99,7 +99,7 @@ def skip(ctx, ctx_name) -> List[str]: ] elif base == "MFModel": skip = ["packages", "export_netcdf", "nc_filerecord"] - refs = ctx.get("meta", dict()).get("fkeys", dict()) + refs = ctx.get("foreign_keys", dict()) if any(refs) and ctx["name"] != (None, "nam"): for k in refs.keys(): if ctx["vars"].get(k, None): @@ -166,30 +166,31 @@ def type(var: dict) -> str: class Vars: @pass_context - def attrs(ctx, vars) -> List[str]: - ctx_name = ctx["name"] - base = Filters.Cls.base(ctx_name) + def attrs(ctx, variables) -> List[str]: + name = ctx["name"] + base = Filters.Cls.base(name) def _attr(var: dict) -> Optional[str]: var_name = var["name"] var_type = var["type"] var_shape = var.get("shape", None) var_block = var.get("block", None) - var_ref = var.get("fkey", None) + var_subpkg = var.get("subpackage", None) if ( (var_type in _SCALARS and not var_shape) or var_name in ["cvoptions", "output"] - or (ctx_name.r == "dis" and var_name == "packagedata") + or (name.r == "dis" and var_name == "packagedata") or ( var_name != "packages" - and (ctx_name.l is not None and ctx_name.r == "nam") + and (name.l is not None and name.r == "nam") ) ): return None is_array = ( - var_type in ["integer", "double precision"] and var_shape + var_type in ["string", "integer", "double precision"] + and var_shape ) is_composite = var_type in ["list", "record", "union"] if is_array or is_composite: @@ -197,36 +198,36 @@ def _attr(var: dict) -> Optional[str]: raise ValueError("Need block") if not is_array: - if var_ref: + if var_subpkg: # if the variable is a subpackage reference, use the original key # (which has been replaced already with the referenced variable) args = [ - f"'{ctx_name.r}'", + f"'{name.r}'", f"'{var_block}'", - f"'{var_ref['key']}'", + f"'{var_subpkg['key']}'", ] - if ctx_name.l is not None and ctx_name.l not in [ + if name.l is not None and name.l not in [ "sim", "sln", "utl", "exg", ]: - args.insert(0, f"'{ctx_name.l}6'") - return f"{var_ref['key']} = ListTemplateGenerator(({', '.join(args)}))" + args.insert(0, f"'{name.l}6'") + return f"{var_subpkg['key']} = ListTemplateGenerator(({', '.join(args)}))" def _args(): args = [ - f"'{ctx_name.r}'", + f"'{name.r}'", f"'{var_block}'", f"'{var_name}'", ] - if ctx_name.l is not None and ctx_name.l not in [ + if name.l is not None and name.l not in [ "sim", "sln", "utl", "exg", ]: - args.insert(0, f"'{ctx_name.l}6'") + args.insert(0, f"'{name.l}6'") return args kind = "array" if is_array else "list" @@ -235,7 +236,7 @@ def _args(): return None def _dfn() -> List[List[str]]: - dfn, meta = ctx["meta"]["dfn"] + dfn, meta = ctx["dfn"] def _meta(): exclude = ["subpackage", "parent_name_type"] @@ -247,7 +248,7 @@ def _dfn(): def _var(var: dict) -> List[str]: exclude = ["longname", "description"] name = var["name"] - var_ = vars.get(name, None) + var_ = variables.get(name, None) keys = [ "construct_package", "construct_data", @@ -266,20 +267,20 @@ def _var(var: dict) -> List[str]: return [["header"] + _meta()] + _dfn() - attrs = list(filter(None, [_attr(v) for v in vars.values()])) + attrs = list(filter(None, [_attr(v) for v in variables.values()])) if base == "MFModel": - attrs.append(f"model_type = {ctx_name.l}") + attrs.append(f"model_type = {name.l}") elif base == "MFPackage": attrs.extend( [ - f"package_abbr = '{ctx_name.r}'" - if ctx_name.l == "exg" - else f"package_abbr = '{'' if ctx_name.l in ['sln', 'sim', 'exg', None] else ctx_name.l}{ctx_name.r}'", - f"_package_type = '{ctx_name.r}'", - f"dfn_file_name = '{ctx_name.l}-{ctx_name.r}.dfn'" - if ctx_name.l == "exg" - else f"dfn_file_name = '{ctx_name.l or 'sim'}-{ctx_name.r}.dfn'", + f"package_abbr = '{name.r}'" + if name.l == "exg" + else f"package_abbr = '{'' if name.l in ['sln', 'sim', 'exg', None] else name.l}{name.r}'", + f"_package_type = '{name.r}'", + f"dfn_file_name = '{name.l}-{name.r}.dfn'" + if name.l == "exg" + else f"dfn_file_name = '{name.l or 'sim'}-{name.r}.dfn'", f"dfn = {pformat(_dfn(), indent=10)}", ] ) @@ -319,11 +320,11 @@ def _should_set(var: dict) -> bool: f"self.{name} = self.name_file.{name}" ) - fkey = var.get("fkey", None) - if fkey and fkey["key"] not in refs: - refs[fkey["key"]] = fkey + subpkg = var.get("subpackage", None) + if subpkg and subpkg["key"] not in refs: + refs[subpkg["key"]] = subpkg stmts.append( - f"self.{fkey['param']} = self._create_package('{fkey['abbr']}', {fkey['param']})" + f"self.{subpkg['param']} = self._create_package('{subpkg['abbr']}', {subpkg['param']})" ) elif base == "MFModel": @@ -349,19 +350,17 @@ def _should_set(var: dict) -> bool: f"self.{name} = self.name_file.{name}" ) - fkey = var.get("fkey", None) - if fkey and fkey["key"] not in refs: - refs[fkey["key"]] = fkey + subpkg = var.get("subpackage", None) + if subpkg and subpkg["key"] not in refs: + refs[subpkg["key"]] = subpkg stmts.append( - f"self.{fkey['param']} = self._create_package('{fkey['abbr']}', {fkey['param']})" + f"self.{subpkg['param']} = self._create_package('{subpkg['abbr']}', {subpkg['param']})" ) elif base == "MFPackage": def _should_build(var: dict) -> bool: - if var.get("fkey", None) and ctx_name != ( - None, - "nam", - ): + subpkg = var.get("subpackage", None) + if subpkg and ctx_name != (None, "nam"): return False return var["name"] not in [ "simulation", @@ -388,11 +387,11 @@ def _should_build(var: dict) -> bool: if name in kwlist: name = f"{name}_" - fkey = var.get("fkey", None) + subpkg = var.get("subpackage", None) if _should_build(var): - if fkey and ctx["name"] == (None, "nam"): + if subpkg and ctx["name"] == (None, "nam"): stmts.append( - f"self.{'_' if fkey else ''}{fkey['key']} = self.build_mfdata('{fkey['key']}', None)" + f"self.{'_' if subpkg else ''}{subpkg['key']} = self.build_mfdata('{subpkg['key']}', None)" ) else: _name = ( @@ -400,20 +399,20 @@ def _should_build(var: dict) -> bool: ) name = name.replace("-", "_") stmts.append( - f"self.{'_' if fkey else ''}{name} = self.build_mfdata('{_name}', {name})" + f"self.{'_' if subpkg else ''}{name} = self.build_mfdata('{_name}', {name})" ) if ( - fkey - and fkey["key"] not in refs + subpkg + and subpkg["key"] not in refs and ctx["name"].r != "nam" ): - refs[fkey["key"]] = fkey + refs[subpkg["key"]] = subpkg stmts.append( - f"self._{fkey['key']} = self.build_mfdata('{fkey['key']}', None)" + f"self._{subpkg['key']} = self.build_mfdata('{subpkg['key']}', None)" ) stmts.append( - f"self._{fkey['abbr']}_package = self.build_child_package('{fkey['abbr']}', {fkey['val']}, '{fkey['param']}', self._{fkey['key']})" + f"self._{subpkg['abbr']}_package = self.build_child_package('{subpkg['abbr']}', {subpkg['val']}, '{subpkg['param']}', self._{subpkg['key']})" ) return stmts diff --git a/flopy/mf6/utils/codegen/templates/package.py.jinja b/flopy/mf6/utils/codegen/templates/package.py.jinja index f3776ad2cd..5b4265b569 100644 --- a/flopy/mf6/utils/codegen/templates/package.py.jinja +++ b/flopy/mf6/utils/codegen/templates/package.py.jinja @@ -72,7 +72,7 @@ class Modflow{{ title }}(MFPackage): self._init_complete = True -{% if "subpkg" in meta and name.r != "hpc" %} +{% if subpackage is defined and name.r != "hpc" %} class {{ title }}Packages(MFChildPackages): """ {{ title }}Packages is a container class for the Modflow{{ title }} class. From 62639e397bef30209c78626253a7877a4bb5ede6 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 7 Nov 2024 15:39:21 -0500 Subject: [PATCH 66/75] order --- flopy/mf6/utils/codegen/dfn.py | 4 ++-- flopy/mf6/utils/codegen/dfn2toml.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index 42c6af00c0..d737e8fb1d 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -510,13 +510,13 @@ def _multi() -> bool: return cls( name=name, - vars=vars_, - dfn=dfn, foreign_keys=fkeys, package_type=package_type, subpackage=subpackage, solution=solution, multi=multi, + vars=vars_, + dfn=dfn, ) @classmethod diff --git a/flopy/mf6/utils/codegen/dfn2toml.py b/flopy/mf6/utils/codegen/dfn2toml.py index 9f0caaebca..8a18021a32 100644 --- a/flopy/mf6/utils/codegen/dfn2toml.py +++ b/flopy/mf6/utils/codegen/dfn2toml.py @@ -22,6 +22,7 @@ def _drop_none(d): def _shim(d): del d["dfn"] + del d["foreign_keys"] d["name"] = str(d["name"]) return d From 3ab0b2b2d95872e96fa9098a58012bb2d7d1a307 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Sun, 10 Nov 2024 11:17:38 -0500 Subject: [PATCH 67/75] improve docstrings, misc cleanup --- flopy/mf6/utils/codegen/__init__.py | 6 +- flopy/mf6/utils/codegen/context.py | 28 +++---- flopy/mf6/utils/codegen/filters.py | 79 ++++++++++++++++--- .../utils/codegen/templates/exchange.py.jinja | 2 +- .../mf6/utils/codegen/templates/macros.jinja | 8 +- .../utils/codegen/templates/model.py.jinja | 2 +- .../utils/codegen/templates/package.py.jinja | 14 ++-- .../codegen/templates/simulation.py.jinja | 2 +- 8 files changed, 95 insertions(+), 46 deletions(-) diff --git a/flopy/mf6/utils/codegen/__init__.py b/flopy/mf6/utils/codegen/__init__.py index e586a652d6..36884112bc 100644 --- a/flopy/mf6/utils/codegen/__init__.py +++ b/flopy/mf6/utils/codegen/__init__.py @@ -21,15 +21,15 @@ def _get_template_env(): env.filters["description"] = Filters.Cls.description env.filters["prefix"] = Filters.Cls.prefix env.filters["parent"] = Filters.Cls.parent - env.filters["skip"] = Filters.Cls.skip + env.filters["skip_init"] = Filters.Cls.skip_init env.filters["attrs"] = Filters.Vars.attrs env.filters["init"] = Filters.Vars.init - env.filters["maybe_file"] = Filters.Var.maybe_file + env.filters["untag"] = Filters.Var.untag env.filters["type"] = Filters.Var.type - env.filters["safe_str"] = Filters.safe_str + env.filters["safe_name"] = Filters.safe_name env.filters["escape_trailing_underscore"] = ( Filters.escape_trailing_underscore ) diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py index b597b94989..69603d34b8 100644 --- a/flopy/mf6/utils/codegen/context.py +++ b/flopy/mf6/utils/codegen/context.py @@ -16,18 +16,7 @@ class Context(TypedDict): than one input context (e.g. model DFNs yield a model class and a package class). - Notes - ----- - A context minimally consists of a name and a map of variables. - - The context class may inherit from a base class, and may specify - a parent context within which it can be created (the parent then - becomes the first `__init__` method parameter). - - The context class may reference other contexts via foreign key - relations held by its variables, and may itself be referenced - by other contexts if desired. - + A context consists minimally of a name and a map of variables. """ class Name(NamedTuple): @@ -70,7 +59,7 @@ def from_dfn(dfn: Dfn) -> List["Context.Name"]: ("gwf", "gnc"), ("gwt", "mvt"), ]: - # TODO: remove special cases, deduplicate mfmvr.py/mfgwfmvr.py etc + # TODO: deduplicate mfmvr.py/mfgwfmvr.py etc and remove special cases return [ Context.Name(*name), Context.Name(None, name.r), @@ -85,14 +74,15 @@ def from_dfn(cls, dfn: Dfn) -> Iterator["Context"]: """ Extract context class descriptor(s) from an input definition. These are structured representations of input context classes. + Each input definition yields one or more input contexts. """ - def _to_context(n, d): - d = d.copy() - d.pop("name", None) - vars_ = d.pop("vars", dict()) - return Context(name=n, vars=vars_, **d) + def _ctx(name, _dfn): + _dfn = _dfn.copy() + _dfn.pop("name", None) + _vars = _dfn.pop("vars", dict()) + return Context(name=name, vars=_vars, **_dfn) for name in Context.Name.from_dfn(dfn): - yield _to_context(name, dfn) + yield _ctx(name, dfn) diff --git a/flopy/mf6/utils/codegen/filters.py b/flopy/mf6/utils/codegen/filters.py index d1a4b1f1e8..c0aefde819 100644 --- a/flopy/mf6/utils/codegen/filters.py +++ b/flopy/mf6/utils/codegen/filters.py @@ -64,11 +64,13 @@ def description(ctx_name) -> str: ) def prefix(ctx_name) -> str: + """The input context class name prefix, e.g. 'MF' or 'Modflow'.""" base = Filters.Cls.base(ctx_name) return "MF" if base == "MFSimulationBase" else "Modflow" @pass_context def parent(ctx, ctx_name) -> str: + """The input context's parent context type, if it can have a parent.""" subpkg = ctx.get("subpackage", None) if subpkg: return subpkg["parent"] @@ -87,7 +89,8 @@ def parent(ctx, ctx_name) -> str: return "model" @pass_context - def skip(ctx, ctx_name) -> List[str]: + def skip_init(ctx, ctx_name) -> List[str]: + """Variables to skip in input context's `__init__` method.""" base = Filters.Cls.base(ctx_name) if base == "MFSimulationBase": return [ @@ -113,7 +116,13 @@ def skip(ctx, ctx_name) -> List[str]: return [] class Var: - def maybe_file(var: dict) -> dict: + def untag(var: dict) -> dict: + """ + If the variable is a tagged record, remove the leading + tag field. If the variable is a tagged file path input + record, remove both leading tag and 'filein'/'fileout' + keyword following it. + """ name = var["name"] tagged = var.get("tagged", False) fields = var.get("children", None) @@ -144,6 +153,15 @@ def maybe_file(var: dict) -> dict: return var def type(var: dict) -> str: + """ + Get a readable representation of the variable's type. + TODO: eventually replace this with a proper `type` in + the variable spec when we add type hints. For now try + to match the existing format, with a few tweaks; e.g. + distinguishing lists from records by square and round + brackets, respectively, and separating each choice in + a keystring by '|'. + """ _type = var["type"] shape = var.get("shape", None) children = var.get("children", None) @@ -167,6 +185,14 @@ def type(var: dict) -> str: class Vars: @pass_context def attrs(ctx, variables) -> List[str]: + """ + Map the context's input variables to corresponding class attributes, + where applicable. TODO: this should get much simpler if we can drop + all the `ListTemplateGenerator`/`ArrayTemplateGenerator` attributes. + Ultimately I (WPB) think we can aim for context classes consisting + of just a class attr for each variable, with anything complicated + happening in a decorator or base class. + """ name = ctx["name"] base = Filters.Cls.base(name) @@ -289,6 +315,18 @@ def _var(var: dict) -> List[str]: @pass_context def init(ctx, vars) -> List[str]: + """ + Map the context's input variables to statements in the class' + `__init__` method body, if applicable. TODO: consider how we + can dispatch as necessary based on a variable's type instead + of explicitly choosing among: + + - self.var = var + - self.var = self.build_mfdata(...) + - self.subppkg_var = self._create_package(...) + - ... + + """ ctx_name = ctx["name"] base = Filters.Cls.base(ctx_name) @@ -323,8 +361,9 @@ def _should_set(var: dict) -> bool: subpkg = var.get("subpackage", None) if subpkg and subpkg["key"] not in refs: refs[subpkg["key"]] = subpkg + args = f"'{subpkg['abbr']}', {subpkg['param']}" stmts.append( - f"self.{subpkg['param']} = self._create_package('{subpkg['abbr']}', {subpkg['param']})" + f"self.{subpkg['param']} = self._create_package({args})" ) elif base == "MFModel": @@ -353,8 +392,9 @@ def _should_set(var: dict) -> bool: subpkg = var.get("subpackage", None) if subpkg and subpkg["key"] not in refs: refs[subpkg["key"]] = subpkg + args = f"'{subpkg['abbr']}', {subpkg['param']}" stmts.append( - f"self.{subpkg['param']} = self._create_package('{subpkg['abbr']}', {subpkg['param']})" + f"self.{subpkg['param']} = self._create_package({args})" ) elif base == "MFPackage": @@ -391,7 +431,8 @@ def _should_build(var: dict) -> bool: if _should_build(var): if subpkg and ctx["name"] == (None, "nam"): stmts.append( - f"self.{'_' if subpkg else ''}{subpkg['key']} = self.build_mfdata('{subpkg['key']}', None)" + f"self.{'_' if subpkg else ''}{subpkg['key']} " + f"= self.build_mfdata('{subpkg['key']}', None)" ) else: _name = ( @@ -399,7 +440,8 @@ def _should_build(var: dict) -> bool: ) name = name.replace("-", "_") stmts.append( - f"self.{'_' if subpkg else ''}{name} = self.build_mfdata('{_name}', {name})" + f"self.{'_' if subpkg else ''}{name} " + f"= self.build_mfdata('{_name}', {name})" ) if ( @@ -409,23 +451,40 @@ def _should_build(var: dict) -> bool: ): refs[subpkg["key"]] = subpkg stmts.append( - f"self._{subpkg['key']} = self.build_mfdata('{subpkg['key']}', None)" + f"self._{subpkg['key']} " + f"= self.build_mfdata('{subpkg['key']}', None)" + ) + args = ( + f"'{subpkg['abbr']}', {subpkg['val']}, " + f"'{subpkg['param']}', self._{subpkg['key']}" ) stmts.append( - f"self._{subpkg['abbr']}_package = self.build_child_package('{subpkg['abbr']}', {subpkg['val']}, '{subpkg['param']}', self._{subpkg['key']})" + f"self._{subpkg['abbr']}_package " + f"= self.build_child_package({args})" ) return stmts return list(filter(None, _statements())) - def safe_str(v: str) -> str: - return (f"{v}_" if v in kwlist else v).replace("-", "_") + def safe_name(name: str) -> str: + """ + Make sure a string is safe to use as a variable name in Python code. + If the string is a reserved keyword, add a trailing underscore to it. + Also replace any hyphens with underscores. + """ + return (f"{name}_" if name in kwlist else name).replace("-", "_") def escape_trailing_underscore(v: str) -> str: + """If the string has a trailing underscore, escape it.""" return f"{v[:-1]}\\\\_" if v.endswith("_") else v def value(v: Any) -> str: + """ + Format a value to appear in the RHS of an assignment or argument- + passing expression: if it's an enum, get its value; if it's `str`, + quote it. + """ v = try_get_enum_value(v) if isinstance(v, str) and v[0] not in ["'", '"']: v = f"'{v}'" diff --git a/flopy/mf6/utils/codegen/templates/exchange.py.jinja b/flopy/mf6/utils/codegen/templates/exchange.py.jinja index bf052f1a12..e6bd047253 100644 --- a/flopy/mf6/utils/codegen/templates/exchange.py.jinja +++ b/flopy/mf6/utils/codegen/templates/exchange.py.jinja @@ -27,7 +27,7 @@ class Modflow{{ title }}(MFPackage): exgtype="{{ name.r[:3].upper() }}6-{{ name.r[3:].upper() }}6", exgmnamea=None, exgmnameb=None, - {{ macros.init_vars(vars, indent=8, skip=name|skip) }} + {{ macros.init_vars(vars, indent=8, skip=name|skip_init) }} filename=None, pname=None, **kwargs, diff --git a/flopy/mf6/utils/codegen/templates/macros.jinja b/flopy/mf6/utils/codegen/templates/macros.jinja index 990ee8c84e..0188891ebf 100644 --- a/flopy/mf6/utils/codegen/templates/macros.jinja +++ b/flopy/mf6/utils/codegen/templates/macros.jinja @@ -1,15 +1,15 @@ {% macro init_vars(vars, alias=false, indent=0, skip=none) %} {%- for name, var in vars.items() if name not in skip %} -{% set v = var|maybe_file -%} -{% set n = (name if alias else v.name)|safe_str -%} +{% set v = var|untag -%} +{% set n = (name if alias else v.name)|safe_name -%} {{ ""|indent(indent, first=true) }}{{ n }}{%- if v.default is defined %}={{ v.default|value }}{%- endif -%}, {%- endfor %} {% endmacro %} {% macro vars_docs(vars, indent=0) %} {%- for var in vars.values() recursive %} -{% set v = var|maybe_file -%} -{% set n = v.name|safe_str|escape_trailing_underscore -%} +{% set v = var|untag -%} +{% set n = v.name|safe_name|escape_trailing_underscore -%} {{ ""|indent(indent, first=true) }}{% if loop.depth > 1 %}* {% endif %}{{ n }} : {{ v|type }} {%- if v.description is defined and v.description is not none %} {{ v.description|wordwrap|indent(indent + (loop.depth * 4), first=true) }} diff --git a/flopy/mf6/utils/codegen/templates/model.py.jinja b/flopy/mf6/utils/codegen/templates/model.py.jinja index 651dcf6a61..6fc07aaaf7 100644 --- a/flopy/mf6/utils/codegen/templates/model.py.jinja +++ b/flopy/mf6/utils/codegen/templates/model.py.jinja @@ -34,7 +34,7 @@ class Modflow{{ title }}(MFModel): version="mf6", exe_name="mf6", model_rel_path=".", - {{ macros.init_vars(vars, indent=8, skip=name|skip) }} + {{ macros.init_vars(vars, indent=8, skip=name|skip_init) }} **kwargs, ): """ diff --git a/flopy/mf6/utils/codegen/templates/package.py.jinja b/flopy/mf6/utils/codegen/templates/package.py.jinja index 5b4265b569..4eb0f450a3 100644 --- a/flopy/mf6/utils/codegen/templates/package.py.jinja +++ b/flopy/mf6/utils/codegen/templates/package.py.jinja @@ -25,7 +25,7 @@ class Modflow{{ title }}(MFPackage): self, {{ name|parent }}, loading_package=False, - {{ macros.init_vars(vars, indent=8, skip=name|skip) }} + {{ macros.init_vars(vars, indent=8, skip=name|skip_init) }} filename=None, pname=None, **kwargs, @@ -94,14 +94,14 @@ class {{ title }}Packages(MFChildPackages): def initialize( self, - {{ macros.init_vars(vars, alias=true, indent=8, skip=name|skip) }} + {{ macros.init_vars(vars, alias=true, indent=8, skip=name|skip_init) }} filename=None, pname=None, ): new_package = Modflow{{ title }}( self._cpparent, - {%- for n, var in vars.items() if n not in name|skip %} - {{ n|safe_str }}={{ n|safe_str }}, + {%- for n, var in vars.items() if n not in name|skip_init %} + {{ n|safe_name }}={{ n|safe_name }}, {%- endfor %} filename=filename, pname=pname, @@ -112,14 +112,14 @@ class {{ title }}Packages(MFChildPackages): {% if name.r != "obs" %} def append_package( self, - {{ macros.init_vars(vars, alias=true, indent=8, skip=name|skip) }} + {{ macros.init_vars(vars, alias=true, indent=8, skip=name|skip_init) }} filename=None, pname=None, ): new_package = Modflow{{ title }}( self._cpparent, - {%- for n, var in vars.items() if n not in name|skip %} - {{ n|safe_str }}={{ n|safe_str }}, + {%- for n, var in vars.items() if n not in name|skip_init %} + {{ n|safe_name }}={{ n|safe_name }}, {%- endfor %} filename=filename, pname=pname, diff --git a/flopy/mf6/utils/codegen/templates/simulation.py.jinja b/flopy/mf6/utils/codegen/templates/simulation.py.jinja index bdfbf0547f..ca83fd656a 100644 --- a/flopy/mf6/utils/codegen/templates/simulation.py.jinja +++ b/flopy/mf6/utils/codegen/templates/simulation.py.jinja @@ -35,7 +35,7 @@ class MF{{ title }}(MFSimulationBase): write_headers: bool = True, use_pandas: bool = True, lazy_io: bool = False, - {{ macros.init_vars(vars, indent=8, skip=name|skip) }} + {{ macros.init_vars(vars, indent=8, skip=name|skip_init) }} ): """ {{ name|description }} From 0ef4820371ebaec7b117dc393de984642d413ba3 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Sun, 10 Nov 2024 11:40:42 -0500 Subject: [PATCH 68/75] improve dev guide --- docs/mf6_dev_guide.md | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/docs/mf6_dev_guide.md b/docs/mf6_dev_guide.md index fd1921d3dd..456266ac41 100644 --- a/docs/mf6_dev_guide.md +++ b/docs/mf6_dev_guide.md @@ -1,13 +1,24 @@ -Introduction ------------------------------------------------ +# Developing FloPy for MF6 -This file provides an overview of how FloPy's MODFLOW 6 module `flopy.mf6` works under the hood. It is intended for anyone who wants to add a new package, new model, or new features to this library. + + -Code generation ------------------------------------------------ +- [Introduction](#introduction) +- [Code generation](#code-generation) +- [Input specification](#input-specification) + + + +## Introduction + +This file provides an overview of how FloPy's MODFLOW 6 module `flopy.mf6` works under the hood. It is intended for FloPy maintainers, as well as anyone who wants to add a new package, new model, or new features to this library. + +## Code generation MODFLOW 6 describes its input specification with definition files. These are currently a custom text-based format. Definition files have suffix `.dfn` by convention. +We plan to move soon to TOML definition files. More on this below. + Definition files describe components (e.g. simulations, models, packages) supported by MODFLOW 6, and are used to generate both source code and documentation. FloPy has two scripts that can be used to generate a MODFLOW 6 compatibility layer: @@ -19,12 +30,25 @@ The latter is typically used with e.g. `python -m flopy.mf6.utils.generate_class Generated files are created in `flopy/mf6/modflow/` and contain interface classes, one file/class per input component. These can be used to initialize and access model/package data as well as the input specification itself. +**Note**: Code generation requires a few extra dependencies, grouped in the `codegen` optional dependency group: `Jinja2`, `boltons`, `tomlkit` and `modflow-devtools`. + **Note**: Code generation scripts previously used `flopy/mf6/data/mfstructure.py` to read and represent definition files, and wrote Python by hand. They now use the `flopy.mf6.utils.codegen` module, which uses Jinja2. **Note**: Each component's input definition is currently reproduced almost verbatim in the `dfn` class attribute. Currently, `flopy/mf6/data/mfstructure.py` is used to introspect the input specification using the `dfn` attribute. This can eventually be removed in favor of direct introspection. -Input specification -------------------- +The `flopy.mf6.utils.codegen` module is small and meant to be easy to iterate on. Its purpose is to load and convert input definition files to Python source code by way of an intermediate representation. + +As such, there are 2 abstractions: `Dfn` and `Context`. A `Dfn` corresponds to a definition file, and knows how to load itself from one. A `Context` corresponds to a Python source file, which it is fed to a Jinja template to create. ('Context' is a term borrowed from Jinja.) `Dfn` and `Context` typically map 1-1, but can be 1-many (e.g. a model definition file yields a model class and namefile package class). + +Both `Dfn` and `Context` are structured representations of an input component/block/variable hierarchy. For now, we have to infer this structure from a flat representation in the definition file. This is a bit like [object-relational impedance mismatch](https://en.wikipedia.org/wiki/Object%E2%80%93relational_impedance_mismatch) and seriously complicates the `Dfn` load routines, but happily this is temporary — once we move to TOML and define input components in structured form, we can simply load the nested variable hierarchy directly from the definition file. + +For now we use a data structure from the `boltons` library to maintain an unstructured (flat) map of variables before structural parsing, where variables can have duplicate names. + +Some quirks of the legacy framework are handled in a "shim" of Jinja filters that transform the template context, as well as some macros. These can ideally be removed as refactoring goes on. The templates should also get simpler over time. + +## Input specification + +**Note**: the following describes the legacy input specification mechanism. The `flopy.mf6.data.mfstructure.MFStructure` class represents an input specification. The class is a singleton, meaning only one instance of this class can be created. The class contains a sim_struct attribute (which is a flopy.mf6.data.mfstructure.MFSimulationStructure object) which contains all of the meta-data for all package files. Meta-data is stored in a structured format. MFSimulationStructure contains MFModelStructure and MFInputFileStructure objects, which contain the meta-data for each model type and each "simulation-level" package (tdis, ims, ...). MFModelStructure contains model specific meta-data and a MFInputFileStructure object for each package in that model. MFInputFileStructure contains package specific meta-data and a MFBlockStructure object for each block contained in the package file. MFBlockStructure contains block specific meta-data and a MFDataStructure object for each data structure defined in the block, and MFDataStructure contains data structure specific meta-data and a MFDataItemStructure object for each data item contained in the data structure. Data structures define the structure of data that is naturally grouped together, for example, the data in a numpy recarray. Data item structures define the structure of specific pieces of data, for example, a single column of a numpy recarray. The meta-data defined in these classes provides all the information FloPy needs to read and write MODFLOW 6 package and name files, create the Flopy interface, and check the data for various constraints. From 907d129ba4eeeeeeeb07d7c0dfaaf073b0cd0de3 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 13 Nov 2024 07:59:19 -0500 Subject: [PATCH 69/75] revisions --- flopy/mf6/utils/codegen/__init__.py | 4 +--- flopy/mf6/utils/codegen/context.py | 4 ++-- flopy/mf6/utils/codegen/filters.py | 16 ++-------------- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/flopy/mf6/utils/codegen/__init__.py b/flopy/mf6/utils/codegen/__init__.py index 36884112bc..8e1418dc42 100644 --- a/flopy/mf6/utils/codegen/__init__.py +++ b/flopy/mf6/utils/codegen/__init__.py @@ -47,9 +47,7 @@ def make_init(dfns: dict, outdir: PathLike, verbose: bool = False): from flopy.mf6.utils.codegen.context import Context contexts = list( - chain( - *[[ctx for ctx in Context.from_dfn(dfn)] for dfn in dfns.values()] - ) + chain.from_iterable(Context.from_dfn(dfn) for dfn in dfns.values()) ) target_name = "__init__.py" target_path = outdir / target_name diff --git a/flopy/mf6/utils/codegen/context.py b/flopy/mf6/utils/codegen/context.py index 69603d34b8..61c334b830 100644 --- a/flopy/mf6/utils/codegen/context.py +++ b/flopy/mf6/utils/codegen/context.py @@ -69,8 +69,8 @@ def from_dfn(dfn: Dfn) -> List["Context.Name"]: name: Name vars: Vars - @classmethod - def from_dfn(cls, dfn: Dfn) -> Iterator["Context"]: + @staticmethod + def from_dfn(dfn: Dfn) -> Iterator["Context"]: """ Extract context class descriptor(s) from an input definition. These are structured representations of input context classes. diff --git a/flopy/mf6/utils/codegen/filters.py b/flopy/mf6/utils/codegen/filters.py index c0aefde819..8b58eb01e6 100644 --- a/flopy/mf6/utils/codegen/filters.py +++ b/flopy/mf6/utils/codegen/filters.py @@ -82,10 +82,6 @@ def parent(ctx, ctx_name) -> str: or ctx_name.l in ["sim", "exg", "sln"] ): return "simulation" - elif subpkg: - if ctx_name.l == "utl" and ctx_name.r == "hpc": - return "simulation" - return "package" return "model" @pass_context @@ -101,13 +97,7 @@ def skip_init(ctx, ctx_name) -> List[str]: "solutiongroup", ] elif base == "MFModel": - skip = ["packages", "export_netcdf", "nc_filerecord"] - refs = ctx.get("foreign_keys", dict()) - if any(refs) and ctx["name"] != (None, "nam"): - for k in refs.keys(): - if ctx["vars"].get(k, None): - skip.append(k) - return skip + return ["packages", "export_netcdf", "nc_filerecord"] else: if ctx_name.r == "nam": return ["export_netcdf", "nc_filerecord"] @@ -295,9 +285,7 @@ def _var(var: dict) -> List[str]: attrs = list(filter(None, [_attr(v) for v in variables.values()])) - if base == "MFModel": - attrs.append(f"model_type = {name.l}") - elif base == "MFPackage": + if base == "MFPackage": attrs.extend( [ f"package_abbr = '{name.r}'" From 09fbc383ec65b9767a35776cbc8111754fbf92d0 Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 13 Nov 2024 20:17:22 -0500 Subject: [PATCH 70/75] whitespace mgmt --- flopy/mf6/utils/codegen/__init__.py | 8 ++++- flopy/mf6/utils/codegen/dfn.py | 2 +- flopy/mf6/utils/codegen/filters.py | 9 +++--- .../utils/codegen/templates/exchange.py.jinja | 4 +-- .../mf6/utils/codegen/templates/macros.jinja | 30 +++++++++++-------- .../utils/codegen/templates/model.py.jinja | 2 +- .../utils/codegen/templates/package.py.jinja | 14 ++++----- .../codegen/templates/simulation.py.jinja | 2 +- 8 files changed, 40 insertions(+), 31 deletions(-) diff --git a/flopy/mf6/utils/codegen/__init__.py b/flopy/mf6/utils/codegen/__init__.py index 8e1418dc42..2814430713 100644 --- a/flopy/mf6/utils/codegen/__init__.py +++ b/flopy/mf6/utils/codegen/__init__.py @@ -12,7 +12,13 @@ def _get_template_env(): # expect optional deps at module init time jinja = import_optional_dependency("jinja2") loader = jinja.PackageLoader("flopy", "mf6/utils/codegen/templates/") - env = jinja.Environment(loader=loader) + env = jinja.Environment( + loader=loader, + trim_blocks=True, + lstrip_blocks=True, + line_statement_prefix="_", + keep_trailing_newline=True, + ) from flopy.mf6.utils.codegen.filters import Filters diff --git a/flopy/mf6/utils/codegen/dfn.py b/flopy/mf6/utils/codegen/dfn.py index d737e8fb1d..7717c41bee 100644 --- a/flopy/mf6/utils/codegen/dfn.py +++ b/flopy/mf6/utils/codegen/dfn.py @@ -396,7 +396,7 @@ def _fields() -> Vars: block=block, children=None, description=( - f"* Contains data for the {ref['abbr']} package. Data can be " + f"Contains data for the {ref['abbr']} package. Data can be " f"stored in a dictionary containing data for the {ref['abbr']} " "package with variable names as keys and package data as " f"values. Data just for the {ref['val']} variable is also " diff --git a/flopy/mf6/utils/codegen/filters.py b/flopy/mf6/utils/codegen/filters.py index 8b58eb01e6..201db7395d 100644 --- a/flopy/mf6/utils/codegen/filters.py +++ b/flopy/mf6/utils/codegen/filters.py @@ -146,17 +146,16 @@ def type(var: dict) -> str: """ Get a readable representation of the variable's type. TODO: eventually replace this with a proper `type` in - the variable spec when we add type hints. For now try - to match the existing format, with a few tweaks; e.g. - distinguishing lists from records by square and round - brackets, respectively, and separating each choice in - a keystring by '|'. + the variable spec when we add type hints """ _type = var["type"] shape = var.get("shape", None) children = var.get("children", None) if children: if _type == "list": + import pdb; pdb.set_trace() + if len(children) == 1 and (first := list(children.values())[0])["type"] == "record": + return f"{Filters.Var.type(first)}" children = ", ".join( [v["name"] for v in children.values()] ) diff --git a/flopy/mf6/utils/codegen/templates/exchange.py.jinja b/flopy/mf6/utils/codegen/templates/exchange.py.jinja index e6bd047253..d95a831c50 100644 --- a/flopy/mf6/utils/codegen/templates/exchange.py.jinja +++ b/flopy/mf6/utils/codegen/templates/exchange.py.jinja @@ -18,7 +18,7 @@ class Modflow{{ title }}(MFPackage): {% for attr in vars|attrs %} {{ attr }} - {%- endfor %} + {% endfor %} def __init__( self, @@ -83,6 +83,6 @@ class Modflow{{ title }}(MFPackage): {% for statement in vars|init %} {{ statement }} - {%- endfor %} + {% endfor %} self._init_complete = True \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/macros.jinja b/flopy/mf6/utils/codegen/templates/macros.jinja index 0188891ebf..5dd6ea7a20 100644 --- a/flopy/mf6/utils/codegen/templates/macros.jinja +++ b/flopy/mf6/utils/codegen/templates/macros.jinja @@ -1,21 +1,25 @@ {% macro init_vars(vars, alias=false, indent=0, skip=none) %} -{%- for name, var in vars.items() if name not in skip %} -{% set v = var|untag -%} -{% set n = (name if alias else v.name)|safe_name -%} -{{ ""|indent(indent, first=true) }}{{ n }}{%- if v.default is defined %}={{ v.default|value }}{%- endif -%}, -{%- endfor %} +{% for name, var in vars.items() if name not in skip %} +{% set v = var|untag %} +{% set n = (name if alias else v.name)|safe_name %} +{{ ""|indent(indent, first=false) }}{{ n }}{% if v.default is defined %}={{ v.default|value }}{% endif %}, +{% endfor %} {% endmacro %} {% macro vars_docs(vars, indent=0) %} -{%- for var in vars.values() recursive %} -{% set v = var|untag -%} -{% set n = v.name|safe_name|escape_trailing_underscore -%} +{% for var in vars.values() recursive %} +{% set v = var|untag %} +{% set n = v.name|safe_name|escape_trailing_underscore %} {{ ""|indent(indent, first=true) }}{% if loop.depth > 1 %}* {% endif %}{{ n }} : {{ v|type }} -{%- if v.description is defined and v.description is not none %} +{% if v.description is defined and v.description is not none %} {{ v.description|wordwrap|indent(indent + (loop.depth * 4), first=true) }} -{%- endif %} -{%- if v.children is defined and v.children is not none -%} +{% endif %} +{% if v.children is defined and v.children is not none %} +{% if v.type == "list" and v.childen|length == 1 and (v.children.values()|first).type == "record" %} +{{ loop((v.children.values()|first).children.values())|indent(indent, first=true) }} +{% else %} {{ loop(v.children.values())|indent(indent, first=true) }} -{%- endif %} -{% endfor -%} +{% endif %} +{% endif %} +{% endfor %} {% endmacro %} \ No newline at end of file diff --git a/flopy/mf6/utils/codegen/templates/model.py.jinja b/flopy/mf6/utils/codegen/templates/model.py.jinja index 6fc07aaaf7..b9d1bfa483 100644 --- a/flopy/mf6/utils/codegen/templates/model.py.jinja +++ b/flopy/mf6/utils/codegen/templates/model.py.jinja @@ -77,7 +77,7 @@ class Modflow{{ title }}(MFModel): {% for statement in vars|init %} {{ statement }} - {%- endfor %} + {% endfor %} @classmethod def load( diff --git a/flopy/mf6/utils/codegen/templates/package.py.jinja b/flopy/mf6/utils/codegen/templates/package.py.jinja index 4eb0f450a3..b0ce7c6033 100644 --- a/flopy/mf6/utils/codegen/templates/package.py.jinja +++ b/flopy/mf6/utils/codegen/templates/package.py.jinja @@ -14,12 +14,12 @@ class Modflow{{ title }}(MFPackage): Parameters ---------- - {{ macros.vars_docs(vars, indent=4) }} +{{ macros.vars_docs(vars, indent=4) }} """ {% for attr in vars|attrs %} {{ attr }} - {%- endfor %} + {% endfor %} def __init__( self, @@ -68,7 +68,7 @@ class Modflow{{ title }}(MFPackage): {% for statement in vars|init %} {{ statement }} - {%- endfor %} + {% endfor %} self._init_complete = True @@ -100,9 +100,9 @@ class {{ title }}Packages(MFChildPackages): ): new_package = Modflow{{ title }}( self._cpparent, - {%- for n, var in vars.items() if n not in name|skip_init %} + {% for n, var in vars.items() if n not in name|skip_init %} {{ n|safe_name }}={{ n|safe_name }}, - {%- endfor %} + {% endfor %} filename=filename, pname=pname, child_builder_call=True, @@ -118,9 +118,9 @@ class {{ title }}Packages(MFChildPackages): ): new_package = Modflow{{ title }}( self._cpparent, - {%- for n, var in vars.items() if n not in name|skip_init %} + {% for n, var in vars.items() if n not in name|skip_init %} {{ n|safe_name }}={{ n|safe_name }}, - {%- endfor %} + {% endfor %} filename=filename, pname=pname, child_builder_call=True, diff --git a/flopy/mf6/utils/codegen/templates/simulation.py.jinja b/flopy/mf6/utils/codegen/templates/simulation.py.jinja index ca83fd656a..446c1bc8bb 100644 --- a/flopy/mf6/utils/codegen/templates/simulation.py.jinja +++ b/flopy/mf6/utils/codegen/templates/simulation.py.jinja @@ -82,7 +82,7 @@ class MF{{ title }}(MFSimulationBase): {% for statement in vars|init %} {{ statement }} - {%- endfor %} + {% endfor %} @classmethod def load( From 97084ad9e9050e7de6fac7d31b2bbaabdc049eae Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 13 Nov 2024 22:34:32 -0500 Subject: [PATCH 71/75] better composite variable docstring format --- flopy/mf6/utils/codegen/filters.py | 7 ++++--- flopy/mf6/utils/codegen/templates/exchange.py.jinja | 6 +++--- flopy/mf6/utils/codegen/templates/macros.jinja | 10 +++++----- flopy/mf6/utils/codegen/templates/model.py.jinja | 6 +++--- flopy/mf6/utils/codegen/templates/package.py.jinja | 8 ++++---- flopy/mf6/utils/codegen/templates/simulation.py.jinja | 6 +++--- 6 files changed, 22 insertions(+), 21 deletions(-) diff --git a/flopy/mf6/utils/codegen/filters.py b/flopy/mf6/utils/codegen/filters.py index 201db7395d..9d19f18d6c 100644 --- a/flopy/mf6/utils/codegen/filters.py +++ b/flopy/mf6/utils/codegen/filters.py @@ -153,9 +153,10 @@ def type(var: dict) -> str: children = var.get("children", None) if children: if _type == "list": - import pdb; pdb.set_trace() - if len(children) == 1 and (first := list(children.values())[0])["type"] == "record": - return f"{Filters.Var.type(first)}" + if len(children) == 1: + first = list(children.values())[0] + if first["type"] in ["record", "union"]: + return f"[{Filters.Var.type(first)}]" children = ", ".join( [v["name"] for v in children.values()] ) diff --git a/flopy/mf6/utils/codegen/templates/exchange.py.jinja b/flopy/mf6/utils/codegen/templates/exchange.py.jinja index d95a831c50..9987286803 100644 --- a/flopy/mf6/utils/codegen/templates/exchange.py.jinja +++ b/flopy/mf6/utils/codegen/templates/exchange.py.jinja @@ -13,7 +13,7 @@ class Modflow{{ title }}(MFPackage): Parameters ---------- - {{ macros.vars_docs(vars, indent=4) }} +{{ macros.vars_docs(vars, indent=4) }} """ {% for attr in vars|attrs %} @@ -27,7 +27,7 @@ class Modflow{{ title }}(MFPackage): exgtype="{{ name.r[:3].upper() }}6-{{ name.r[3:].upper() }}6", exgmnamea=None, exgmnameb=None, - {{ macros.init_vars(vars, indent=8, skip=name|skip_init) }} +{{ macros.init_vars(vars, indent=8, skip=name|skip_init) }} filename=None, pname=None, **kwargs, @@ -64,7 +64,7 @@ class Modflow{{ title }}(MFPackage): GWE Model with the name exgmnameb must correspond to the GWF Model with the name gwfmodelname2. - {{ macros.vars_docs(vars, indent=8) }} +{{ macros.vars_docs(vars, indent=8) }} """ super().__init__( diff --git a/flopy/mf6/utils/codegen/templates/macros.jinja b/flopy/mf6/utils/codegen/templates/macros.jinja index 5dd6ea7a20..6e567fc39c 100644 --- a/flopy/mf6/utils/codegen/templates/macros.jinja +++ b/flopy/mf6/utils/codegen/templates/macros.jinja @@ -2,11 +2,11 @@ {% for name, var in vars.items() if name not in skip %} {% set v = var|untag %} {% set n = (name if alias else v.name)|safe_name %} -{{ ""|indent(indent, first=false) }}{{ n }}{% if v.default is defined %}={{ v.default|value }}{% endif %}, +{{ ""|indent(indent, first=true) }}{{ n }}{% if v.default is defined %}={{ v.default|value }}{% endif %}, {% endfor %} {% endmacro %} -{% macro vars_docs(vars, indent=0) %} +{% macro vars_docs(vars, indent=0, recurse=true) %} {% for var in vars.values() recursive %} {% set v = var|untag %} {% set n = v.name|safe_name|escape_trailing_underscore %} @@ -14,12 +14,12 @@ {% if v.description is defined and v.description is not none %} {{ v.description|wordwrap|indent(indent + (loop.depth * 4), first=true) }} {% endif %} -{% if v.children is defined and v.children is not none %} -{% if v.type == "list" and v.childen|length == 1 and (v.children.values()|first).type == "record" %} +{% if recurse and v.children is defined and v.children is not none %} +{% if v.type == "list" and v.children|length == 1 and (v.children.values()|first).type == "record" %} {{ loop((v.children.values()|first).children.values())|indent(indent, first=true) }} {% else %} {{ loop(v.children.values())|indent(indent, first=true) }} {% endif %} {% endif %} {% endfor %} -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/flopy/mf6/utils/codegen/templates/model.py.jinja b/flopy/mf6/utils/codegen/templates/model.py.jinja index b9d1bfa483..89e072ce5a 100644 --- a/flopy/mf6/utils/codegen/templates/model.py.jinja +++ b/flopy/mf6/utils/codegen/templates/model.py.jinja @@ -14,7 +14,7 @@ class Modflow{{ title }}(MFModel): Parameters ---------- - {{ macros.vars_docs(vars, indent=4) }} +{{ macros.vars_docs(vars, indent=4) }} Methods ------- @@ -34,7 +34,7 @@ class Modflow{{ title }}(MFModel): version="mf6", exe_name="mf6", model_rel_path=".", - {{ macros.init_vars(vars, indent=8, skip=name|skip_init) }} +{{ macros.init_vars(vars, indent=8, skip=name|skip_init) }} **kwargs, ): """ @@ -61,7 +61,7 @@ class Modflow{{ title }}(MFModel): Simulation that this model is a part of. Model is automatically added to simulation when it is initialized. - {{ macros.vars_docs(vars, indent=8) }} +{{ macros.vars_docs(vars, indent=8) }} """ super().__init__( diff --git a/flopy/mf6/utils/codegen/templates/package.py.jinja b/flopy/mf6/utils/codegen/templates/package.py.jinja index b0ce7c6033..fc2bf0ebee 100644 --- a/flopy/mf6/utils/codegen/templates/package.py.jinja +++ b/flopy/mf6/utils/codegen/templates/package.py.jinja @@ -25,7 +25,7 @@ class Modflow{{ title }}(MFPackage): self, {{ name|parent }}, loading_package=False, - {{ macros.init_vars(vars, indent=8, skip=name|skip_init) }} +{{ macros.init_vars(vars, indent=8, skip=name|skip_init) }} filename=None, pname=None, **kwargs, @@ -43,7 +43,7 @@ class Modflow{{ title }}(MFPackage): Do not set this parameter. It is intended for debugging and internal processing purposes only. - {{ macros.vars_docs(vars, indent=8) }} +{{ macros.vars_docs(vars, indent=8) }} filename : str File name for this package. @@ -94,7 +94,7 @@ class {{ title }}Packages(MFChildPackages): def initialize( self, - {{ macros.init_vars(vars, alias=true, indent=8, skip=name|skip_init) }} +{{ macros.init_vars(vars, alias=true, indent=8, skip=name|skip_init) }} filename=None, pname=None, ): @@ -112,7 +112,7 @@ class {{ title }}Packages(MFChildPackages): {% if name.r != "obs" %} def append_package( self, - {{ macros.init_vars(vars, alias=true, indent=8, skip=name|skip_init) }} +{{ macros.init_vars(vars, alias=true, indent=8, skip=name|skip_init) }} filename=None, pname=None, ): diff --git a/flopy/mf6/utils/codegen/templates/simulation.py.jinja b/flopy/mf6/utils/codegen/templates/simulation.py.jinja index 446c1bc8bb..d50df39aa6 100644 --- a/flopy/mf6/utils/codegen/templates/simulation.py.jinja +++ b/flopy/mf6/utils/codegen/templates/simulation.py.jinja @@ -13,7 +13,7 @@ class MF{{ title }}(MFSimulationBase): Parameters ---------- - {{ macros.vars_docs(vars, indent=4) }} +{{ macros.vars_docs(vars, indent=4) }} Methods ------- @@ -35,7 +35,7 @@ class MF{{ title }}(MFSimulationBase): write_headers: bool = True, use_pandas: bool = True, lazy_io: bool = False, - {{ macros.init_vars(vars, indent=8, skip=name|skip_init) }} +{{ macros.init_vars(vars, indent=8, skip=name|skip_init) }} ): """ {{ name|description }} @@ -66,7 +66,7 @@ class MF{{ title }}(MFSimulationBase): lazy_io Whether to use lazy IO - {{ macros.vars_docs(vars, indent=8) }} +{{ macros.vars_docs(vars, indent=8) }} """ super().__init__( From 589da742d2a21c4742a594d4682597c730de6c6e Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 13 Nov 2024 22:41:41 -0500 Subject: [PATCH 72/75] rebase and ruff --- autotest/test_codegen.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/autotest/test_codegen.py b/autotest/test_codegen.py index 453d78ba0f..5248a90457 100644 --- a/autotest/test_codegen.py +++ b/autotest/test_codegen.py @@ -8,9 +8,7 @@ MF6_PATH = PROJ_ROOT / "flopy" / "mf6" DFN_PATH = MF6_PATH / "data" / "dfn" DFN_NAMES = [ - dfn.stem - for dfn in DFN_PATH.glob("*.dfn") - if dfn.stem not in ["common", "flopy"] + dfn.stem for dfn in DFN_PATH.glob("*.dfn") if dfn.stem not in ["common", "flopy"] ] From a21f19a01d03e59d93e6c675692b9477011e996a Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Thu, 14 Nov 2024 06:19:47 -0500 Subject: [PATCH 73/75] indentation --- .../utils/codegen/templates/exchange.py.jinja | 6 +++--- flopy/mf6/utils/codegen/templates/macros.jinja | 16 ++++++++-------- flopy/mf6/utils/codegen/templates/model.py.jinja | 6 +++--- .../mf6/utils/codegen/templates/package.py.jinja | 10 +++++----- .../utils/codegen/templates/simulation.py.jinja | 6 +++--- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/flopy/mf6/utils/codegen/templates/exchange.py.jinja b/flopy/mf6/utils/codegen/templates/exchange.py.jinja index 9987286803..7d6f6e16ef 100644 --- a/flopy/mf6/utils/codegen/templates/exchange.py.jinja +++ b/flopy/mf6/utils/codegen/templates/exchange.py.jinja @@ -13,7 +13,7 @@ class Modflow{{ title }}(MFPackage): Parameters ---------- -{{ macros.vars_docs(vars, indent=4) }} +{{ macros.vars_docs(vars)|indent(4, first=true) }} """ {% for attr in vars|attrs %} @@ -27,7 +27,7 @@ class Modflow{{ title }}(MFPackage): exgtype="{{ name.r[:3].upper() }}6-{{ name.r[3:].upper() }}6", exgmnamea=None, exgmnameb=None, -{{ macros.init_vars(vars, indent=8, skip=name|skip_init) }} +{{ macros.init_vars(vars, skip=name|skip_init)|indent(8, first=true) }} filename=None, pname=None, **kwargs, @@ -64,7 +64,7 @@ class Modflow{{ title }}(MFPackage): GWE Model with the name exgmnameb must correspond to the GWF Model with the name gwfmodelname2. -{{ macros.vars_docs(vars, indent=8) }} +{{ macros.vars_docs(vars)|indent(8, first=true) }} """ super().__init__( diff --git a/flopy/mf6/utils/codegen/templates/macros.jinja b/flopy/mf6/utils/codegen/templates/macros.jinja index 6e567fc39c..adbf9b646a 100644 --- a/flopy/mf6/utils/codegen/templates/macros.jinja +++ b/flopy/mf6/utils/codegen/templates/macros.jinja @@ -1,24 +1,24 @@ -{% macro init_vars(vars, alias=false, indent=0, skip=none) %} +{% macro init_vars(vars, alias=false, skip=none) %} {% for name, var in vars.items() if name not in skip %} {% set v = var|untag %} {% set n = (name if alias else v.name)|safe_name %} -{{ ""|indent(indent, first=true) }}{{ n }}{% if v.default is defined %}={{ v.default|value }}{% endif %}, +{{ n }}{% if v.default is defined %}={{ v.default|value }}{% endif %}, {% endfor %} {% endmacro %} -{% macro vars_docs(vars, indent=0, recurse=true) %} +{% macro vars_docs(vars, recurse=true) %} {% for var in vars.values() recursive %} {% set v = var|untag %} {% set n = v.name|safe_name|escape_trailing_underscore %} -{{ ""|indent(indent, first=true) }}{% if loop.depth > 1 %}* {% endif %}{{ n }} : {{ v|type }} +{% if loop.depth > 1 %}* {% endif %}{{ n }} : {{ v|type }} {% if v.description is defined and v.description is not none %} -{{ v.description|wordwrap|indent(indent + (loop.depth * 4), first=true) }} +{{ v.description|wordwrap|indent(loop.depth * 4, first=true) }} {% endif %} {% if recurse and v.children is defined and v.children is not none %} -{% if v.type == "list" and v.children|length == 1 and (v.children.values()|first).type == "record" %} -{{ loop((v.children.values()|first).children.values())|indent(indent, first=true) }} +{% if v.type == "list" and v.children|length == 1 and (v.children.values()|first).type in ["record", "union"] %} +{{ loop((v.children.values()|first).children.values())|indent(loop.depth * 4, first=true) }} {% else %} -{{ loop(v.children.values())|indent(indent, first=true) }} +{{ loop(v.children.values())|indent(loop.depth * 4, first=true) }} {% endif %} {% endif %} {% endfor %} diff --git a/flopy/mf6/utils/codegen/templates/model.py.jinja b/flopy/mf6/utils/codegen/templates/model.py.jinja index 89e072ce5a..dc5dccbf22 100644 --- a/flopy/mf6/utils/codegen/templates/model.py.jinja +++ b/flopy/mf6/utils/codegen/templates/model.py.jinja @@ -14,7 +14,7 @@ class Modflow{{ title }}(MFModel): Parameters ---------- -{{ macros.vars_docs(vars, indent=4) }} +{{ macros.vars_docs(vars)|indent(4, first=true) }} Methods ------- @@ -34,7 +34,7 @@ class Modflow{{ title }}(MFModel): version="mf6", exe_name="mf6", model_rel_path=".", -{{ macros.init_vars(vars, indent=8, skip=name|skip_init) }} +{{ macros.init_vars(vars, skip=name|skip_init)|indent(8, first=true) }} **kwargs, ): """ @@ -61,7 +61,7 @@ class Modflow{{ title }}(MFModel): Simulation that this model is a part of. Model is automatically added to simulation when it is initialized. -{{ macros.vars_docs(vars, indent=8) }} +{{ macros.vars_docs(vars)|indent(8, first=true) }} """ super().__init__( diff --git a/flopy/mf6/utils/codegen/templates/package.py.jinja b/flopy/mf6/utils/codegen/templates/package.py.jinja index fc2bf0ebee..821ccdcef5 100644 --- a/flopy/mf6/utils/codegen/templates/package.py.jinja +++ b/flopy/mf6/utils/codegen/templates/package.py.jinja @@ -14,7 +14,7 @@ class Modflow{{ title }}(MFPackage): Parameters ---------- -{{ macros.vars_docs(vars, indent=4) }} +{{ macros.vars_docs(vars)|indent(4, first=true) }} """ {% for attr in vars|attrs %} @@ -25,7 +25,7 @@ class Modflow{{ title }}(MFPackage): self, {{ name|parent }}, loading_package=False, -{{ macros.init_vars(vars, indent=8, skip=name|skip_init) }} +{{ macros.init_vars(vars, skip=name|skip_init)|indent(8, first=true) }} filename=None, pname=None, **kwargs, @@ -43,7 +43,7 @@ class Modflow{{ title }}(MFPackage): Do not set this parameter. It is intended for debugging and internal processing purposes only. -{{ macros.vars_docs(vars, indent=8) }} +{{ macros.vars_docs(vars)|indent(8, first=true) }} filename : str File name for this package. @@ -94,7 +94,7 @@ class {{ title }}Packages(MFChildPackages): def initialize( self, -{{ macros.init_vars(vars, alias=true, indent=8, skip=name|skip_init) }} +{{ macros.init_vars(vars, alias=true, skip=name|skip_init)|indent(8, first=true) }} filename=None, pname=None, ): @@ -112,7 +112,7 @@ class {{ title }}Packages(MFChildPackages): {% if name.r != "obs" %} def append_package( self, -{{ macros.init_vars(vars, alias=true, indent=8, skip=name|skip_init) }} +{{ macros.init_vars(vars, alias=true, skip=name|skip_init)|indent(8, first=true) }} filename=None, pname=None, ): diff --git a/flopy/mf6/utils/codegen/templates/simulation.py.jinja b/flopy/mf6/utils/codegen/templates/simulation.py.jinja index d50df39aa6..41c828a729 100644 --- a/flopy/mf6/utils/codegen/templates/simulation.py.jinja +++ b/flopy/mf6/utils/codegen/templates/simulation.py.jinja @@ -13,7 +13,7 @@ class MF{{ title }}(MFSimulationBase): Parameters ---------- -{{ macros.vars_docs(vars, indent=4) }} +{{ macros.vars_docs(vars)|indent(4, first=true) }} Methods ------- @@ -35,7 +35,7 @@ class MF{{ title }}(MFSimulationBase): write_headers: bool = True, use_pandas: bool = True, lazy_io: bool = False, -{{ macros.init_vars(vars, indent=8, skip=name|skip_init) }} +{{ macros.init_vars(vars, skip=name|skip_init)|indent(8, first=true) }} ): """ {{ name|description }} @@ -66,7 +66,7 @@ class MF{{ title }}(MFSimulationBase): lazy_io Whether to use lazy IO -{{ macros.vars_docs(vars, indent=8) }} +{{ macros.vars_docs(vars)|indent(8, first=true) }} """ super().__init__( From 330895ee7da8c998aae37b11f77dffd5b1ae190c Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 20 Nov 2024 15:37:15 -0500 Subject: [PATCH 74/75] cleanup --- flopy/mf6/utils/codegen/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flopy/mf6/utils/codegen/__init__.py b/flopy/mf6/utils/codegen/__init__.py index 2814430713..a11ffbf42c 100644 --- a/flopy/mf6/utils/codegen/__init__.py +++ b/flopy/mf6/utils/codegen/__init__.py @@ -96,7 +96,7 @@ def _get_template_name(ctx_name) -> str: print(f"Wrote {target_path}") -def make_all(dfndir: Path, outdir: Path, verbose: bool = False): +def make_all(dfndir: Path, outdir: PathLike, verbose: bool = False): """Generate Python source files from the DFN files in the given location.""" from flopy.mf6.utils.codegen.dfn import Dfn From 9a25689db4f01fbe39a83b224a3100c46bf4b0be Mon Sep 17 00:00:00 2001 From: wpbonelli Date: Wed, 4 Dec 2024 17:05:14 -0500 Subject: [PATCH 75/75] fix pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index db4ccfcc3a..5d9e60af0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ dynamic = ["version", "readme"] [project.optional-dependencies] -dev = ["flopy[gen,lint,test,optional,doc]", "tach"] +dev = ["flopy[codegen,lint,test,optional,doc]", "tach"] codegen = [ "boltons>=1.0", "Jinja2>=3.0",