Add capability to parse nested suites in capgen

CMakeLists.txt

@@ -20,6 +20,7 @@ option(CCPP_RUN_ADVECTION_TEST          "Enable advection regression test" OFF)
 option(CCPP_RUN_CAPGEN_TEST             "Enable capgen regression test" OFF)
 option(CCPP_RUN_DDT_HOST_TEST           "Enable ddt host regression test" OFF)
 option(CCPP_RUN_VAR_COMPATIBILITY_TEST  "Enable variable compatibility regression test" OFF)
+option(CCPP_RUN_NESTED_SUITE_TEST       "Enable nested suite regression test" OFF)
 
 message("")
 message("OPENMP .............................. ${OPENMP}")
@@ -31,6 +32,7 @@ message("CCPP_RUN_ADVECTION_TEST ............. ${CCPP_RUN_ADVECTION_TEST}")
 message("CCPP_RUN_CAPGEN_TEST ................ ${CCPP_RUN_CAPGEN_TEST}")
 message("CCPP_RUN_DDT_HOST_TEST .............. ${CCPP_RUN_DDT_HOST_TEST}")
 message("CCPP_RUN_VAR_COMPATIBILITY_TEST ..... ${CCPP_RUN_VAR_COMPATIBILITY_TEST}")
+message("CCPP_RUN_NESTED_SUITE_TEST .......... ${CCPP_RUN_NESTED_SUITE_TEST}")
 message("")
 
 set(CCPP_VERBOSITY "0" CACHE STRING "Verbosity level of output (default: 0)")

schema/suite_v2_0.xsd

@@ -0,0 +1,159 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
+
+<xs:schema elementFormDefault="qualified"
+           xmlns:xs="http://www.w3.org/2001/XMLSchema">
+
+  <!-- identifier types -->
+
+  <xs:simpleType name="version_type">
+    <xs:restriction base="xs:string">
+      <xs:pattern value="[1-9][0-9]*[.][0-9]+"/>
+    </xs:restriction>
+  </xs:simpleType>
+
+  <xs:simpleType name="fortran_id_type">
+    <xs:restriction base="xs:string">
+      <xs:pattern value="[A-Za-z][A-Za-z0-9_]{0,63}"/>
+    </xs:restriction>
+  </xs:simpleType>
+
+  <xs:simpleType name="subcycle_type">
+    <xs:restriction base="xs:string">
+      <xs:pattern value="[a-z][a-z0-9_]*"/>
+      <xs:pattern value="[1-9][0-9]*"/>
+    </xs:restriction>
+  </xs:simpleType>
+
+  <!-- attributes -->
+
+  <xs:attribute name="version" type="version_type"/>
+
+  <!-- definition of complex types -->
+
+  <xs:complexType name="scheme_type">
+    <xs:simpleContent>
+      <xs:extension base="xs:string">
+        <xs:attribute name="lib" type="xs:string" use="optional"/>
+        <xs:attribute name="version" type="xs:string" use="optional"/>
+      </xs:extension>
+    </xs:simpleContent>
+  </xs:complexType>
+
+  <xs:complexType name="nested_suite_group_type">
+    <xs:simpleContent>
+      <xs:extension base="xs:string">
+        <xs:attribute name="name" type="xs:string" use="required"/>
+        <xs:attribute name="group" type="xs:string" use="required"/>
+        <xs:attribute name="file" type="xs:string" use="optional"/>
+      </xs:extension>
+    </xs:simpleContent>
+  </xs:complexType>
+
+  <xs:complexType name="nested_suite_type">
+    <xs:simpleContent>
+      <xs:extension base="xs:string">
+        <xs:attribute name="name" type="xs:string" use="required"/>
+        <xs:attribute name="file" type="xs:string" use="optional"/>
+      </xs:extension>
+    </xs:simpleContent>
+  </xs:complexType>
+
+  <!-- definition of suite elements -->
+
+  <xs:element name="time_split">
+    <xs:complexType>
+      <xs:choice minOccurs="0" maxOccurs="unbounded">
+        <xs:element ref="time_split"/>
+        <xs:element ref="process_split"/>
+        <xs:element ref="subcycle"/>
+        <xs:element ref="subcol"/>
+        <xs:element name="scheme" type="scheme_type"/>
+      </xs:choice>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="process_split">
+    <xs:complexType>
+      <xs:choice minOccurs="0" maxOccurs="unbounded">
+        <xs:element ref="time_split"/>
+        <xs:element ref="process_split"/>
+        <xs:element ref="subcycle"/>
+        <xs:element ref="subcol"/>
+        <xs:element name="scheme" type="scheme_type"/>
+      </xs:choice>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="subcycle">
+    <xs:complexType>
+      <xs:choice minOccurs="0" maxOccurs="unbounded">
+        <xs:element ref="time_split"/>
+        <xs:element ref="process_split"/>
+        <xs:element ref="subcycle"/>
+        <xs:element ref="subcol"/>
+        <xs:element name="scheme" type="scheme_type"/>
+      </xs:choice>
+      <xs:attribute name="loop" type="subcycle_type" use="optional"/>
+      <xs:attribute name="name" type="xs:string" use="optional"/>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="subcol">
+    <xs:complexType>
+      <xs:choice minOccurs="0" maxOccurs="unbounded">
+        <xs:element ref="time_split"/>
+        <xs:element ref="process_split"/>
+        <xs:element ref="subcycle"/>
+        <xs:element ref="subcol"/>
+        <xs:element name="scheme" type="scheme_type"/>
+      </xs:choice>
+      <xs:attribute name="gen" type="fortran_id_type" use="required"/>
+      <xs:attribute name="avg" type="fortran_id_type" use="required"/>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="group">
+    <xs:complexType>
+      <xs:choice minOccurs="0" maxOccurs="unbounded">
+        <xs:element ref="time_split"/>
+        <xs:element ref="process_split"/>
+        <xs:element ref="subcycle"/>
+        <xs:element ref="subcol"/>
+        <xs:element name="scheme" type="scheme_type"/>
+        <xs:element name="nested_suite" type="nested_suite_group_type"/>
+      </xs:choice>
+      <xs:attribute name="name" type="xs:ID" use="required"/>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="suite">
+    <xs:complexType>
+      <xs:sequence>
+        <xs:choice minOccurs="0" maxOccurs="1">
+          <xs:element name="init"      type="scheme_type"/>
+          <xs:element name="initalize" type="scheme_type"/>
+        </xs:choice>
+        <xs:element ref="group" minOccurs="1" maxOccurs="unbounded"/>
+        <xs:choice minOccurs="0" maxOccurs="unbounded">
+          <xs:element name="nested_suite" type="nested_suite_type"/>
+        </xs:choice>
+        <xs:choice  minOccurs="0" maxOccurs="1">
+          <xs:element name="final"    type="scheme_type"/>
+          <xs:element name="finalize" type="scheme_type"/>
+        </xs:choice>
+      </xs:sequence>
+      <xs:attribute name="name" type="xs:ID"     use="required"/>
+      <xs:attribute name="lib"  type="xs:string" use="optional"/>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="suites">
+    <xs:complexType>
+      <xs:sequence>
+        <xs:element ref="suite" minOccurs="1" maxOccurs="unbounded"/>
+      </xs:sequence>
+      <xs:attribute name="version" type="version_type" use="required"/>
+    </xs:complexType>
+  </xs:element>
+
+</xs:schema>

scripts/ccpp_datafile.py

@@ -25,7 +25,7 @@
 from framework_env import CCPPFrameworkEnv
 from metadata_table import UNKNOWN_PROCESS_TYPE
 from metavar import Var
-from parse_tools import read_xml_file, PrettyElementTree
+from parse_tools import read_xml_file, write_xml_file
 from parse_tools import ParseContext, ParseSource
 from suite_objects import VerticalLoop, Subcycle
 
@@ -1182,8 +1182,7 @@ def generate_ccpp_datatable(run_env, host_model, api, scheme_headers,
     # end for
     _add_dependencies(datatable, scheme_depends, host_depends)
     # Write tree
-    datatable_tree = PrettyElementTree(datatable)
-    datatable_tree.write(run_env.datatable_file)
+    write_xml_file(datatable, run_env.datatable_file)
 
 ###############################################################################
 

scripts/ccpp_suite.py

@@ -19,7 +19,8 @@
 from metavar import Var, VarDictionary, ccpp_standard_var
 from parse_tools import ParseContext, ParseSource
 from parse_tools import ParseInternalError, CCPPError
-from parse_tools import read_xml_file, validate_xml_file, find_schema_version
+from parse_tools import read_xml_file, validate_xml_file, write_xml_file
+from parse_tools import find_schema_version, expand_nested_suites
 from parse_tools import init_log, set_log_to_null
 from suite_objects import CallList, Group, Scheme
 from metavar import CCPP_LOOP_VAR_STDNAMES
@@ -82,7 +83,7 @@ class Suite(VarDictionary):
 
     __scheme_template = '<scheme>{}</scheme>'
 
-    def __init__(self, filename, api, run_env):
+    def __init__(self, filename, suite_xml, api, run_env):
         """Initialize this Suite object from the SDF, <filename>.
         <api> serves as the Suite's parent."""
         self.__run_env = run_env
@@ -114,7 +115,7 @@ def __init__(self, filename, api, run_env):
             raise CCPPError(emsg.format(self.__sdf_name))
         # end if
         # Parse the SDF
-        self.parse(run_env)
+        self.parse(suite_xml, run_env)
 
     @property
     def name(self):
@@ -186,27 +187,13 @@ def new_group_from_name(self, group_name, run_env):
         group_xml = '<group name="{}"></group>'.format(group_name)
         return self.new_group(group_xml, group_name, run_env)
 
-    def parse(self, run_env):
+    def parse(self, suite_xml, run_env):
         """Parse the suite definition file."""
         success = True
-
-        _, suite_xml = read_xml_file(self.__sdf_name, run_env.logger)
         # We do not have line number information for the XML file
         self.__context = ParseContext(filename=self.__sdf_name)
-        # Validate the XML file
-        version = find_schema_version(suite_xml)
-        res = validate_xml_file(self.__sdf_name, 'suite', version,
-                                run_env.logger)
-        if not res:
-            emsg = "Invalid suite definition file, '{}'"
-            raise CCPPError(emsg.format(self.__sdf_name))
-        # end if
         self.__name = suite_xml.get('name')
         self.__module = 'ccpp_{}_cap'.format(self.name)
-        lmsg = "Reading suite definition file for '{}'"
-        if run_env.logger and run_env.logger.isEnabledFor(logging.INFO):
-            run_env.logger.info(lmsg.format(self.name))
-        # end if
         gname = Suite.__register_group_name
         self.__suite_reg_group = self.new_group_from_name(gname, run_env)
         gname = Suite.__initial_group_name
@@ -681,13 +668,48 @@ def __init__(self, sdfs, host_model, scheme_headers, run_env):
                 raise CCPPError(errmsg.format(header.title))
             # end if
         # end for
+
         # Turn the SDF files into Suites
         for sdf in sdfs:
-            suite = Suite(sdf, self, run_env)
-            suite.analyze(self.host_model, scheme_library,
-                          self.__ddt_lib, run_env)
-            self.__suites.append(suite)
+            # Load the suite definition file to determine the schema version,
+            # validate the file, and expand nested suites if applicable
+            _, xml_root = read_xml_file(sdf, run_env.logger)
+            # We do not have line number information for the XML file
+            self.__context = ParseContext(filename=sdf)
+            # Validate the XML file
+            schema_version = find_schema_version(xml_root)
+            res = validate_xml_file(sdf, 'suite', schema_version, run_env.logger)
+            if not res:
+                raise CCPPError(f"Invalid suite definition file, '{sdf}'")
+
+            # Processing of the sdf depends on the schema version
+            if xml_root.tag.lower() == "suite" and schema_version[0] == 1:
+                suite = Suite(sdf, xml_root, self, run_env)
+                suite.analyze(self.host_model, scheme_library,
+                              self.__ddt_lib, run_env)
+                self.__suites.append(suite)
+            elif xml_root.tag.lower() == "suites" and schema_version[0] == 2:
+                # Preprocess the sdf to expand nested suites
+                expand_nested_suites(xml_root, logger=run_env.logger)
+                # Write the expanded sdf to the capgen output directory;
+                # this file isn't used by capgen (everything is in memory
+                # from here onwards), but it is useful for developers/users
+                # (although the output can also be found in the datatable).
+                sdf_expanded = os.path.join(run_env.output_dir,
+                    os.path.split(sdf)[1].replace(".xml", "_expanded.xml"))
+                write_xml_file(xml_root, sdf_expanded, run_env.logger)
+                for suite_item in xml_root:
+                    suite = Suite(sdf, suite_item, self, run_env)
+                    suite.analyze(self.host_model, scheme_library,
+                                  self.__ddt_lib, run_env)
+                    self.__suites.append(suite)
+            else:
+                errmsg = f"Suite XML schema not supported: " + \
+                    "root={xml_root.tag}, version={schema_version}"
+                raise CCPPError(errmsg)
+            # end if
         # end for
+
         # We will need the correct names for errmsg and errcode
         evar = self.host_model.find_variable(standard_name='ccpp_error_message')
         if evar is not None:

scripts/parse_tools/__init__.py

@@ -30,7 +30,7 @@
 from preprocess import PreprocStack
 from xml_tools import find_schema_file, find_schema_version
 from xml_tools import read_xml_file, validate_xml_file
-from xml_tools import PrettyElementTree
+from xml_tools import expand_nested_suites, write_xml_file
 from fortran_conditional import FORTRAN_CONDITIONAL_REGEX_WORDS, FORTRAN_CONDITIONAL_REGEX
 # pylint: enable=wrong-import-position
 
@@ -51,6 +51,7 @@
     'check_valid_values',
     'check_molar_mass',
     'context_string',
+    'expand_nested_suites',
     'find_schema_file',
     'find_schema_version',
     'flush_log',
@@ -65,7 +66,6 @@
     'ParseSyntaxError',
     'ParseObject',
     'PreprocStack',
-    'PrettyElementTree',
     'read_xml_file',
     'register_fortran_ddt_name',
     'registered_fortran_ddt_name',
@@ -78,6 +78,7 @@
     'type_name',
     'unique_standard_name',
     'validate_xml_file',
+    'write_xml_file',
     'FORTRAN_CONDITIONAL_REGEX_WORDS',
     'FORTRAN_CONDITIONAL_REGEX'
 ]