Add capability to parse nested suites in capgen

.github/workflows/capgen_unit_tests.yaml

@@ -25,7 +25,11 @@ jobs:
           python3 \
           git \
           libxml2-utils
-        pip install --user pytest
+        python -m pip install --upgrade pip
+        pip install pytest
+        which xmllint
+        xmllint --version
+        which pytest
 
     - name: Build the framework
       run: |
@@ -42,11 +46,11 @@ jobs:
       run: |
         BUILD_DIR=./build \
         PYTHONPATH=test/:scripts/ \
-            pytest \
-              test/capgen_test/capgen_test_reports.py \
-              test/advection_test/advection_test_reports.py \
-              test/ddthost_test/ddthost_test_reports.py \
-              test/var_compatibility_test/var_compatibility_test_reports.py
+          pytest \
+            test/capgen_test/capgen_test_reports.py \
+            test/advection_test/advection_test_reports.py \
+            test/ddthost_test/ddthost_test_reports.py \
+            test/var_compatibility_test/var_compatibility_test_reports.py
 
     - name: Run Fortran to metadata test
       run: cd test && ./test_fortran_to_metadata.sh

.github/workflows/python.yaml

@@ -20,8 +20,15 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
+        sudo apt-get update
+        sudo apt-get install -y \
+          libxml2-utils
         python -m pip install --upgrade pip
         pip install pytest
+        which xmllint
+        xmllint --version
+        which pytest
+
     - name: Test with pytest
       run: |
         export PYTHONPATH=$(pwd)/scripts:$(pwd)/scripts/parse_tools

CMakeLists.txt

@@ -20,6 +20,7 @@ option(CCPP_RUN_ADVECTION_TEST          "Enable advection regression test" OFF)
 option(CCPP_RUN_CAPGEN_TEST             "Enable capgen regression test" OFF)
 option(CCPP_RUN_DDT_HOST_TEST           "Enable ddt host regression test" OFF)
 option(CCPP_RUN_VAR_COMPATIBILITY_TEST  "Enable variable compatibility regression test" OFF)
+option(CCPP_RUN_NESTED_SUITE_TEST       "Enable nested suite regression test" OFF)
 
 message("")
 message("OPENMP .............................. ${OPENMP}")
@@ -31,6 +32,7 @@ message("CCPP_RUN_ADVECTION_TEST ............. ${CCPP_RUN_ADVECTION_TEST}")
 message("CCPP_RUN_CAPGEN_TEST ................ ${CCPP_RUN_CAPGEN_TEST}")
 message("CCPP_RUN_DDT_HOST_TEST .............. ${CCPP_RUN_DDT_HOST_TEST}")
 message("CCPP_RUN_VAR_COMPATIBILITY_TEST ..... ${CCPP_RUN_VAR_COMPATIBILITY_TEST}")
+message("CCPP_RUN_NESTED_SUITE_TEST .......... ${CCPP_RUN_NESTED_SUITE_TEST}")
 message("")
 
 set(CCPP_VERBOSITY "0" CACHE STRING "Verbosity level of output (default: 0)")

schema/suite_v1_0.xsd

@@ -26,7 +26,7 @@
 
   <!-- attributes -->
 
-  <xs:attribute name="version"        type="version_type"/>
+  <xs:attribute name="version" type="version_type"/>
 
   <!-- definition of complex types -->
 

schema/suite_v2_0.xsd

@@ -0,0 +1,150 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
+
+<xs:schema elementFormDefault="qualified"
+           xmlns:xs="http://www.w3.org/2001/XMLSchema">
+
+  <!-- identifier types -->
+
+  <xs:simpleType name="version_type">
+    <xs:restriction base="xs:string">
+      <xs:pattern value="[1-9][0-9]*[.][0-9]+"/>
+    </xs:restriction>
+  </xs:simpleType>
+
+  <xs:simpleType name="fortran_id_type">
+    <xs:restriction base="xs:string">
+      <xs:pattern value="[A-Za-z][A-Za-z0-9_]{0,63}"/>
+    </xs:restriction>
+  </xs:simpleType>
+
+  <xs:simpleType name="subcycle_type">
+    <xs:restriction base="xs:string">
+      <xs:pattern value="[a-z][a-z0-9_]*"/>
+      <xs:pattern value="[1-9][0-9]*"/>
+    </xs:restriction>
+  </xs:simpleType>
+
+  <!-- attributes -->
+
+  <xs:attribute name="version" type="version_type"/>
+
+  <!-- definition of complex types -->
+
+  <xs:complexType name="scheme_type">
+    <xs:simpleContent>
+      <xs:extension base="xs:string">
+        <xs:attribute name="version" type="xs:string" use="optional"/>
+      </xs:extension>
+    </xs:simpleContent>
+  </xs:complexType>
+
+  <xs:complexType name="nested_suite_group_type">
+    <xs:simpleContent>
+      <xs:extension base="xs:string">
+        <xs:attribute name="name" type="xs:string" use="required"/>
+        <xs:attribute name="group" type="xs:string" use="required"/>
+        <xs:attribute name="file" type="xs:string" use="required"/>
+      </xs:extension>
+    </xs:simpleContent>
+  </xs:complexType>
+
+  <xs:complexType name="nested_suite_type">
+    <xs:simpleContent>
+      <xs:extension base="xs:string">
+        <xs:attribute name="name" type="xs:string" use="required"/>
+        <xs:attribute name="group" type="xs:string" use="optional"/>
+        <xs:attribute name="file" type="xs:string" use="required"/>
+      </xs:extension>
+    </xs:simpleContent>
+  </xs:complexType>
+
+  <!-- definition of suite elements -->
+
+  <xs:element name="time_split">
+    <xs:complexType>
+      <xs:choice minOccurs="0" maxOccurs="unbounded">
+        <xs:element ref="time_split"/>
+        <xs:element ref="process_split"/>
+        <xs:element ref="subcycle"/>
+        <xs:element ref="subcol"/>
+        <xs:element name="scheme" type="scheme_type"/>
+      </xs:choice>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="process_split">
+    <xs:complexType>
+      <xs:choice minOccurs="0" maxOccurs="unbounded">
+        <xs:element ref="time_split"/>
+        <xs:element ref="process_split"/>
+        <xs:element ref="subcycle"/>
+        <xs:element ref="subcol"/>
+        <xs:element name="scheme" type="scheme_type"/>
+      </xs:choice>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="subcycle">
+    <xs:complexType>
+      <xs:choice minOccurs="0" maxOccurs="unbounded">
+        <xs:element ref="time_split"/>
+        <xs:element ref="process_split"/>
+        <xs:element ref="subcycle"/>
+        <xs:element ref="subcol"/>
+        <xs:element name="scheme" type="scheme_type"/>
+      </xs:choice>
+      <xs:attribute name="loop" type="subcycle_type" use="optional"/>
+      <xs:attribute name="name" type="xs:string" use="optional"/>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="subcol">
+    <xs:complexType>
+      <xs:choice minOccurs="0" maxOccurs="unbounded">
+        <xs:element ref="time_split"/>
+        <xs:element ref="process_split"/>
+        <xs:element ref="subcycle"/>
+        <xs:element ref="subcol"/>
+        <xs:element name="scheme" type="scheme_type"/>
+      </xs:choice>
+      <xs:attribute name="gen" type="fortran_id_type" use="required"/>
+      <xs:attribute name="avg" type="fortran_id_type" use="required"/>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="group">
+    <xs:complexType>
+      <xs:choice minOccurs="0" maxOccurs="unbounded">
+        <xs:element ref="time_split"/>
+        <xs:element ref="process_split"/>
+        <xs:element ref="subcycle"/>
+        <xs:element ref="subcol"/>
+        <xs:element name="scheme" type="scheme_type"/>
+        <xs:element name="nested_suite" type="nested_suite_group_type"/>
+      </xs:choice>
+      <xs:attribute name="name" type="xs:ID" use="required"/>
+    </xs:complexType>
+  </xs:element>
+
+  <xs:element name="suite">
+    <xs:complexType>
+      <xs:sequence>
+        <xs:choice minOccurs="0" maxOccurs="1">
+          <xs:element name="init"      type="scheme_type"/>
+          <xs:element name="initalize" type="scheme_type"/>
+        </xs:choice>
+        <xs:element ref="group" minOccurs="0" maxOccurs="unbounded"/>
+        <xs:choice minOccurs="0" maxOccurs="unbounded">
+          <xs:element name="nested_suite" type="nested_suite_type"/>
+        </xs:choice>
+        <xs:choice  minOccurs="0" maxOccurs="1">
+          <xs:element name="final"    type="scheme_type"/>
+          <xs:element name="finalize" type="scheme_type"/>
+        </xs:choice>
+      </xs:sequence>
+      <xs:attribute name="name"    type="xs:ID"        use="required"/>
+      <xs:attribute name="version" type="version_type" use="required"/>
+    </xs:complexType>
+  </xs:element>
+
+</xs:schema>

scripts/ccpp_datafile.py

@@ -25,7 +25,7 @@
 from framework_env import CCPPFrameworkEnv
 from metadata_table import UNKNOWN_PROCESS_TYPE
 from metavar import Var
-from parse_tools import read_xml_file, PrettyElementTree
+from parse_tools import read_xml_file, write_xml_file
 from parse_tools import ParseContext, ParseSource
 from suite_objects import VerticalLoop, Subcycle
 
@@ -1182,8 +1182,7 @@ def generate_ccpp_datatable(run_env, host_model, api, scheme_headers,
     # end for
     _add_dependencies(datatable, scheme_depends, host_depends)
     # Write tree
-    datatable_tree = PrettyElementTree(datatable)
-    datatable_tree.write(run_env.datatable_file)
+    write_xml_file(datatable, run_env.datatable_file)
 
 ###############################################################################
 

scripts/ccpp_suite.py

@@ -19,7 +19,8 @@
 from metavar import Var, VarDictionary, ccpp_standard_var
 from parse_tools import ParseContext, ParseSource
 from parse_tools import ParseInternalError, CCPPError
-from parse_tools import read_xml_file, validate_xml_file, find_schema_version
+from parse_tools import read_xml_file, validate_xml_file, write_xml_file
+from parse_tools import find_schema_version, expand_nested_suites
 from parse_tools import init_log, set_log_to_null
 from suite_objects import CallList, Group, Scheme
 from metavar import CCPP_LOOP_VAR_STDNAMES
@@ -82,7 +83,7 @@ class Suite(VarDictionary):
 
     __scheme_template = '<scheme>{}</scheme>'
 
-    def __init__(self, filename, api, run_env):
+    def __init__(self, filename, suite_xml, api, run_env):
         """Initialize this Suite object from the SDF, <filename>.
         <api> serves as the Suite's parent."""
         self.__run_env = run_env
@@ -114,7 +115,7 @@ def __init__(self, filename, api, run_env):
             raise CCPPError(emsg.format(self.__sdf_name))
         # end if
         # Parse the SDF
-        self.parse(run_env)
+        self.parse(suite_xml, run_env)
 
     @property
     def name(self):
@@ -186,27 +187,13 @@ def new_group_from_name(self, group_name, run_env):
         group_xml = '<group name="{}"></group>'.format(group_name)
         return self.new_group(group_xml, group_name, run_env)
 
-    def parse(self, run_env):
+    def parse(self, suite_xml, run_env):
         """Parse the suite definition file."""
         success = True
-
-        _, suite_xml = read_xml_file(self.__sdf_name, run_env.logger)
         # We do not have line number information for the XML file
         self.__context = ParseContext(filename=self.__sdf_name)
-        # Validate the XML file
-        version = find_schema_version(suite_xml)
-        res = validate_xml_file(self.__sdf_name, 'suite', version,
-                                run_env.logger)
-        if not res:
-            emsg = "Invalid suite definition file, '{}'"
-            raise CCPPError(emsg.format(self.__sdf_name))
-        # end if
         self.__name = suite_xml.get('name')
         self.__module = 'ccpp_{}_cap'.format(self.name)
-        lmsg = "Reading suite definition file for '{}'"
-        if run_env.logger and run_env.logger.isEnabledFor(logging.INFO):
-            run_env.logger.info(lmsg.format(self.name))
-        # end if
         gname = Suite.__register_group_name
         self.__suite_reg_group = self.new_group_from_name(gname, run_env)
         gname = Suite.__initial_group_name
@@ -681,13 +668,46 @@ def __init__(self, sdfs, host_model, scheme_headers, run_env):
                 raise CCPPError(errmsg.format(header.title))
             # end if
         # end for
+
         # Turn the SDF files into Suites
         for sdf in sdfs:
-            suite = Suite(sdf, self, run_env)
-            suite.analyze(self.host_model, scheme_library,
-                          self.__ddt_lib, run_env)
-            self.__suites.append(suite)
+            # Load the suite definition file to determine the schema version,
+            # validate the file, and expand nested suites if applicable
+            _, xml_root = read_xml_file(sdf, run_env.logger)
+            # We do not have line number information for the XML file
+            self.__context = ParseContext(filename=sdf)
+            # Validate the XML file
+            schema_version = find_schema_version(xml_root)
+            _ = validate_xml_file(sdf, 'suite', schema_version, run_env.logger)
+
+            # Write the expanded sdf to the capgen output directory.
+            # This file isn't used by capgen (everything is in memory
+            # from here onwards), but it is useful for developers/users
+            # (although the output can also be found in the datatable).
+            (sdf_path, sdf_name) = os.path.split(sdf)
+            sdf_expanded = os.path.join(run_env.output_dir,
+                sdf_name.replace(".xml", "_expanded.xml"))
+            if schema_version[0] in [1, 2]:
+                # Preprocess the sdf to expand nested suites
+                if schema_version[0] == 2:
+                    expand_nested_suites(xml_root, sdf_path, logger=run_env.logger)
+                # For both versions 1 and 2, write the SDF (expanded for
+                # version 2, original for version 1) to the current directory
+                write_xml_file(xml_root, sdf_expanded, run_env.logger)
+                # Validate the expanded SDF for version 2
+                if schema_version[0] == 2:
+                    _ = validate_xml_file(sdf, 'suite', schema_version, run_env.logger)
+                suite = Suite(sdf, xml_root, self, run_env)
+                suite.analyze(self.host_model, scheme_library,
+                              self.__ddt_lib, run_env)
+                self.__suites.append(suite)
+            else:
+                errmsg = f"Suite XML schema not supported: " + \
+                    "root={xml_root.tag}, version={schema_version}"
+                raise CCPPError(errmsg)
+            # end if
         # end for
+
         # We will need the correct names for errmsg and errcode
         evar = self.host_model.find_variable(standard_name='ccpp_error_message')
         if evar is not None:

scripts/parse_tools/__init__.py

@@ -30,7 +30,7 @@
 from preprocess import PreprocStack
 from xml_tools import find_schema_file, find_schema_version
 from xml_tools import read_xml_file, validate_xml_file
-from xml_tools import PrettyElementTree
+from xml_tools import expand_nested_suites, write_xml_file
 from fortran_conditional import FORTRAN_CONDITIONAL_REGEX_WORDS, FORTRAN_CONDITIONAL_REGEX
 # pylint: enable=wrong-import-position
 
@@ -51,6 +51,7 @@
     'check_valid_values',
     'check_molar_mass',
     'context_string',
+    'expand_nested_suites',
     'find_schema_file',
     'find_schema_version',
     'flush_log',
@@ -65,7 +66,6 @@
     'ParseSyntaxError',
     'ParseObject',
     'PreprocStack',
-    'PrettyElementTree',
     'read_xml_file',
     'register_fortran_ddt_name',
     'registered_fortran_ddt_name',
@@ -78,6 +78,7 @@
     'type_name',
     'unique_standard_name',
     'validate_xml_file',
+    'write_xml_file',
     'FORTRAN_CONDITIONAL_REGEX_WORDS',
     'FORTRAN_CONDITIONAL_REGEX'
 ]