Bug fixes for ccpp_prebuild to work with partially case-insensitive capgen parser

scripts/ccpp_prebuild.py

@@ -13,6 +13,7 @@
 import sys
 
 # CCPP framework imports
+from common import lowercase_keys_and_values
 from common import encode_container, decode_container, decode_container_as_dict
 from common import CCPP_STAGES, CCPP_INTERNAL_VARIABLES, CCPP_STATIC_API_MODULE, CCPP_INTERNAL_VARIABLE_DEFINITON_FILE
 from common import STANDARD_VARIABLE_TYPES, STANDARD_INTEGER_TYPE, CCPP_TYPE
@@ -361,11 +362,12 @@ def check_schemes_in_suites(arguments, suites):
     """Check that all schemes that are requested in the suites exist"""
     success = True
     logging.info("Checking for existence of schemes in suites ...")
+    argument_keys = [x.lower() for x in arguments.keys()]
     for suite in suites:
         for group in suite.groups:
             for subcycle in group.subcycles:
                 for scheme_name in subcycle.schemes:
-                    if not scheme_name in arguments.keys():
+                    if not scheme_name in argument_keys:
                         success = False
                         logging.critical("Scheme {} in suite {} cannot be found".format(scheme_name, suite.name))
     return success
@@ -401,19 +403,19 @@ def filter_metadata(metadata, arguments, dependencies, schemes_in_files, suites)
     # Filter argument lists
     for scheme in arguments.keys():
         for suite in suites:
-            if scheme in suite.all_schemes_called:
+            if scheme.lower() in suite.all_schemes_called:
                 arguments_filtered[scheme] = arguments[scheme]
                 break
     # Filter dependencies
     for scheme in dependencies.keys():
         for suite in suites:
-            if scheme in suite.all_schemes_called:
+            if scheme.lower() in suite.all_schemes_called:
                 dependencies_filtered[scheme] = dependencies[scheme]
                 break
     # Filter schemes_in_files
     for scheme in schemes_in_files.keys():
         for suite in suites:
-            if scheme in suite.all_schemes_called:
+            if scheme.lower() in suite.all_schemes_called:
                 schemes_in_files_filtered[scheme] = schemes_in_files[scheme]
     return (success, metadata_filtered, arguments_filtered, dependencies_filtered, schemes_in_files_filtered)
 
@@ -749,6 +751,9 @@ def main():
         logging.info('CCPP prebuild clean completed successfully, exiting.')
         sys.exit(0)
 
+    # Convert TYPEDEFS_NEW_METATA config to lowercase
+    config['typedefs_new_metadata'] = lowercase_keys_and_values(config['typedefs_new_metadata'])
+
     # If no suite definition files were given, get all of them
     if not sdfs:
         (success, sdfs) = get_all_suites(config['suites_dir'])

scripts/ccpp_track_variables.py

@@ -10,6 +10,7 @@
 from metadata_table import find_scheme_names, parse_metadata_file
 from ccpp_prebuild import import_config, gather_variable_definitions
 from mkstatic import Suite
+from common import lowercase_keys
 from parse_checkers import registered_fortran_ddt_names
 from parse_tools import init_log, set_log_level
 from framework_env import CCPPFrameworkEnv
@@ -78,7 +79,7 @@ def create_metadata_filename_dict(metapath):
         # The above returns a list of schemes in each filename, but
         # we want a dictionary of schemes associated with filenames:
         for scheme in schemes:
-            metadata_dict[scheme] = scheme_fn
+            metadata_dict[scheme.lower()] = scheme_fn
 
     return metadata_dict
 

scripts/common.py

@@ -101,13 +101,14 @@ def split_var_name_and_array_reference(var_name):
 def encode_container(*args):
     """Encodes a container, i.e. the location of a metadata table for CCPP.
     Currently, there are three possibilities with different numbers of input
-    arguments: module, module+typedef, module+scheme+subroutine."""
+    arguments: module, module+typedef, module+scheme+subroutine. Convert all
+    names to lowercase to support the new case-insensitive capgen parser."""
     if len(args)==3:
-        container = 'MODULE_{0} SCHEME_{1} SUBROUTINE_{2}'.format(*args)
+        container = 'MODULE_{0} SCHEME_{1} SUBROUTINE_{2}'.format(*[arg.lower() for arg in args])
     elif len(args)==2:
-        container = 'MODULE_{0} TYPE_{1}'.format(*args)
+        container = 'MODULE_{0} TYPE_{1}'.format(*[arg.lower() for arg in args])
     elif len(args)==1:
-        container = 'MODULE_{0}'.format(*args)
+        container = 'MODULE_{0}'.format(*[arg.lower() for arg in args])
     else:
         raise Exception("encode_container not implemented for {0} arguments".format(len(args)))
     return container
@@ -184,3 +185,49 @@ def string_to_python_identifier(string):
         return string
     else:
         raise Exception("Resulting string '{0}' is not a valid Python identifier".format(string))
+
+# New utilities added 2025/07/25 for dealing with case-insensitivity changes from capgen
+
+def lowercase_keys_and_values(d):
+    """Recursively convert all keys and values in a regular dictionary to lowercase"""
+    if isinstance(d, dict):
+        return {
+            (k.lower() if isinstance(k, str) else k):
+                lowercase_keys_and_values(v)
+            for k, v in d.items()
+        }
+    elif isinstance(d, list):
+        return [lowercase_keys_and_values(item) for item in d]
+    elif isinstance(d, str):
+        return d.lower()
+    else:
+        return d
+
+def lowercase_keys(d):
+    """Recursively convert all keys in an OrderedDict to lowercase"""
+    if isinstance(d, OrderedDict):
+        new_dict = OrderedDict()
+        for k, v in d.items():
+            new_key = k.lower() if isinstance(k, str) else k
+            new_dict[new_key] = lowercase_keys(v)
+        return new_dict
+    elif isinstance(d, list):
+        return [lowercase_keys(item) for item in d]
+    else:
+        return d
+
+def lowercase_xml(element):
+    """Recursively convert XML elements to lowercase"""
+    # Lowercase the tag name
+    element.tag = element.tag.lower()
+    # Lowercase the text content, if it exists
+    if element.text:
+        element.text = element.text.lower()
+    if element.tail:
+        element.tail = element.tail.lower()
+    # Lowercase attribute keys and values
+    element.attrib = {k.lower(): v.lower() for k, v in element.attrib.items()}
+    # Recurse into child elements
+    for i in range(len(element)):
+        element[i] = lowercase_xml(element[i])
+    return element

scripts/metadata_parser.py

@@ -236,12 +236,12 @@ def read_new_metadata(filename, module_name, table_name, scheme_name = None, sub
                 # units from capgen to the "+"-format (i.e. "m2 s-2" --> "m+2 s-2")
                 units = insert_plus_sign_for_positive_exponents(new_var.get_prop_value('units'))
 
-                var = Var(standard_name = standard_name,
+                var = Var(standard_name = standard_name.lower(),
                           long_name     = new_var.get_prop_value('long_name') + legacy_note,
                           units         = units,
-                          local_name    = new_var.get_prop_value('local_name'),
+                          local_name    = new_var.get_prop_value('local_name').lower(),
                           type          = new_var.get_prop_value('type').lower(),
-                          dimensions    = dimensions,
+                          dimensions    = [dim.lower() for dim in dimensions],
                           container     = container,
                           kind          = kind,
                           intent        = new_var.get_prop_value('intent'),

scripts/mkstatic.py

@@ -13,6 +13,7 @@
 import xml.etree.ElementTree as ET
 
 from common import encode_container
+from common import lowercase_keys, lowercase_xml
 from common import CCPP_STAGES
 from common import CCPP_T_INSTANCE_VARIABLE, CCPP_ERROR_CODE_VARIABLE, CCPP_ERROR_MSG_VARIABLE, CCPP_LOOP_COUNTER, CCPP_LOOP_EXTENT
 from common import CCPP_BLOCK_NUMBER, CCPP_BLOCK_COUNT, CCPP_BLOCK_SIZES, CCPP_THREAD_NUMBER, CCPP_THREAD_COUNT, CCPP_INTERNAL_VARIABLES
@@ -291,6 +292,23 @@ class API(object):
    public :: {subroutines}
 
    contains
+
+   ! Necessary to convert incoming suite and group names to lowercase
+   function to_lower(str) result(lower)
+      implicit none
+      character(len=*), intent(in) :: str
+      character(len=len(str)) :: lower
+      integer :: i, ichar_val
+
+      do i = 1, len(str)
+         ichar_val = ichar(str(i:i))
+         if (ichar_val >= ichar('A') .and. ichar_val <= ichar('Z')) then
+            lower(i:i) = char(ichar_val + 32)
+         else
+            lower(i:i) = str(i:i)
+         end if
+      end do
+   end function to_lower
 '''
 
     sub = '''
@@ -310,7 +328,7 @@ class API(object):
 {suite_switch}
       else
 
-         write({ccpp_var_name}%errmsg,'(*(a))') 'Invalid suite ' // trim(suite_name)
+         write({ccpp_var_name}%errmsg,'(*(a))') 'Invalid suite ' // to_lower(trim(suite_name))
          ierr = 1
 
       end if
@@ -436,15 +454,15 @@ def write(self):
                         clause = 'else if'
                     argument_list_group = create_argument_list_wrapped_explicit(group.arguments[ccpp_stage])
                     group_calls += '''
-            {clause} (trim(group_name)=="{group_name}") then
+            {clause} (to_lower(trim(group_name))=="{group_name}") then
                ierr = {suite_name}_{group_name}_{stage}_cap({arguments})'''.format(clause=clause,
                                                                                    suite_name=group.suite,
                                                                                    group_name=group.name,
                                                                                    stage=CCPP_STAGES[ccpp_stage],
                                                                                    arguments=argument_list_group)
                 group_calls += '''
             else
-               write({ccpp_var_name}%errmsg, '(*(a))') 'Group ' // trim(group_name) // ' not found'
+               write({ccpp_var_name}%errmsg, '(*(a))') 'Group ' // to_lower(trim(group_name)) // ' not found'
                ierr = 1
             end if
 '''.format(ccpp_var_name=ccpp_var.local_name, group_name=group.name)
@@ -463,7 +481,7 @@ def write(self):
                 else:
                     clause = 'else if'
                 suite_switch += '''
-      {clause} (trim(suite_name)=="{suite_name}") then
+      {clause} (to_lower(trim(suite_name))=="{suite_name}") then
 
          if (present(group_name)) then
 {group_calls}
@@ -690,19 +708,8 @@ def parse(self, make_call_tree=False):
             return success
 
         tree = ET.parse(self._sdf_name)
-        suite_xml = tree.getroot()
+        suite_xml = lowercase_xml(tree.getroot())
         self._name = suite_xml.get('name')
-        # Validate name of suite in XML tag against filename; could be moved to common.py
-        if not (os.path.basename(self._sdf_name) == '{}.xml'.format(self._name)):
-            if (os.path.basename(self._sdf_name) == 'suite_{}.xml'.format(self._name)):
-                logging.debug("Parsing suite using legacy naming convention")
-                logging.debug(f"Filename {os.path.basename(self._sdf_name)}")
-                logging.debug(f"Suite name {format(self._name)}")
-            else:
-                logging.critical("Invalid suite name {0} in suite definition file {1}.".format(
-                                                                   self._name, self._sdf_name))
-                success = False
-                return success
 
         # Check if suite name is too long
         if len(self._name) > SUITE_NAME_MAX_CHARS:
@@ -726,15 +733,15 @@ def parse(self, make_call_tree=False):
 
             self._call_tree[group_xml.attrib['name']] = []
             # Add suite-wide init scheme to group 'init', similar for finalize
-            if group_xml.tag.lower() == 'init' or group_xml.tag.lower() == 'finalize':
+            if group_xml.tag == 'init' or group_xml.tag == 'finalize':
                 self._all_schemes_called.append(group_xml.text)
-                self._all_subroutines_called.append(group_xml.text + '_' + group_xml.tag.lower())
+                self._all_subroutines_called.append(group_xml.text + '_' + group_xml.tag)
                 schemes = [group_xml.text]
                 subcycles.append(Subcycle(loop=1, schemes=schemes))
-                if group_xml.tag.lower() == 'init':
-                    self._groups.append(Group(name=group_xml.tag.lower(), subcycles=subcycles, suite=self._name, init=True))
-                elif group_xml.tag.lower() == 'finalize':
-                    self._groups.append(Group(name=group_xml.tag.lower(), subcycles=subcycles, suite=self._name, finalize=True))
+                if group_xml.tag == 'init':
+                    self._groups.append(Group(name=group_xml.tag, subcycles=subcycles, suite=self._name, init=True))
+                elif group_xml.tag == 'finalize':
+                    self._groups.append(Group(name=group_xml.tag, subcycles=subcycles, suite=self._name, finalize=True))
                 continue
 
             # Parse subcycles of all regular groups
@@ -761,7 +768,6 @@ def parse(self, make_call_tree=False):
         self._all_schemes_called = list(set(self._all_schemes_called))
         self._all_subroutines_called = list(set(self._all_subroutines_called))
 
-
         return success
 
     def print_debug(self):
@@ -1058,10 +1064,13 @@ def __init__(self, **kwargs):
         for key, value in kwargs.items():
             setattr(self, "_"+key, value)
 
-    def write(self, metadata_request, metadata_define, arguments, debug):
+    def write(self, metadata_request, metadata_define, arguments_in, debug):
         """Create caps for all stages of this group. Add additional code for
         debugging if debug flag is True."""
 
+        # First, convert all keys in arguments_in to lowercase (recursively)
+        arguments = lowercase_keys(arguments_in)
+
         # Create an inverse lookup table of local variable names defined (by the host model) and standard names
         standard_name_by_local_name_define = collections.OrderedDict()
         for standard_name in metadata_define.keys():
@@ -1127,8 +1136,7 @@ def write(self, metadata_request, metadata_define, arguments, debug):
 
                     # First, add a few mandatory variables to the list of required
                     # variables. This is mostly for handling horizontal dimensions
-                    # correctly for the different CCPP phases and for cases when
-                    # blocked data structures or chunked arrays are used.
+                    # correctly for the different CCPP phases and for chunked arrays
                     additional_variables_required = []
                     if CCPP_HORIZONTAL_LOOP_EXTENT in metadata_define.keys():
                         for add_var in [ CCPP_CONSTANT_ONE, CCPP_HORIZONTAL_LOOP_EXTENT]:

test/unit_tests/test_common.py

@@ -47,18 +47,18 @@ def test_encode_container(self):
         typename = "COMPLEX"
         schemename = "testscheme"
         subroutinename = "testsubroutine"
-        self.assertEqual(common.encode_container(modulename),f"MODULE_{modulename}")
-        self.assertEqual(common.encode_container(modulename,typename),f"MODULE_{modulename} TYPE_{typename}")
+        self.assertEqual(common.encode_container(modulename),f"MODULE_{modulename.lower()}")
+        self.assertEqual(common.encode_container(modulename,typename),f"MODULE_{modulename.lower()} TYPE_{typename.lower()}")
         self.assertEqual(common.encode_container(modulename,schemename,subroutinename),
-                         f"MODULE_{modulename} SCHEME_{schemename} SUBROUTINE_{subroutinename}")
+                         f"MODULE_{modulename.lower()} SCHEME_{schemename.lower()} SUBROUTINE_{subroutinename.lower()}")
         self.assertRaises(Exception,common.encode_container,modulename,typename,schemename,subroutinename)
         self.assertRaises(Exception,common.encode_container)
 
     def test_decode_container(self):
         """Test decode_container() function"""
 
-        modulename = "ABCD1234"
-        typename = "COMPLEX"
+        modulename = "abcd1234"
+        typename = "complex"
         schemename = "testscheme"
         subroutinename = "testsubroutine"
         self.assertEqual(common.decode_container(f"MODULE_{modulename}"),f"MODULE {modulename}")