DAS-2238 - updates to enable 3D variables without dimension scales to… (

#27) * DAS-2238 - updates to enable 3D variables without dimension scales to work * DAS-2238 - precommit fixes for trailing spaces * DAS-2238 - updates based on PR feedback * DAS-2238 - updates based on PR feedback * DAS-2238 - removed incorrect comment * DAS-2238 - PR updates to comments * DAS-2238 - precommit updates * DAS-2238 - changed the name for get_dimension_names_from_coordinates to create_spatial_dimension_names_from_coordinates
nasa · Jan 31, 2025 · 212fda9 · 212fda9
1 parent cc433c8
commit 212fda9
Show file tree

Hide file tree

Showing 12 changed files with 6,033 additions and 72 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,11 @@
+## v1.1.3
+### 2025-01-29
+
+This version of HOSS supports configuration updates to hoss_config.json to
+add dimension configurations for 3D variables. Functions were updated to provide the
+ability to spatial subset 3D variables for products like SMAP L3 which did
+not have dimension arrays.
+
 ## v1.1.2
 ### 2025-01-20
 
@@ -13,8 +21,10 @@
 This version of HOSS merges the feature branch that contains V1.1.0 to the main branch.
 Additional updates included code quality improvements with additional unit tests, revised methodology
 in functions that selected the data points from the coordinate datasets and calculation of the dimension
-arrays. Functions were added to determine the dimension order for 2D variables.
-
+arrays. Functions were added to determine the dimension order for 2D variables. These updates,
+V1.1.1 and V1.1.0 are entirely to support SMAP L3 data - in particular SPL2SMAP_S, SPL3SMAP, SPL3SMA -
+all of which have “anonymous” dimensions (without dimension names and dimension variables).  No functional
+updates are present, nor changes that effect existing collection support.
 
 ## v1.1.0
 ### 2024-11-25

diff --git a/docker/service_version.txt b/docker/service_version.txt
@@ -1 +1 @@
-1.1.2
+1.1.3
diff --git a/hoss/coordinate_utilities.py b/hoss/coordinate_utilities.py
@@ -12,6 +12,7 @@
     IncompatibleCoordinateVariables,
     InvalidCoordinateData,
     InvalidCoordinateDataset,
+    InvalidDimensionNames,
     MissingCoordinateVariable,
     MissingVariable,
     UnsupportedDimensionOrder,
@@ -60,8 +61,13 @@ def get_variables_with_anonymous_dims(
     return set(
         variable
         for variable in variables
-        if (len(varinfo.get_variable(variable).dimensions) == 0)
-        or (any_absent_dimension_variables(varinfo, variable))
+        if (
+            varinfo.get_variable(variable)
+            and (
+                (len(varinfo.get_variable(variable).dimensions) == 0)
+                or (any_absent_dimension_variables(varinfo, variable))
+            )
+        )
     )
 
 
@@ -70,42 +76,57 @@ def any_absent_dimension_variables(varinfo: VarInfoFromDmr, variable: str) -> bo
     that have been created by opendap, but are not really
     dimension variables
     """
+
     return any(
         varinfo.get_variable(dimension) is None
         for dimension in varinfo.get_variable(variable).dimensions
     )
 
 
-def get_dimension_array_names_from_coordinate_variables(
+def get_dimension_array_names(
     varinfo: VarInfoFromDmr,
     variable_name: str,
 ) -> list[str]:
     """
-    Returns the dimensions names from coordinate variables
+    Returns the dimensions names from coordinate variables or from
+    configuration
     """
+    variable = varinfo.get_variable(variable_name)
+    if variable is None:
+        return []
+
+    dimension_names = variable.dimensions
 
+    if len(dimension_names) >= 2:
+        return dimension_names
+
+    # creating dimension names from coordinates
     latitude_coordinates, longitude_coordinates = get_coordinate_variables(
         varinfo, [variable_name]
     )
-
-    # for one variable, the coordinate array length will always be 1 or 0
+    # Given variable has coordinates: use latitude coordinate
+    # to define variable spatial dimensions.
     if len(latitude_coordinates) == 1 and len(longitude_coordinates) == 1:
-        dimension_array_names = get_dimension_array_names(
+        dimension_array_names = create_spatial_dimension_names_from_coordinates(
             varinfo, latitude_coordinates[0]
         )
-    # if variable does not have coordinates (len = 0)
-    elif (
-        varinfo.get_variable(variable_name).is_latitude()
-        or varinfo.get_variable(variable_name).is_longitude()
-    ):
-        dimension_array_names = get_dimension_array_names(varinfo, variable_name)
+
+    # Given variable variable has no coordinate attribute itself,
+    # but is itself a coordinate (latitude or longitude):
+    # use as a coordinate to define spatial dimensions
+    elif variable.is_latitude() or variable.is_longitude():
+        dimension_array_names = create_spatial_dimension_names_from_coordinates(
+            varinfo, variable_name
+        )
     else:
         dimension_array_names = []
 
     return dimension_array_names
 
 
-def get_dimension_array_names(varinfo: VarInfoFromDmr, variable_name: str) -> str:
+def create_spatial_dimension_names_from_coordinates(
+    varinfo: VarInfoFromDmr, variable_name: str
+) -> str:
     """returns the x-y variable names that would
     match the group of the input variable. The 'dim_y' dimension
     and 'dim_x' names are returned with the group pathname
@@ -138,6 +159,9 @@ def create_dimension_arrays_from_coordinates(
     3) Generate the x-y dimscale array and return to the calling method
 
     """
+    if len(projected_dimension_names) < 2:
+        raise InvalidDimensionNames(projected_dimension_names)
+
     lat_arr = get_2d_coordinate_array(
         prefetch_dataset,
         latitude_coordinate.full_name_path,
@@ -172,7 +196,10 @@ def create_dimension_arrays_from_coordinates(
         col_dim_values, np.transpose(col_indices)[1], col_size
     )
 
-    projected_y, projected_x = tuple(projected_dimension_names)
+    projected_y, projected_x = (
+        projected_dimension_names[-2],
+        projected_dimension_names[-1],
+    )
 
     if dim_order_is_y_x:
         return {projected_y: y_dim, projected_x: x_dim}
@@ -322,8 +349,16 @@ def get_dimension_order_and_dim_values(
     projected spatial dimension. The input lat lon arrays and dimension
     indices are assumed to be 2D in this implementation of the function.
     """
-    lat_arr_values = [lat_array_points[i][j] for i, j in grid_dimension_indices]
-    lon_arr_values = [lon_array_points[i][j] for i, j in grid_dimension_indices]
+    if lat_array_points.ndim == 1 and lon_array_points.ndim == 1:
+        lat_arr_values = lat_array_points
+        lon_arr_values = lon_array_points
+    elif lat_array_points.ndim == 2 and lon_array_points.ndim == 2:
+        lat_arr_values = [lat_array_points[i][j] for i, j in grid_dimension_indices]
+        lon_arr_values = [lon_array_points[i][j] for i, j in grid_dimension_indices]
+    else:
+        # assuming a nominal z,y,x order
+        lat_arr_values = [lat_array_points[0][i][j] for i, j in grid_dimension_indices]
+        lon_arr_values = [lon_array_points[0][i][j] for i, j in grid_dimension_indices]
 
     from_geo_transformer = Transformer.from_crs(4326, crs)
     x_values, y_values = (  # pylint: disable=unpacking-non-sequence

diff --git a/hoss/dimension_utilities.py b/hoss/dimension_utilities.py
@@ -24,7 +24,7 @@
 
 from hoss.coordinate_utilities import (
     get_coordinate_variables,
-    get_dimension_array_names_from_coordinate_variables,
+    get_dimension_array_names,
 )
 from hoss.exceptions import (
     InvalidIndexSubsetRequest,
@@ -441,17 +441,17 @@ def add_index_range(
     if variable.dimensions:
         variable_dimensions = variable.dimensions
     else:
-        # Anonymous dimensions, so check for dimension derived from coordinates:
-        variable_dimensions = get_dimension_array_names_from_coordinate_variables(
-            varinfo, variable_name
-        )
+        # Anonymous dimensions, so check for dimension derived from coordinates
+        # or from configuration
+        variable_dimensions = get_dimension_array_names(varinfo, variable_name)
 
     range_strings = get_range_strings(variable_dimensions, index_ranges)
 
     if all(range_string == '[]' for range_string in range_strings):
         indices_string = ''
     else:
         indices_string = ''.join(range_strings)
+
     return f'{variable_name}{indices_string}'
 
 

diff --git a/hoss/exceptions.py b/hoss/exceptions.py
@@ -208,6 +208,18 @@ def __init__(self, coordinate_name):
         )
 
 
+class InvalidDimensionNames(CustomError):
+    """This exception is raised when the list of dimension names
+    is not what is expected. It has to be at least 2 dimensions.
+    """
+
+    def __init__(self, dimension_names: str):
+        super().__init__(
+            'InvalidDimensionNames',
+            f'Dimension Names "{dimension_names}" not valid.',
+        )
+
+
 class UnsupportedDimensionOrder(CustomError):
     """This exception is raised when the granule file included in the input
     request is not the nominal dimension order which is 'y,x'.

diff --git a/hoss/hoss_config.json b/hoss/hoss_config.json
@@ -1,6 +1,6 @@
 {
   "Identification": "hoss_config",
-  "Version": 19,
+  "Version": 20,
   "CollectionShortNamePath": [
     "/HDF5_GLOBAL/short_name",
     "/NC_GLOBAL/short_name",
@@ -314,6 +314,48 @@
       ],
       "_Description": "Ensure variables in /Soil_Moisture_Retrieval_Data_Polar_PM group point to correct coordinate variables."
     },
+    {
+      "Applicability": {
+        "Mission": "SMAP",
+        "ShortNamePath": "SPL3FT(A|P|P_E)",
+        "VariablePattern": "^/Freeze_Thaw_Retrieval_Data(?:_(Global|Polar))/(transition_direction$|transition_state_flag$)"
+      },
+      "Attributes": [
+        {
+          "Name": "dimensions",
+          "Value": "y_dim x_dim"
+        }
+      ],
+      "_Description": "Only these fully referenced variables are 2D, with this rule overriding the 3D rule defined broadly for all variables"
+    },
+    {
+      "Applicability": {
+        "Mission": "SMAP",
+        "ShortNamePath": "SPL3FT(A|P|P_E)",
+        "VariablePattern": "^/Freeze_Thaw_Retrieval_Data(?:_(Global|Polar))/((?!transition_state_flag$)(?!transition_direction$).)*$|/Radar_Data/.*"
+      },
+      "Attributes": [
+        {
+          "Name": "dimensions",
+          "Value": "am_pm y_dim x_dim"
+        }
+      ],
+      "_Description": "SMAP L3 data are HDF5 and without dimension settings. Overrides here define the dimensions, a useful reference name, and critically, the dimension order."
+    },
+    {
+      "Applicability": {
+        "Mission": "SMAP",
+        "ShortNamePath": "SPL3SMP",
+        "VariablePattern": ".*landcover.*"
+      },
+      "Attributes": [
+        {
+          "Name": "dimensions",
+          "Value": "y_dim x_dim lc_type"
+        }
+      ],
+      "_Description": "SMAP L3 data are HDF5 and without dimension settings. Overrides here define the dimensions, a useful reference name, and critically, the dimension order."
+    },
     {
       "Applicability": {
         "Mission": "ICESat2",

diff --git a/hoss/spatial.py b/hoss/spatial.py
@@ -38,7 +38,7 @@
 from hoss.coordinate_utilities import (
     create_dimension_arrays_from_coordinates,
     get_coordinate_variables,
-    get_dimension_array_names_from_coordinate_variables,
+    get_dimension_array_names,
     get_variables_with_anonymous_dims,
 )
 from hoss.dimension_utilities import (
@@ -247,9 +247,7 @@ def get_x_y_index_ranges_from_coordinates(
 
     crs = get_variable_crs(non_spatial_variable, varinfo)
 
-    projected_dimension_names = get_dimension_array_names_from_coordinate_variables(
-        varinfo, non_spatial_variable
-    )
+    projected_dimension_names = get_dimension_array_names(varinfo, non_spatial_variable)
 
     dimension_arrays = create_dimension_arrays_from_coordinates(
         prefetch_coordinate_datasets,