Merge pull request #193 from UrbanSystemsLab/josiahkp

Swap vartype directory structure
UrbanSystemsLab · Nov 5, 2024 · 91d0144 · 91d0144
2 parents 31d7d8e + 526d20e
commit 91d0144
Show file tree

Hide file tree

Showing 2 changed files with 8 additions and 7 deletions.
diff --git a/usl_pipeline/cloud_functions/main.py b/usl_pipeline/cloud_functions/main.py
@@ -679,10 +679,11 @@ def _build_feature_matrix(
             # Write a separate file for each variable type
             # (spatial, spatiotemporal, lu_index).
             for var_type, feature_matrix in feature_matrices.items():
-                feature_folder_name = pathlib.PurePosixPath(chunk_path).with_suffix("")
-                feature_file_name = (feature_folder_name / var_type.value).with_suffix(
-                    ".npy"
-                )
+                feature_path = pathlib.PurePosixPath(chunk_path)
+                feature_path_parent = feature_path.parent
+                feature_file_name = (
+                    feature_path_parent / var_type.value / feature_path.name
+                ).with_suffix(".npy")
                 feature_blob = storage_client.bucket(output_bucket).blob(
                     str(feature_file_name)
                 )

diff --git a/usl_pipeline/cloud_functions/main_test.py b/usl_pipeline/cloud_functions/main_test.py
@@ -407,7 +407,7 @@ def test_build_feature_matrix_wrf(mock_storage_client, mock_firestore_client, _)
     # Create a mock blob for feature matrix we will upload.
     mock_feature_blob = mock.MagicMock()
     mock_feature_blob.name = (
-        "study_area/met_em.d03.2010-02-02_18:00:00/spatiotemporal.npy"
+        "study_area/spatiotemporal/met_em.d03.2010-02-02_18:00:00.npy"
     )
     mock_feature_blob.bucket.name = "climateiq-study-area-feature-chunks"
 
@@ -437,7 +437,7 @@ def test_build_feature_matrix_wrf(mock_storage_client, mock_firestore_client, _)
             mock.call().bucket("climateiq-study-area-feature-chunks"),
             mock.call()
             .bucket()
-            .blob("study_area/met_em.d03.2010-02-02_18:00:00/spatiotemporal.npy"),
+            .blob("study_area/spatiotemporal/met_em.d03.2010-02-02_18:00:00.npy"),
         ]
     )
 
@@ -476,7 +476,7 @@ def test_build_feature_matrix_wrf(mock_storage_client, mock_firestore_client, _)
                     ),
                     "feature_matrix_path": (
                         "gs://climateiq-study-area-feature-chunks/study_area/"
-                        "met_em.d03.2010-02-02_18:00:00/spatiotemporal.npy"
+                        "spatiotemporal/met_em.d03.2010-02-02_18:00:00.npy"
                     ),
                     "time": datetime.datetime(2010, 2, 2, 18, 0, 0),
                     "error": firestore.DELETE_FIELD,