bair-climate-initiative · RitwikGupta · Sep 8, 2022 · Sep 8, 2022
diff --git a/config/smap.yml b/config/smap.yml
@@ -0,0 +1,29 @@
+default_collection:
+  outdir: data
+  datetime: 2022-08-04/2022-08-07
+  aoi_file: config/aoi/conus-expanded.json
+
+providers:
+  - id : EARTHDATA
+    kwargs:
+      subprovider_id: NSIDC_ECS
+
+    collections:
+      # SMAP Enhanced L3 Radiometer Global and Polar Grid Daily 9 km EASE-Grid Soil Moisture V005
+      - id: SPL3SMP_E
+        assets:
+          - all
+
+system:
+  # how many concurrent procs to use for 
+  # extracting the assets
+  # set to -1 to use all available procs
+  max_concurrent_extractions: 5
+
+  # Remove existing files if their filesize does not match the size 
+  # of the asset on the remote server and re-extract
+  remove_existing_if_wrong_size: True
+
+  # If assets are missing file sizes, query using HTTP request
+  # (this can be slow for large collections)
+  query_asset_sizes: False
diff --git a/nbs/smap.ipynb b/nbs/smap.ipynb
@@ -0,0 +1,229 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from datetime import date\n",
+    "import h5py\n",
+    "import matplotlib.pyplot as plt\n",
+    "from matplotlib import colors\n",
+    "from pathlib import Path\n",
+    "import numpy as np\n",
+    "\n",
+    "# Test should run from root\n",
+    "import os\n",
+    "\n",
+    "os.chdir(\"..\")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## SPL3SMP_E\n",
+    "This Level-3 (L3) soil moisture product provides a daily composite of global land surface conditions retrieved by both the Soil Moisture Active Passive (SMAP) radar and radiometer. SMAP L-band soil moisture data are resampled to an Earth-fixed, global, cylindrical 9 km Equal-Area Scalable Earth Grid, Version 2.0 (EASE-Grid 2.0)."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/shared/ritwik/miniconda3/envs/metaearth/lib/python3.8/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from .autonotebook import tqdm as notebook_tqdm\n",
+      "\u001b[34m07:38:27\u001b[0m \u001b[33mINFO\u001b[0m - \u001b[1mExtraction details for provider ProviderKey.EARTHDATA with collection SPL3SMP_E: \n",
+      "id: SPL3SMP_E\n",
+      "assets:\n",
+      "- all\n",
+      "outdir: null\n",
+      "datetime: null\n",
+      "aoi_file: null\n",
+      "max_items: 1\n",
+      "\u001b[0m\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "default_collection:\n",
+      "  id: null\n",
+      "  assets: null\n",
+      "  outdir: data\n",
+      "  datetime: 2022-08-04/2022-08-07\n",
+      "  aoi_file: config/aoi/conus-expanded.json\n",
+      "  max_items: -1\n",
+      "providers:\n",
+      "- id: EARTHDATA\n",
+      "  kwargs:\n",
+      "    subprovider_id: NSIDC_ECS\n",
+      "  collections:\n",
+      "  - id: SPL3SMP_E\n",
+      "    assets:\n",
+      "    - all\n",
+      "    outdir: null\n",
+      "    datetime: null\n",
+      "    aoi_file: null\n",
+      "    max_items: -1\n",
+      "system:\n",
+      "  log_outdir: ./logs\n",
+      "  log_level: INFO\n",
+      "  dry_run: false\n",
+      "  max_concurrent_extractions: 5\n",
+      "  max_download_attempts: 3\n",
+      "  remove_existing_if_wrong_size: true\n",
+      "  query_asset_sizes: false\n",
+      "run_id: ''\n",
+      "\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[34m07:38:28\u001b[0m \u001b[33mINFO\u001b[0m - \u001b[1mEarthData Provider returned 1 items for SPL3SMP_E for datetime 2022-08-04/2022-08-07\u001b[0m\n",
+      "\u001b[34m07:38:28\u001b[0m \u001b[33mINFO\u001b[0m - \u001b[1m2 assets have unknown file size\u001b[0m\n",
+      "\u001b[34m07:38:28\u001b[0m \u001b[33mINFO\u001b[0m - \u001b[1m\n",
+      "\n",
+      "****************************************************************************************************\n",
+      "To Extract:\n",
+      "EarthData Provider: SPL3SMP_E.v005\n",
+      "\n",
+      "\n",
+      "Collection               | Key                 | Description\n",
+      "--------------------------------------------------------------------------------\n",
+      "SPL3SMP_E.v005           | data                | \n",
+      "SPL3SMP_E.v005           | metadata            | \n",
+      "\n",
+      "\n",
+      "Collection size                     0 MB\n",
+      "Size of remaining data to download  0 MB\n",
+      "Number of assets with unknown size: 2\n",
+      "\n",
+      "****************************************************************************************************\n",
+      "\u001b[0m\n",
+      "\u001b[34m07:38:28\u001b[0m \u001b[33mINFO\u001b[0m - \u001b[1mStarting data download\u001b[0m\n",
+      "Assets: 100%|██████████| 2/2 [00:57<00:00, 28.67s/it]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Successfully extracted assets.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "from omegaconf import OmegaConf\n",
+    "from metaearth.api import extract_assets\n",
+    "from metaearth.config import ConfigSchema\n",
+    "\n",
+    "cfg = OmegaConf.load(\"config/smap.yml\")\n",
+    "cfg_schema = OmegaConf.structured(ConfigSchema)\n",
+    "cfg = OmegaConf.merge(cfg_schema, cfg)\n",
+    "\n",
+    "print(OmegaConf.to_yaml(cfg))\n",
+    "\n",
+    "# Constrain the test to only pull one item\n",
+    "for pvdr in cfg.providers:\n",
+    "    for coll in pvdr.collections:\n",
+    "        coll.max_items = 1\n",
+    "\n",
+    "success = extract_assets(cfg)\n",
+    "print(\"Successfully extracted assets.\" if success else \"Asset extraction failed.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Find the correct H5 file and load it\n",
+    "# We avoid hardcoding the path here in case the API changes down the road\n",
+    "\n",
+    "fpath = Path(cfg[\"default_collection\"][\"outdir\"])\n",
+    "collection_for_nb = \"SPL3SMP_E\"\n",
+    "nc_files = list((fpath / collection_for_nb).glob(\"**/*.h5\"))\n",
+    "\n",
+    "# We know there is only one file pulled, assert\n",
+    "assert (\n",
+    "    len(nc_files) == 1\n",
+    "), f\"There were {len(nc_files)} files pulled. `max_items: 1` failed to constrain.\"\n",
+    "\n",
+    "fpath = nc_files[0]\n",
+    "hfile = h5py.File(fpath)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "soil_moisture_am = hfile[\"Soil_Moisture_Retrieval_Data_AM\"][\"soil_moisture\"][:]\n",
+    "soil_moisture_pm = hfile[\"Soil_Moisture_Retrieval_Data_PM\"][\"soil_moisture_pm\"][:]\n",
+    "\n",
+    "# Do not display NODATA values\n",
+    "soil_moisture_am[soil_moisture_am == -9999.0] = np.nan\n",
+    "soil_moisture_pm[soil_moisture_pm == -9999.0] = np.nan\n",
+    "\n",
+    "fig, ax = plt.subplots(2, 1, figsize=(20, 20))\n",
+    "ax[0].imshow(soil_moisture_am)\n",
+    "ax[0].title.set_text(\"Soil moisture AM ($cm^3$)\")\n",
+    "\n",
+    "ax[1].imshow(soil_moisture_pm)\n",
+    "ax[1].title.set_text(\"Soil moisture PM ($cm^3$)\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3.8.13 ('metaearth')",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.13"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "66d3687f8083b32394ec4773685811dd39c313691fed3fb3fd0f0241182948c3"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/setup.cfg b/setup.cfg
@@ -81,6 +81,8 @@ tests =
     netCDF4>=1.6.0
     # for integration tests
     matplotlib
+    # for SMAP integration tests
+    h5py
 
     # for UAVSAR tests
     pyproj>=2.2,<4