Stream GPM brightness temperature and precip with earthaccess

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
         run: python -c "from tams.data import download_examples; download_examples()"
 
       - name: pytest
-        run: pytest test_tams.py
+        run: pytest -v -ra test_tams.py
 
       - name: Run as script
         run: python -m tams.core

.gitignore

@@ -1,4 +1,5 @@
 *.ipynb_checkpoints
+.dodsrc
 Satellite_data.nc
 MPAS_data.nc
 MPAS_unstructured_data.nc

docs/api.rst

@@ -49,6 +49,8 @@ Data
    tams.data.download_examples
    tams.data.load_example_ir
    tams.data.tb_from_ir
+   tams.data.get_mergir
+   tams.data.get_imerg
 
 
 Utilities

docs/conf.py

@@ -40,6 +40,7 @@
     "geopandas": ("https://geopandas.org/en/stable/", None),
     "dask": ("https://docs.dask.org/en/latest/", None),
     "shapely": ("https://shapely.readthedocs.io/en/stable/", None),
+    "earthaccess": ("https://earthaccess.readthedocs.io/en/stable/", None),
 }
 
 napoleon_google_docstring = False
@@ -77,6 +78,7 @@
     "examples/tracking-options.ipynb",
 ]
 nb_execution_raise_on_error = True
+nb_execution_timeout = 90
 
 myst_enable_extensions = [
     "dollarmath",

docs/examples/get.ipynb

@@ -0,0 +1,142 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "0",
+   "metadata": {},
+   "source": [
+    "# Stream input data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import cartopy.crs as ccrs\n",
+    "import matplotlib as mpl\n",
+    "import matplotlib.pyplot as plt\n",
+    "\n",
+    "import tams"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "2",
+   "metadata": {},
+   "source": [
+    "## GPM precipitation and brightness temperature\n",
+    "\n",
+    "With the help of [earthaccess](https://earthaccess.readthedocs.io/), we can stream these data without downloading the files to disk.\n",
+    "* {func}`tams.data.get_imerg`\n",
+    "* {func}`tams.data.get_mergir`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "\n",
+    "pr = tams.data.get_imerg(\"2024-06-01 02:30\")[\"pr\"]\n",
+    "pr"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "\n",
+    "tb = tams.data.get_mergir(pr.time.item())[\"tb\"]\n",
+    "tb"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "fig = plt.figure(figsize=(13, 5))\n",
+    "ax = fig.add_subplot(projection=ccrs.PlateCarree())\n",
+    "\n",
+    "ax.coastlines(color=\"magenta\")\n",
+    "\n",
+    "tb.plot(x=\"lon\", cmap=\"gist_gray_r\", robust=True, ax=ax)\n",
+    "pr.plot(x=\"lon\", norm=mpl.colors.LogNorm(0.01, pr.quantile(0.99)), alpha=0.85, ax=ax)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6",
+   "metadata": {},
+   "source": [
+    "The white dots in the brightness temperature field above represent missing data.\n",
+    "For most CE identification methods, this will influence results.\n",
+    "Since they are mostly scattered points, not large regions,\n",
+    "a reasonable way to fill in the missing data is a nearest-neighbor interpolation."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%%time\n",
+    "\n",
+    "kws = dict(method=\"nearest\", fill_value=\"extrapolate\", assume_sorted=True)\n",
+    "tb_ = tb.interpolate_na(\"lat\", **kws).interpolate_na(\"lon\", **kws)\n",
+    "\n",
+    "print(f\"{tb.isnull().sum().item() / tb.size:.3%} -> {tb_.isnull().sum().item() / tb_.size:.3%} null\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "box = dict(lon=slice(-115, -80), lat=slice(-53, -20))\n",
+    "tb_.sel(**box).plot(cmap=\"gist_gray_r\", robust=True)\n",
+    "plt.gca().set_aspect(\"equal\", \"box\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.13"
+  },
+  "mystnb": {
+   "execution_raise_on_error": false
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

docs/index.md

@@ -29,6 +29,7 @@ examples/tams-run.ipynb
 examples/tracking-options.ipynb
 examples/sample-mpas-ug-data.ipynb
 examples/identify.ipynb
+examples/get.ipynb
 ```
 
 ```{toctree}

environment-dev.yml

@@ -17,6 +17,7 @@ dependencies:
   # Extras
   - cartopy >=0.21
   - dask-core
+  - earthaccess
   - gdown
   - joblib
   - netcdf4