Skip to content

Commit

Permalink
feat: Add telemac support
Browse files Browse the repository at this point in the history
fixes #27
  • Loading branch information
pmav99 committed Jun 5, 2024
1 parent 999de49 commit 54640be
Show file tree
Hide file tree
Showing 10 changed files with 500 additions and 344 deletions.
26 changes: 13 additions & 13 deletions .github/workflows/run_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,20 +7,20 @@ on:
- main
- master
paths:
- '**.py'
- '.github/workflows/test*.yml'
- 'pyproject.toml'
- 'poetry.lock'
- 'conda-lock.yml'
- 'requirements/*'
- "**.py"
- ".github/workflows/test*.yml"
- "pyproject.toml"
- "poetry.lock"
- "conda-lock.yml"
- "requirements/*"
pull_request:
paths:
- '**.py'
- '.github/workflows/test*.yml'
- 'pyproject.toml'
- 'poetry.lock'
- 'conda-lock.yml'
- 'requirements/*'
- "**.py"
- ".github/workflows/test*.yml"
- "pyproject.toml"
- "poetry.lock"
- "conda-lock.yml"
- "requirements/*"

jobs:
run_tests:
Expand All @@ -30,7 +30,7 @@ jobs:
matrix:
os:
- "ubuntu"
- "macos"
# - "macos"
python-version:
- "3.9"
- "3.10"
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ list:
@LC_ALL=C $(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | sort | grep -E -v -e '^[^[:alnum:]]' -e '^$@$$'

init:
poetry install --with=dev -E grib
poetry install --with=dev -E all
pre-commit install

style:
Expand Down
70 changes: 52 additions & 18 deletions inspectds/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,23 +15,41 @@
if TYPE_CHECKING:
import xarray

if importlib.util.find_spec("cfgrib"):
IS_GRIB_AVAILABLE = True
else:
IS_GRIB_AVAILABLE = False
IS_GRIB_AVAILABLE = bool(importlib.util.find_spec("cfgrib"))
IS_SELAFIN_AVAILABLE = bool(importlib.util.find_spec("xarray_selafin"))


if IS_GRIB_AVAILABLE:
if IS_GRIB_AVAILABLE and IS_SELAFIN_AVAILABLE:
SUPPORTED_DATASETS = {"netcdf", "zarr", "grib", "selafin"}

class DATASET_TYPE(str, enum.Enum):
AUTO = "auto"
NETCDF = "netcdf"
ZARR = "zarr"
GRIB = "grib"
SELAFIN = "selafin"

elif IS_GRIB_AVAILABLE:
SUPPORTED_DATASETS = {
"grib",
"netcdf",
"zarr",
"grib",
}

class DATASET_TYPE(str, enum.Enum):
class DATASET_TYPE(str, enum.Enum): # type: ignore[no-redef]
AUTO = "auto"
NETCDF = "netcdf"
ZARR = "zarr"
GRIB = "grib"

elif IS_SELAFIN_AVAILABLE:
SUPPORTED_DATASETS = {"netcdf", "zarr", "selafin"}

class DATASET_TYPE(str, enum.Enum): # type: ignore[no-redef]
AUTO = "auto"
NETCDF = "netcdf"
ZARR = "zarr"
SELAFIN = "selafin"

else:
SUPPORTED_DATASETS = {
Expand Down Expand Up @@ -92,11 +110,18 @@ def echo_dataset(
# 3. we want to allow the user to override the inferring
# This is why we keep this function
def infer_dataset_type(path: Path) -> DATASET_TYPE:
if path.suffix in (".grib", ".grib2"):
if path.suffix in (".grb", ".grib", ".grib2"):
if IS_GRIB_AVAILABLE:
dataset_type = DATASET_TYPE.GRIB
else:
msg = "GRIB file detected, but GRIB support is not available. Please install cfgrib."
msg = "GRIB file detected, but GRIB support is not available. Please install 'cfgrib'."
typer.echo(msg)
raise typer.Exit(code=1)
elif path.suffix in (".slf", ".selafin"):
if IS_SELAFIN_AVAILABLE:
dataset_type = DATASET_TYPE.SELAFIN
else:
msg = "SELAFIN file detected, but SELAFIN support is not available. Please install 'xarray-selafin'."
typer.echo(msg)
raise typer.Exit(code=1)
elif path.is_dir() or path.suffix == ".zarr" or path.suffix == ".zip":
Expand Down Expand Up @@ -128,6 +153,13 @@ def get_kwargs(dataset_type: DATASET_TYPE) -> dict[str, Any]:
engine="netcdf4",
)
)
elif IS_SELAFIN_AVAILABLE and dataset_type == DATASET_TYPE.SELAFIN:
open_dataset_kwargs.update(
dict(
mask_and_scale=None,
engine="selafin",
)
)
else:
raise ValueError("WTF??? Unknown Dataset type...")
return open_dataset_kwargs
Expand All @@ -150,7 +182,8 @@ def version_callback(value: bool) -> None:
@app.command(no_args_is_help=True)
def inspect_dataset(
path: Path = typer.Argument(dir_okay=True, file_okay=True, exists=True, readable=True, help="The path to the dataset"),
dataset_type: DATASET_TYPE = typer.Option(default=DATASET_TYPE.AUTO, help="The dataset type. If 'auto', then it gets inferred from PATH"),
#dataset_type: DATASET_TYPE = typer.Option(default=DATASET_TYPE.AUTO.value, help="The dataset type. If 'auto', then it gets inferred from PATH"),
dataset_type: DATASET_TYPE = typer.Option(default="auto", help="The dataset type. If 'auto', then it gets inferred from PATH"),
mask_and_scale: bool = typer.Option(False, help="Whether to mask and scale the dataset"),
dimensions: bool = typer.Option(default=True, help="Whether to include 'Dimensions' in the output"),
coordinates: bool = typer.Option(default=True, help="Whether to include 'Coordinates' in the output"),
Expand All @@ -161,24 +194,25 @@ def inspect_dataset(
version: bool = typer.Option(False, "--version", help="Display the version", callback=version_callback, is_eager=True),
) -> int: # fmt: skip
xr = lazy_imports.import_xarray()

if dataset_type == DATASET_TYPE.AUTO:
dataset_type = infer_dataset_type(path)

open_dataset_kwargs = get_kwargs(dataset_type=dataset_type)

# Some netcdf files are not compatible with Xarray
# More specifically you can't have a dimension as a variable too.
# https://github.com/pydata/xarray/issues/1709#issuecomment-343714896
# When we find such variables we drop them:
drop_variables: list[str] = []
default_open_kwargs = {
"filename_or_obj": path,
"mask_and_scale": "mask_and_scale",
"drop_variables": "drop_variables",
}
dataset_type_open__kwargs = get_kwargs(dataset_type=dataset_type)
open_kwargs = default_open_kwargs | dataset_type_open__kwargs
while True:
try:
ds = xr.open_dataset(
filename_or_obj=path,
mask_and_scale=mask_and_scale,
drop_variables=drop_variables,
**open_dataset_kwargs,
)
ds = xr.open_dataset(**open_kwargs)
except Exception as exc:
if "already exists as a scalar variable" in str(exc):
to_be_dropped = str(exc).split("'")[-2]
Expand Down
Loading

0 comments on commit 54640be

Please sign in to comment.