European Centre for Medium-Range Weather Forecasts
GRIB_subCentre :
0
Conventions :
CF-1.7
institution :
European Centre for Medium-Range Weather Forecasts
history :
GRIB to CDM+CF via cfgrib-0.9.5/ecCodes-2.17.0 with source='/Users/baudouin/git/climet/test.grib', filter_by_keys={}, encode_cf=('parameter', 'time', 'geography', 'vertical')
"
+ "
GRIB_edition :
1
GRIB_centre :
ecmf
GRIB_centreDescription :
European Centre for Medium-Range Weather Forecasts
GRIB_subCentre :
0
Conventions :
CF-1.7
institution :
European Centre for Medium-Range Weather Forecasts
history :
GRIB to CDM+CF via cfgrib-0.9.5/ecCodes-2.17.0 with source='/Users/baudouin/git/climet/test.grib', filter_by_keys={}, encode_cf=('parameter', 'time', 'geography', 'vertical')
"
],
"text/plain": [
" Size: 2kB\n",
diff --git a/docs/examples/source/index.rst b/docs/examples/source/index.rst
index 2a41989c0..19c693c7c 100644
--- a/docs/examples/source/index.rst
+++ b/docs/examples/source/index.rst
@@ -6,6 +6,7 @@ Data sources
.. toctree::
:maxdepth: 1
+ data.ipynb
files.ipynb
multi_files.ipynb
file_parts.ipynb
diff --git a/src/earthkit/data/field/component/time.py b/src/earthkit/data/field/component/time.py
index 7a9073857..56b516e09 100644
--- a/src/earthkit/data/field/component/time.py
+++ b/src/earthkit/data/field/component/time.py
@@ -70,6 +70,12 @@ def forecast_month(self):
"""Return the forecast month."""
pass
+ @mark_get_key
+ @abstractmethod
+ def indexing_datetime(self):
+ """Return the indexing datetime."""
+ pass
+
def create_time(d: dict) -> "BaseTime":
"""Create a BaseTime object from a dictionary.
@@ -115,6 +121,9 @@ def step(self):
def forecast_month(self):
return None
+ def indexing_datetime(self):
+ return None
+
@classmethod
def from_dict(cls, d):
return cls()
@@ -171,6 +180,10 @@ def forecast_month(self):
"""Return the forecast month."""
return None
+ def indexing_datetime(self):
+ """Return the indexing datetime."""
+ return None
+
def to_dict(self):
return {
"valid_datetime": self.valid_datetime(),
@@ -401,6 +414,10 @@ def forecast_month(self):
"""Return the forecast month."""
return None
+ def indexing_datetime(self):
+ """Return the indexing datetime."""
+ return None
+
def to_dict(self):
return {
"valid_datetime": self.valid_datetime,
@@ -457,6 +474,7 @@ def __init__(
base_datetime=None,
step=None,
forecast_month=None,
+ indexing_datetime=None,
):
if base_datetime is not None:
self._base_datetime = to_datetime(base_datetime)
@@ -468,16 +486,45 @@ def __init__(
if self._forecast_month is not None:
self._forecast_month = int(self._forecast_month)
+ self._indexing_datetime = indexing_datetime
+ if self._indexing_datetime is not None:
+ self._indexing_datetime = to_datetime(self._indexing_datetime)
+
+ def base_datetime(self):
+ """Return the base datetime of the time object."""
+ return self._base_datetime
+
+ def base_date(self):
+ """Return the base datetime of the time object."""
+ return self._base_datetime.date()
+
+ def base_time(self):
+ """Return the base datetime of the time object."""
+ return self._base_datetime.time()
+
+ def valid_datetime(self):
+ """Return the valid datetime of the time object."""
+ return self._base_datetime + self._step
+
+ def step(self):
+ """Return the forecast period of the time object."""
+ return self._step
+
def forecast_month(self):
"""Return the forecast month."""
return self._forecast_month
+ def indexing_datetime(self):
+ """Return the indexing datetime."""
+ return self._indexing_datetime
+
def to_dict(self):
return {
"valid_datetime": self.valid_datetime,
"base_datetime": self.base_datetime,
"step": self.step,
"forecast_month": self.forecast_month,
+ "indexing_datetime": self.indexing_datetime,
}
@classmethod
@@ -487,12 +534,14 @@ def from_base_datetime(
base_datetime=None,
step=None,
forecast_month=None,
+ indexing_datetime=None,
):
"""Set the base datetime of the time object."""
return cls(
base_datetime=base_datetime,
step=step,
forecast_month=forecast_month,
+ indexing_datetime=indexing_datetime,
)
@classmethod
@@ -506,6 +555,8 @@ def from_valid_datetime(
*,
valid_datetime=None,
step=None,
+ forecast_month=None,
+ indexing_datetime=None,
):
"""Set the valid datetime of the time object."""
valid_datetime = to_datetime(valid_datetime)
@@ -514,6 +565,8 @@ def from_valid_datetime(
return cls(
base_datetime=base_datetime,
step=step,
+ forecast_month=forecast_month,
+ indexing_datetime=indexing_datetime,
)
@classmethod
@@ -522,6 +575,8 @@ def from_base_datetime_and_valid_datetime(
*,
base_datetime=None,
valid_datetime=None,
+ forecast_month=None,
+ indexing_datetime=None,
):
valid_datetime = to_datetime(valid_datetime)
base_datetime = to_datetime(base_datetime)
@@ -529,6 +584,8 @@ def from_base_datetime_and_valid_datetime(
return cls(
base_datetime=base_datetime,
step=step,
+ forecast_month=forecast_month,
+ indexing_datetime=indexing_datetime,
)
@classmethod
@@ -543,14 +600,17 @@ def from_dict(cls, d, allow_unused=False):
"base_date",
"base_time",
"forecast_month",
+ "indexing_datetime",
}
d1 = cls.normalise_create_kwargs(d, allowed_keys=KEYS, remove_nones=True)
- d1.pop("valid_datetime", None)
+ d1_reduced = d1.copy()
+ d1_reduced.pop("forecast_month", None)
+ d1_reduced.pop("indexing_datetime", None)
+ d1_reduced.pop("valid_datetime", None)
- keys_s = set(d1.keys())
- found = tuple(sorted(list(KEYS.intersection(keys_s))))
+ found = tuple(sorted(d1_reduced.keys()))
METHODS = {
("base_datetime",): cls.from_base_datetime,
("base_datetime", "step"): cls.from_base_datetime,
@@ -559,87 +619,15 @@ def from_dict(cls, d, allow_unused=False):
("base_date", "base_time", "step"): cls.from_base_date_and_time,
}
- method, _ = METHODS.get(found)
+ method = METHODS.get(found)
if method:
- d1 = {k: d1[k] for k in found}
data = method(**d1)
return data
- if not d1:
- return cls()
-
- # raise ValueError(f"Invalid keys in data: {list(d.keys())}. Expected one of {cls._CREATE_KEYS}.")
-
- def __getstate__(self):
- state = {}
- state["base_datetime"] = self._base_datetime
- state["step"] = self._step
- return state
-
- def __setstate__(self, state):
- self.__init__(
- base_datetime=state["base_datetime"],
- step=state["step"],
- )
-
-
-@component_keys
-class TimeOri(SimpleFieldComponent):
- _base_datetime = None
- _step = ZERO_TIMEDELTA
-
- def __init__(
- self,
- base_datetime=None,
- step=None,
- ):
- if base_datetime is not None:
- self._base_datetime = to_datetime(base_datetime)
-
- if step is not None:
- self._step = to_timedelta(step)
-
- @mark_get_key
- def base_datetime(self):
- """Return the base datetime of the time object."""
- return self._base_datetime
-
- @mark_get_key
- def valid_datetime(self):
- """Return the valid datetime of the time object."""
- return self._base_datetime + self._step
-
- @mark_get_key
- def step(self):
- """Return the forecast period of the time object."""
- return self._step
-
- @mark_get_key
- def base_date(self):
- """Return the base date of the time object."""
- return self._base_datetime.date()
-
- @mark_get_key
- def base_time(self):
- """Return the base time of the time object."""
- return self._base_datetime.time()
-
- @mark_alias("base_datetime")
- def forecast_reference_time(self):
- """Return the forecast reference time (alias of `base_datetime`)."""
- return self.base_datetime()
-
- @mark_alias("step")
- def forecast_period(self):
- """Return the forecast period (alias of `step`)."""
- return self.step()
+ raise ValueError(f"Invalid keys in data: {list(d.keys())}. Expected one of {KEYS}.")
- def to_dict(self):
- return {
- "valid_datetime": self.valid_datetime,
- "base_datetime": self.base_datetime,
- "step": self.step,
- }
+ def set(self, *args, **kwargs):
+ raise NotImplementedError("Setting values on MonthlyForecastTime is not implemented yet.")
def __getstate__(self):
state = {}
@@ -652,267 +640,3 @@ def __setstate__(self, state):
base_datetime=state["base_datetime"],
step=state["step"],
)
-
- @classmethod
- def from_dict(cls, d, allow_unused=False):
- if not isinstance(d, dict):
- raise TypeError("data must be a dictionary")
-
- d1 = normalise_create_kwargs(
- cls, d, allowed_keys=CREATE_METHOD_MAP.core_keys, allow_unused=allow_unused, remove_nones=True
- )
-
- method, method_kwargs = CREATE_METHOD_MAP.get(d1.keys())
- if method:
- d1 = {k: d1[k] for k in method_kwargs if k in d1}
- data = method(**d1)
- return data
-
- if not d1:
- return cls()
-
- raise ValueError(f"Invalid keys in data: {list(d.keys())}. Expected one of {cls._CREATE_KEYS}.")
-
- @classmethod
- def from_base_date_and_time(cls, *, base_date=None, base_time=None, step=None):
- dt = datetime_from_date_and_time(base_date, base_time)
- return cls.from_base_datetime(base_datetime=dt, step=step)
-
- @classmethod
- def from_date_and_time(cls, *, date=None, time=None, step=None):
- dt = datetime_from_date_and_time(date, time)
- return cls.from_base_datetime(base_datetime=dt, step=step)
-
- @classmethod
- def from_valid_datetime(
- cls,
- *,
- valid_datetime=None,
- step=None,
- ):
- """Set the valid datetime of the time object."""
- valid_datetime = to_datetime(valid_datetime)
- step = to_timedelta(step) if step is not None else ZERO_TIMEDELTA
- base_datetime = valid_datetime - step
- return cls(
- base_datetime=base_datetime,
- step=step,
- )
-
- @classmethod
- def from_base_datetime_and_valid_datetime(
- cls,
- *,
- base_datetime=None,
- valid_datetime=None,
- ):
- valid_datetime = to_datetime(valid_datetime)
- base_datetime = to_datetime(base_datetime)
- step = valid_datetime - base_datetime
- return cls(
- base_datetime=base_datetime,
- step=step,
- )
-
- @classmethod
- def from_base_datetime(
- cls,
- *,
- base_datetime=None,
- step=None,
- ):
- """Set the base datetime of the time object."""
- return cls(
- base_datetime=base_datetime,
- step=step,
- )
-
- def set(self, *args, **kwargs):
- d = normalise_set_kwargs(
- self, *args, allowed_keys=SET_METHOD_MAP.core_keys, remove_nones=True, **kwargs
- )
- method, method_kwargs = SET_METHOD_MAP.get(d.keys())
- if method:
- # method = getattr(cls, method_name)
- d = {k: d[k] for k in method_kwargs if k in d}
- return method(self, **d)
-
- return None
-
- def _set_generic(
- self,
- *,
- base_datetime=None,
- step=None,
- ):
- d = self.to_dict()
- d.pop("valid_datetime", None)
-
- def _add(key, value):
- if value is not None:
- d[key] = value
-
- _add("base_datetime", base_datetime)
- _add("step", step)
-
- return TimeOri(**d)
-
- def _set_valid_datetime(self, *, valid_datetime=None):
- valid_datetime = to_datetime(valid_datetime)
- step = valid_datetime - self.base_datetime
- return self._set_generic(step=step)
-
- def _set_valid_datetime_and_step(self, *, valid_datetime=None, step=None):
- valid_datetime = to_datetime(valid_datetime)
- step = to_timedelta(step)
- base_datetime = valid_datetime - step
- return self._set_generic(base_datetime=base_datetime, step=step)
-
- def _set_base_datetime_and_valid_datetime(
- self,
- *,
- base_datetime=None,
- valid_datetime=None,
- ):
- base_datetime = to_datetime(base_datetime)
- valid_datetime = to_datetime(valid_datetime)
- step = valid_datetime - base_datetime
- return self._set_generic(base_datetime=base_datetime, step=step)
-
- def _set_base_datetime_valid_datetime_and_step(
- self,
- *,
- base_datetime=None,
- valid_datetime=None,
- step=None,
- ):
- base_datetime = to_datetime(base_datetime)
- valid_datetime = to_datetime(valid_datetime)
- step = to_timedelta(step)
- if valid_datetime - base_datetime != step:
- raise ValueError(f"Inconsistent step value. {base_datetime=} + {step=} != {valid_datetime=}")
- return self._set_generic(base_datetime=base_datetime, step=step)
-
- # def _set_date(self, *, date):
- # time = self.base_datetime.time()
- # dt = datetime_from_date_and_time(date, time)
- # return self._set_generic(base_datetime=dt)
-
- # def _set_date_and_time(self, *, date, time=None):
- # if time is None:
- # time = self.base_datetime.time()
- # dt = datetime_from_date_and_time(date, time)
- # return self._set_generic(base_datetime=dt)
-
- def _set_base_date_and_time(self, *, base_date, base_time=None, step=None):
- if base_time is None:
- base_time = self.base_datetime.time()
- if step is None:
- step = self.step
- dt = datetime_from_date_and_time(base_date, base_time)
- return self._set_generic(base_datetime=dt, step=step)
-
-
-# class AnalysisTime(Time):
-# pass
-
-
-# class ForecastTime(Time):
-# pass
-
-
-# class MonthlyForecastTime(Time):
-# pass
-
-
-class MethodMap:
- def __init__(self, core_keys, extra_keys, methods):
- self.core_keys = core_keys
- self.extra_keys = extra_keys or set()
- self.methods = methods
-
- assert self.core_keys
- assert isinstance(self.core_keys, set)
- assert isinstance(self.extra_keys, set)
- # assert self.methods
- assert isinstance(self.methods, dict)
-
- self.mapping = {}
- self.method_kwargs = {}
- for k, method in self.methods.items():
- keys = sorted(list(k))
- if method not in self.method_kwargs:
- self.method_kwargs[method] = self.get_method_kwargs(method)
-
- kwargs = self.method_kwargs[method]
- assert set(keys).issubset(set(kwargs)), f"{method=} {keys=} {kwargs=}"
- assert set(keys).issubset(self.core_keys), f"{method=} {keys=} {self.core_keys=}"
- key = tuple(keys)
- self.mapping[key] = method
- self.method_kwargs[method] = kwargs
-
- # self.ALL_KEYS = set(self.CORE_KEYS).union(set(self.EXTRA_KEYS))
-
- def get(self, keys):
- keys_s = set(keys)
- found = tuple(sorted(list(self.core_keys.intersection(keys_s))))
- found = self.reduce(found)
- method = self.mapping.get(found, None)
- return method, self.method_kwargs.get(method)
-
- def get_method_kwargs(self, method):
- import inspect
-
- f = method
- r = inspect.signature(f)
- v = []
- for p in r.parameters.values():
- if p.kind == p.KEYWORD_ONLY:
- v.append(p.name)
- return v
-
- def reduce(self, keys):
- if len(keys) <= 1:
- return keys
- return tuple([k for k in keys if k not in self.extra_keys])
-
-
-CREATE_METHOD_MAP = MethodMap(
- core_keys={"valid_datetime", "base_datetime", "step", "base_date", "base_time", "date", "time"},
- extra_keys=set(),
- methods={
- # ("valid_datetime",): Time.from_valid_datetime,
- # ("valid_datetime", "step"): Time.from_valid_datetime,
- # ("base_datetime",): Time.from_base_datetime,
- # ("base_datetime", "valid_datetime"): Time.from_base_datetime_and_valid_datetime,
- # ("base_datetime", "step"): Time.from_base_datetime,
- # ("base_date",): Time.from_base_date_and_time,
- # ("base_date", "base_time"): Time.from_base_date_and_time,
- # ("base_date", "base_time", "step"): Time.from_base_date_and_time,
- # ("date",): Time.from_date_and_time,
- # ("date", "time"): Time.from_date_and_time,
- # ("date", "time", "step"): Time.from_date_and_time,
- },
-)
-
-
-SET_METHOD_MAP = MethodMap(
- core_keys={"valid_datetime", "base_datetime", "step", "base_date", "base_time"},
- extra_keys=set(),
- methods={
- # ("step",): Time._set_generic,
- # ("base_datetime",): Time._set_generic,
- # ("base_datetime", "step"): Time._set_generic,
- # ("valid_datetime",): Time._set_valid_datetime,
- # ("valid_datetime", "step"): Time._set_valid_datetime_and_step,
- # ("base_datetime", "valid_datetime"): Time._set_base_datetime_and_valid_datetime,
- # (
- # "base_datetime",
- # "valid_datetime",
- # "step",
- # ): Time._set_base_datetime_valid_datetime_and_step,
- # ("base_date",): Time._set_base_date_and_time,
- # ("base_date", "base_time"): Time._set_base_date_and_time,
- # ("base_date", "base_time", "step"): Time._set_base_date_and_time,
- },
-)
diff --git a/src/earthkit/data/field/grib/time.py b/src/earthkit/data/field/grib/time.py
index 0d63da8e8..e675b45f9 100644
--- a/src/earthkit/data/field/grib/time.py
+++ b/src/earthkit/data/field/grib/time.py
@@ -10,7 +10,6 @@
from earthkit.data.utils.dates import datetime_from_grib
from earthkit.data.utils.dates import datetime_to_grib
from earthkit.data.utils.dates import step_to_grib
-from earthkit.data.utils.dates import to_datetime
from earthkit.data.utils.dates import to_timedelta
from .collector import GribContextCollector
@@ -41,15 +40,7 @@ def _datetime(date_key, time_key):
return datetime_from_grib(date, time)
return None
- hdate = _get("hdate")
- if hdate is not None:
- time = _get("dataTime")
- base = datetime_from_grib(hdate, time)
- else:
- base = _datetime("dataDate", "dataTime")
-
- end = None
- # time_span = ZERO_TIMEDELTA
+ base = _datetime("dataDate", "dataTime")
end = _get("endStep")
if end is None:
@@ -59,26 +50,23 @@ def _datetime(date_key, time_key):
end = ZERO_TIMEDELTA
else:
end = to_timedelta(end)
- # start = _get("startStep")
- # if start is not None:
- # start = to_timedelta(start)
- # time_span = end - start
-
- # indexing = _datetime("indexingDate", "indexingTime")
- # reference = _datetime("referenceDate", "referenceTime")
-
- # time_span_method = _get("stepType", "instant").lower()
- # time_span_method = _GRIB_TO_METHOD.get(time_span_method, TimeSpanMethod.INSTANT)
- # time_span = TimeSpan(time_span, time_span_method)
- return dict(
- base_datetime=to_datetime(base),
+ r = dict(
+ base_datetime=base,
step=end,
- # time_span=time_span,
- # indexing_datetime=indexing,
- # reference_datetime=reference,
)
+ fc_month = None
+ indexing = None
+ if handle.is_defined("forecastMonth"):
+ fc_month = _get("forecastMonth")
+ if fc_month is not None:
+ indexing = _datetime("indexingDate", "indexingTime")
+ r["forecast_month"] = fc_month
+ r["indexing_datetime"] = indexing
+
+ return r
+
class GribTimeContextCollector(GribContextCollector):
@staticmethod
@@ -92,15 +80,13 @@ def collect_keys(handler, context):
r["time"] = time
r["step"] = step
- # if spec.time_span.value != ZERO_TIMEDELTA:
- # start = spec.step - spec.time_span.value
- # start = step_to_grib(start)
- # end = step
- # r["stepRange"] = step_range_to_grib(start, end)
- # else:
- # r["stepRange"] = str(step)
+ if component.forecast_month is not None:
+ r["forecastMonth"] = component.forecast_month
+ if component.indexing_datetime is not None:
+ idate, itime = datetime_to_grib(component.indexing_datetime)
+ r["indexingDate"] = idate
+ r["indexingTime"] = itime
- # r["stepType"] = _METHOD_TO_GRIB[spec.time_span.method]
context.update(r)
diff --git a/src/earthkit/data/field/mars/vertical.py b/src/earthkit/data/field/mars/vertical.py
index 75d1766a5..43ec0452e 100644
--- a/src/earthkit/data/field/mars/vertical.py
+++ b/src/earthkit/data/field/mars/vertical.py
@@ -14,13 +14,25 @@
class Converter:
- def from_mars(self, value):
+ def _to_number(self, value):
+ if isinstance(value, str):
+ try:
+ if value.isdecimal():
+ return float(value)
+ elif value.isdigit():
+ return int(value)
+ except (ValueError, TypeError):
+ pass
return value
+ def from_mars(self, value):
+ return self._to_number(value)
+
class PressurePaConverter(Converter):
def from_mars(self, value):
- if value is not None:
+ value = self._to_number(value)
+ if value is not None and isinstance(value, (int, float)):
return value / 100.0
return value
diff --git a/src/earthkit/data/utils/concat.py b/src/earthkit/data/utils/concat.py
index f22b83fe6..e07c91dab 100644
--- a/src/earthkit/data/utils/concat.py
+++ b/src/earthkit/data/utils/concat.py
@@ -35,12 +35,11 @@ def concat(*args):
raise ValueError(f"Cannot concatenate type={type(arg)} object")
elif isinstance(arg, Source):
data.append(arg)
- print(f"concat: data={data}, has_data={has_data}")
from earthkit.data.sources import from_source
result = from_source("multi", *data)
- print(f"concat: result={result}")
+
if not has_data:
if result is None or not hasattr(result, "_source"):
raise ValueError("concat could not create a valid Data object from the provided arguments")
diff --git a/src/earthkit/data/utils/html.py b/src/earthkit/data/utils/html.py
index c8992a2da..17fb50b32 100644
--- a/src/earthkit/data/utils/html.py
+++ b/src/earthkit/data/utils/html.py
@@ -61,7 +61,7 @@ def table_from_dict(vals, title=None):
t = ""
if title is not None and title:
- t = f"