diff --git a/requirements_emc.txt b/requirements_emc.txt index 651f4826..d142fc0d 100644 --- a/requirements_emc.txt +++ b/requirements_emc.txt @@ -10,4 +10,4 @@ pandas>=1.4.0 numpy>=2.0.0 # Additional packages -git+https://github.com/NOAA-EMC/emcpy.git@92aa62f34a1f413d8cb1646bca0e81f267b61365#egg=emcpy +git+https://github.com/NOAA-EMC/emcpy.git@7794574611e760475d61eb5d9458af2d3d2191d8#egg=emcpy diff --git a/src/eva/plotting/batch/base/diagnostics/contour_plot.py b/src/eva/plotting/batch/base/diagnostics/contour_plot.py index ef9d5ce0..2dd3d20d 100644 --- a/src/eva/plotting/batch/base/diagnostics/contour_plot.py +++ b/src/eva/plotting/batch/base/diagnostics/contour_plot.py @@ -87,9 +87,9 @@ def data_prep(self): zdata = slice_var_from_str(self.config['z'], zdata, self.logger) # contour data should be flattened - xdata = xdata.flatten() - ydata = ydata.flatten() - zdata = zdata.flatten() + self.xdata = xdata.flatten() + self.ydata = ydata.flatten() + self.zdata = zdata.flatten() @abstractmethod def configure_plot(self): diff --git a/src/eva/plotting/batch/base/diagnostics/density.py b/src/eva/plotting/batch/base/diagnostics/density.py index 0c90ef65..01c49218 100644 --- a/src/eva/plotting/batch/base/diagnostics/density.py +++ b/src/eva/plotting/batch/base/diagnostics/density.py @@ -2,6 +2,7 @@ from eva.utilities.config import get from eva.utilities.utils import get_schema, update_object, slice_var_from_str import numpy as np +import numpy.ma as ma from abc import ABC, abstractmethod @@ -73,11 +74,22 @@ def data_prep(self): data = slice_var_from_str(self.config['data'], data, self.logger) # Density data should be flattened - data = data.flatten() - - # Missing data should also be removed - mask = ~np.isnan(data) - self.data = data[mask] + data = np.ravel(np.asanyarray(data)) + + # If upstream gave us a masked array, turn masked to NaN for uniform handling + if ma.isMaskedArray(data): + data = data.filled(np.nan) + + # Optional knob: by default density plots *drop* NaNs (keeps current behavior) + # Set `drop_nan: false` in the layer config if you want to preserve length (masked in place) + drop_nan = bool(self.config.get('drop_nan', True)) + + if drop_nan: + # keep only finite values + self.data = data[np.isfinite(data)] + else: + # preserve length, mask non-finite in place; downstream can decide whether to compress + self.data = ma.masked_invalid(data) # -------------------------------------------------------------------------------------------------- diff --git a/src/eva/plotting/batch/base/diagnostics/filled_contour_plot.py b/src/eva/plotting/batch/base/diagnostics/filled_contour_plot.py index bbb6a46e..78b96a11 100644 --- a/src/eva/plotting/batch/base/diagnostics/filled_contour_plot.py +++ b/src/eva/plotting/batch/base/diagnostics/filled_contour_plot.py @@ -88,9 +88,9 @@ def data_prep(self): zdata = slice_var_from_str(self.config['z'], zdata, self.logger) # contour data should be flattened - xdata = xdata.flatten() - ydata = ydata.flatten() - zdata = zdata.flatten() + self.xdata = xdata.flatten() + self.ydata = ydata.flatten() + self.zdata = zdata.flatten() @abstractmethod def configure_plot(self): diff --git a/src/eva/plotting/batch/base/diagnostics/histogram.py b/src/eva/plotting/batch/base/diagnostics/histogram.py index 78dc9548..b0d2759d 100644 --- a/src/eva/plotting/batch/base/diagnostics/histogram.py +++ b/src/eva/plotting/batch/base/diagnostics/histogram.py @@ -2,6 +2,7 @@ from eva.utilities.config import get from eva.utilities.utils import get_schema, update_object, slice_var_from_str import numpy as np +import numpy.ma as ma from abc import ABC, abstractmethod @@ -72,12 +73,25 @@ def data_prep(self): # See if we need to slice data data = slice_var_from_str(self.config['data'], data, self.logger) - # Histogram data should be flattened - data = data.flatten() - - # Missing data should also be removed - mask = ~np.isnan(data) - self.data = data[mask] + # Flatten + arr = np.ravel(np.asanyarray(data)) + + # If masked, convert masked entries to NaN for uniform handling + if ma.isMaskedArray(arr): + arr = arr.filled(np.nan) + + # Read & strip the knob so it never leaks to backends + cfg = dict(self.config) + drop_nan = bool(cfg.get('drop_nan', True)) + cfg.pop('drop_nan', None) + self.config = cfg + + if drop_nan: + # Typical histogram path: use only finite values + self.data = arr[np.isfinite(arr)] + else: + # Preserve length and mask invalids in place (backend must accept masked arrays) + self.data = ma.masked_invalid(arr) # -------------------------------------------------------------------------------------------------- diff --git a/src/eva/plotting/batch/base/diagnostics/line_plot.py b/src/eva/plotting/batch/base/diagnostics/line_plot.py index b38de056..b4beccfc 100644 --- a/src/eva/plotting/batch/base/diagnostics/line_plot.py +++ b/src/eva/plotting/batch/base/diagnostics/line_plot.py @@ -2,7 +2,7 @@ from eva.utilities.config import get from eva.utilities.utils import get_schema, update_object, slice_var_from_str import numpy as np -import pandas as pd +import numpy.ma as ma from abc import ABC, abstractmethod @@ -96,19 +96,28 @@ def data_prep(self): xdata = slice_var_from_str(self.config['x'], xdata, self.logger) ydata = slice_var_from_str(self.config['y'], ydata, self.logger) - # line plot data should be flattened - self.xdata = xdata.flatten() - self.ydata = ydata.flatten() - - # Remove NaN values to enable regression - # -------------------------------------- - mask = pd.notna(xdata) - self.xdata = xdata[mask] - self.ydata = ydata[mask] - - mask = pd.notna(self.ydata) - self.xdata = self.xdata[mask] - self.ydata = self.ydata[mask] + # Flatten, build y with NaNs preserved (as you already added) + x_flat = np.ravel(xdata) + y_flat = ma.array(ydata).filled(np.nan).ravel() + + # Read and remove the config knob so it won't be forwarded to plt.plot + cfg = dict(getattr(self, "config", {}) or {}) + drop_nan = bool(cfg.pop("drop_nan", False)) + self.config = cfg + + if drop_nan: + y_is_finite = np.isfinite(y_flat) + y_plot = y_flat[y_is_finite] + try: + x_plot = x_flat[y_is_finite] + except Exception: + x_plot = np.array(x_flat, dtype=object)[y_is_finite] + else: + y_plot = y_flat + x_plot = x_flat + + self.xdata = x_plot + self.ydata = y_plot @abstractmethod def configure_plot(self): diff --git a/src/eva/plotting/batch/base/diagnostics/scatter.py b/src/eva/plotting/batch/base/diagnostics/scatter.py index 726913a1..3c7e1d24 100644 --- a/src/eva/plotting/batch/base/diagnostics/scatter.py +++ b/src/eva/plotting/batch/base/diagnostics/scatter.py @@ -2,7 +2,7 @@ from eva.utilities.config import get from eva.utilities.utils import get_schema, update_object, slice_var_from_str import numpy as np -import pandas as pd +import numpy.ma as ma from abc import ABC, abstractmethod @@ -72,26 +72,35 @@ def data_prep(self): channel = self.config.get('channel') xdata = self.dataobj.get_variable_data(var0_cgv[0], var0_cgv[1], var0_cgv[2], channel) - xdata1 = self.dataobj.get_variable_data(var0_cgv[0], var0_cgv[1], var0_cgv[2]) ydata = self.dataobj.get_variable_data(var1_cgv[0], var1_cgv[1], var1_cgv[2], channel) - # see if we need to slice data + # Optional slicing xdata = slice_var_from_str(self.config['x'], xdata, self.logger) ydata = slice_var_from_str(self.config['y'], ydata, self.logger) - # scatter data should be flattened - self.xdata = xdata.flatten() - self.ydata = ydata.flatten() - - # Remove NaN values to enable regression - # -------------------------------------- - mask = pd.notna(xdata) - self.xdata = xdata[mask] - self.ydata = ydata[mask] - - mask = pd.notna(self.ydata) - self.xdata = self.xdata[mask] - self.ydata = self.ydata[mask] + # Flatten and normalize (turn masked to NaN for uniform handling) + x = np.ravel(np.asanyarray(xdata)) + y = np.ravel(np.asanyarray(ydata)) + if ma.isMaskedArray(x): + x = x.filled(np.nan) + if ma.isMaskedArray(y): + y = y.filled(np.nan) + + # Read & remove knob so it won't propagate to matplotlib kwargs + cfg = dict(self.config) + drop_nan = bool(cfg.pop('drop_nan', True)) # default True for scatter + self.config = cfg + + if drop_nan: + # Keep only pairs where both x and y are finite + mask = np.isfinite(x) & np.isfinite(y) + self.xdata = x[mask] + self.ydata = y[mask] + else: + # Preserve length; mask invalid pairs in-place (some backends honor masked arrays) + invalid = ~(np.isfinite(x) & np.isfinite(y)) + self.xdata = ma.array(x, mask=invalid) + self.ydata = ma.array(y, mask=invalid) @abstractmethod def configure_plot(self):