Fix issue with dropping nans

requirements_emc.txt

@@ -10,4 +10,4 @@ pandas>=1.4.0
 numpy>=2.0.0
 
 # Additional packages
-git+https://github.com/NOAA-EMC/emcpy.git@92aa62f34a1f413d8cb1646bca0e81f267b61365#egg=emcpy
+git+https://github.com/NOAA-EMC/emcpy.git@7794574611e760475d61eb5d9458af2d3d2191d8#egg=emcpy

src/eva/plotting/batch/base/diagnostics/contour_plot.py

@@ -87,9 +87,9 @@ def data_prep(self):
         zdata = slice_var_from_str(self.config['z'], zdata, self.logger)
 
         # contour data should be flattened
-        xdata = xdata.flatten()
-        ydata = ydata.flatten()
-        zdata = zdata.flatten()
+        self.xdata = xdata.flatten()
+        self.ydata = ydata.flatten()
+        self.zdata = zdata.flatten()
 
     @abstractmethod
     def configure_plot(self):

src/eva/plotting/batch/base/diagnostics/density.py

@@ -2,6 +2,7 @@
 from eva.utilities.config import get
 from eva.utilities.utils import get_schema, update_object, slice_var_from_str
 import numpy as np
+import numpy.ma as ma
 
 from abc import ABC, abstractmethod
 
@@ -73,11 +74,22 @@ def data_prep(self):
         data = slice_var_from_str(self.config['data'], data, self.logger)
 
         # Density data should be flattened
-        data = data.flatten()
-
-        # Missing data should also be removed
-        mask = ~np.isnan(data)
-        self.data = data[mask]
+        data = np.ravel(np.asanyarray(data))
+
+        # If upstream gave us a masked array, turn masked to NaN for uniform handling
+        if ma.isMaskedArray(data):
+            data = data.filled(np.nan)
+
+        # Optional knob: by default density plots *drop* NaNs (keeps current behavior)
+        # Set `drop_nan: false` in the layer config if you want to preserve length (masked in place)
+        drop_nan = bool(self.config.get('drop_nan', True))
+
+        if drop_nan:
+            # keep only finite values
+            self.data = data[np.isfinite(data)]
+        else:
+            # preserve length, mask non-finite in place; downstream can decide whether to compress
+            self.data = ma.masked_invalid(data)
 
 # --------------------------------------------------------------------------------------------------
 

src/eva/plotting/batch/base/diagnostics/filled_contour_plot.py

@@ -88,9 +88,9 @@ def data_prep(self):
         zdata = slice_var_from_str(self.config['z'], zdata, self.logger)
 
         # contour data should be flattened
-        xdata = xdata.flatten()
-        ydata = ydata.flatten()
-        zdata = zdata.flatten()
+        self.xdata = xdata.flatten()
+        self.ydata = ydata.flatten()
+        self.zdata = zdata.flatten()
 
     @abstractmethod
     def configure_plot(self):

src/eva/plotting/batch/base/diagnostics/histogram.py

@@ -2,6 +2,7 @@
 from eva.utilities.config import get
 from eva.utilities.utils import get_schema, update_object, slice_var_from_str
 import numpy as np
+import numpy.ma as ma
 
 from abc import ABC, abstractmethod
 
@@ -72,12 +73,25 @@ def data_prep(self):
         # See if we need to slice data
         data = slice_var_from_str(self.config['data'], data, self.logger)
 
-        # Histogram data should be flattened
-        data = data.flatten()
-
-        # Missing data should also be removed
-        mask = ~np.isnan(data)
-        self.data = data[mask]
+        # Flatten
+        arr = np.ravel(np.asanyarray(data))
+
+        # If masked, convert masked entries to NaN for uniform handling
+        if ma.isMaskedArray(arr):
+            arr = arr.filled(np.nan)
+
+        # Read & strip the knob so it never leaks to backends
+        cfg = dict(self.config)
+        drop_nan = bool(cfg.get('drop_nan', True))
+        cfg.pop('drop_nan', None)
+        self.config = cfg
+
+        if drop_nan:
+            # Typical histogram path: use only finite values
+            self.data = arr[np.isfinite(arr)]
+        else:
+            # Preserve length and mask invalids in place (backend must accept masked arrays)
+            self.data = ma.masked_invalid(arr)
 
 # --------------------------------------------------------------------------------------------------
 

src/eva/plotting/batch/base/diagnostics/line_plot.py

@@ -2,7 +2,7 @@
 from eva.utilities.config import get
 from eva.utilities.utils import get_schema, update_object, slice_var_from_str
 import numpy as np
-import pandas as pd
+import numpy.ma as ma
 
 from abc import ABC, abstractmethod
 
@@ -96,19 +96,28 @@ def data_prep(self):
         xdata = slice_var_from_str(self.config['x'], xdata, self.logger)
         ydata = slice_var_from_str(self.config['y'], ydata, self.logger)
 
-        # line plot data should be flattened
-        self.xdata = xdata.flatten()
-        self.ydata = ydata.flatten()
-
-        # Remove NaN values to enable regression
-        # --------------------------------------
-        mask = pd.notna(xdata)
-        self.xdata = xdata[mask]
-        self.ydata = ydata[mask]
-
-        mask = pd.notna(self.ydata)
-        self.xdata = self.xdata[mask]
-        self.ydata = self.ydata[mask]
+        # Flatten, build y with NaNs preserved (as you already added)
+        x_flat = np.ravel(xdata)
+        y_flat = ma.array(ydata).filled(np.nan).ravel()
+
+        # Read and remove the config knob so it won't be forwarded to plt.plot
+        cfg = dict(getattr(self, "config", {}) or {})
+        drop_nan = bool(cfg.pop("drop_nan", False))
+        self.config = cfg
+
+        if drop_nan:
+            y_is_finite = np.isfinite(y_flat)
+            y_plot = y_flat[y_is_finite]
+            try:
+                x_plot = x_flat[y_is_finite]
+            except Exception:
+                x_plot = np.array(x_flat, dtype=object)[y_is_finite]
+        else:
+            y_plot = y_flat
+            x_plot = x_flat
+
+        self.xdata = x_plot
+        self.ydata = y_plot
 
     @abstractmethod
     def configure_plot(self):

src/eva/plotting/batch/base/diagnostics/scatter.py

@@ -2,7 +2,7 @@
 from eva.utilities.config import get
 from eva.utilities.utils import get_schema, update_object, slice_var_from_str
 import numpy as np
-import pandas as pd
+import numpy.ma as ma
 
 from abc import ABC, abstractmethod
 
@@ -72,26 +72,35 @@ def data_prep(self):
             channel = self.config.get('channel')
 
         xdata = self.dataobj.get_variable_data(var0_cgv[0], var0_cgv[1], var0_cgv[2], channel)
-        xdata1 = self.dataobj.get_variable_data(var0_cgv[0], var0_cgv[1], var0_cgv[2])
         ydata = self.dataobj.get_variable_data(var1_cgv[0], var1_cgv[1], var1_cgv[2], channel)
 
-        # see if we need to slice data
+        # Optional slicing
         xdata = slice_var_from_str(self.config['x'], xdata, self.logger)
         ydata = slice_var_from_str(self.config['y'], ydata, self.logger)
 
-        # scatter data should be flattened
-        self.xdata = xdata.flatten()
-        self.ydata = ydata.flatten()
-
-        # Remove NaN values to enable regression
-        # --------------------------------------
-        mask = pd.notna(xdata)
-        self.xdata = xdata[mask]
-        self.ydata = ydata[mask]
-
-        mask = pd.notna(self.ydata)
-        self.xdata = self.xdata[mask]
-        self.ydata = self.ydata[mask]
+        # Flatten and normalize (turn masked to NaN for uniform handling)
+        x = np.ravel(np.asanyarray(xdata))
+        y = np.ravel(np.asanyarray(ydata))
+        if ma.isMaskedArray(x):
+            x = x.filled(np.nan)
+        if ma.isMaskedArray(y):
+            y = y.filled(np.nan)
+
+        # Read & remove knob so it won't propagate to matplotlib kwargs
+        cfg = dict(self.config)
+        drop_nan = bool(cfg.pop('drop_nan', True))  # default True for scatter
+        self.config = cfg
+
+        if drop_nan:
+            # Keep only pairs where both x and y are finite
+            mask = np.isfinite(x) & np.isfinite(y)
+            self.xdata = x[mask]
+            self.ydata = y[mask]
+        else:
+            # Preserve length; mask invalid pairs in-place (some backends honor masked arrays)
+            invalid = ~(np.isfinite(x) & np.isfinite(y))
+            self.xdata = ma.array(x, mask=invalid)
+            self.ydata = ma.array(y, mask=invalid)
 
     @abstractmethod
     def configure_plot(self):