apply fill value without using enhance modes

Unidata · Apr 28, 2023 · 3f412b3 · 3f412b3
1 parent 884444a
commit 3f412b3
Show file tree

Hide file tree

Showing 3 changed files with 55 additions and 30 deletions.
diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java
@@ -9,12 +9,14 @@
 import ucar.ma2.*;
 import ucar.nc2.*;
 import ucar.nc2.constants.CDM;
+import ucar.nc2.constants.DataFormatType;
 import ucar.nc2.dataset.NetcdfDataset.Enhance;
 import ucar.nc2.filter.ConvertMissing;
 import ucar.nc2.filter.FilterHelpers;
 import ucar.nc2.filter.ScaleOffset;
 import ucar.nc2.filter.UnsignedConversion;
 import ucar.nc2.internal.dataset.CoordinatesHelper;
+import ucar.nc2.iosp.netcdf3.N3iosp;
 import ucar.nc2.util.CancelTask;
 import javax.annotation.Nullable;
 import java.io.IOException;
@@ -170,6 +172,8 @@ protected VariableDS(VariableDS vds, boolean isCopy) {
     this.unsignedConversion = vds.unsignedConversion;
     this.scaleOffset = vds.scaleOffset;
     this.convertMissing = vds.convertMissing;
+    this.fillValue = vds.getFillValue();
+    this.hasFillValue = vds.hasFillValue();
 
     // Add this so that old VariableDS units agrees with new VariableDS units.
     String units = vds.getUnitsString();
@@ -242,6 +246,26 @@ public void enhance(Set<Enhance> enhancements) {
       this.unsignedConversion = UnsignedConversion.createFromVar(this);
       this.dataType = unsignedConversion.getOutType();
     }
+    // need fill value info before convertMissing
+    Attribute fillValueAtt = findAttribute(CDM.FILL_VALUE);
+    if (fillValueAtt != null && !fillValueAtt.isString()) {
+      fillValue = convertUnsigned(fillValueAtt.getNumericValue()).doubleValue();
+      fillValue = applyScaleOffset(fillValue); // This will fail when _FillValue is CHAR.
+      hasFillValue = true;
+    } else {
+      // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value.
+      String fileTypeId = getFileTypeId();
+      boolean isNetcdfIosp = DataFormatType.NETCDF.getDescription().equals(fileTypeId)
+          || DataFormatType.NETCDF4.getDescription().equals(fileTypeId);
+
+      if (isNetcdfIosp) {
+        DataType unsignedConversionType = getUnsignedConversionType();
+        if (unsignedConversionType.isNumeric()) {
+          fillValue = applyScaleOffset(N3iosp.getFillValueDefault(unsignedConversionType));
+          hasFillValue = true;
+        }
+      }
+    }
     if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR)) {
       this.scaleOffset = ScaleOffset.createFromVariable(this);
       this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType;
@@ -542,8 +566,8 @@ public Array getMissingDataArray(int[] shape) {
     }
 
     Array array = Array.factoryConstant(getDataType(), shape, storage);
-    if (convertMissing != null && convertMissing.hasFillValue()) {
-      array.setObject(0, convertMissing.getFillValue());
+    if (hasFillValue) {
+      array.setObject(0, fillValue);
     }
     return array;
   }
@@ -672,12 +696,12 @@ public boolean isInvalidData(double val) {
 
   @Override
   public boolean hasFillValue() {
-    return convertMissing != null ? convertMissing.hasFillValue() : false;
+    return hasFillValue;
   }
 
   @Override
   public double getFillValue() {
-    return convertMissing != null ? convertMissing.getFillValue() : Double.MAX_VALUE;
+    return fillValue;
   }
 
   @Override
@@ -816,6 +840,9 @@ public Array convert(Array in, boolean convertUnsigned, boolean applyScaleOffset
   protected String orgName; // in case Variable was renamed, and we need to keep track of the original name
   String orgFileTypeId; // the original fileTypeId.
 
+  private boolean hasFillValue = false;
+  private double fillValue = Double.MAX_VALUE;
+
   protected VariableDS(Builder<?> builder, Group parentGroup) {
     super(builder, parentGroup);
 
@@ -848,6 +875,27 @@ protected VariableDS(Builder<?> builder, Group parentGroup) {
       this.scaleOffset = ScaleOffset.createFromVariable(this);
       this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType;
     }
+
+    // need fill value info before convertMissing
+    Attribute fillValueAtt = findAttribute(CDM.FILL_VALUE);
+    if (fillValueAtt != null && !fillValueAtt.isString()) {
+      fillValue = convertUnsigned(fillValueAtt.getNumericValue()).doubleValue();
+      fillValue = applyScaleOffset(fillValue); // This will fail when _FillValue is CHAR.
+      hasFillValue = true;
+    } else {
+      // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value.
+      String fileTypeId = getFileTypeId();
+      boolean isNetcdfIosp = DataFormatType.NETCDF.getDescription().equals(fileTypeId)
+          || DataFormatType.NETCDF4.getDescription().equals(fileTypeId);
+
+      if (isNetcdfIosp) {
+        DataType unsignedConversionType = getUnsignedConversionType();
+        if (unsignedConversionType.isNumeric()) {
+          fillValue = applyScaleOffset(N3iosp.getFillValueDefault(unsignedConversionType));
+          hasFillValue = true;
+        }
+      }
+    }
     if (this.enhanceMode.contains(Enhance.ConvertMissing)) {
       this.convertMissing = ConvertMissing.createFromVariable(this);
     }

diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java
@@ -87,28 +87,9 @@ public static ConvertMissing createFromVariable(VariableDS var) {
       }
     }
 
-    /// _FillValue
-    double fillValue = Double.MAX_VALUE;
-    boolean hasFillValue = false;
-    Attribute fillValueAtt = var.findAttribute(CDM.FILL_VALUE);
-    if (fillValueAtt != null && !fillValueAtt.isString()) {
-      fillValue = var.convertUnsigned(fillValueAtt.getNumericValue()).doubleValue();
-      fillValue = var.applyScaleOffset(fillValue); // This will fail when _FillValue is CHAR.
-      hasFillValue = true;
-    } else {
-      // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value.
-      String fileTypeId = var.getFileTypeId();
-      boolean isNetcdfIosp = DataFormatType.NETCDF.getDescription().equals(fileTypeId)
-          || DataFormatType.NETCDF4.getDescription().equals(fileTypeId);
-
-      if (isNetcdfIosp) {
-        DataType unsignedConversionType = var.getUnsignedConversionType();
-        if (unsignedConversionType.isNumeric()) {
-          fillValue = var.applyScaleOffset(N3iosp.getFillValueDefault(unsignedConversionType));
-          hasFillValue = true;
-        }
-      }
-    }
+    /// fill_value
+    boolean hasFillValue = var.hasFillValue();
+    double fillValue = var.getFillValue();
 
     /// missing_value
     double[] missingValue = null;

diff --git a/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhance.java b/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhance.java
@@ -6,10 +6,7 @@
 
 import static com.google.common.truth.Truth.assertThat;
 import java.io.IOException;
-import java.lang.invoke.MethodHandles;
 import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import ucar.ma2.DataType;
 import ucar.nc2.NetcdfFile;
 import ucar.nc2.Variable;
@@ -18,7 +15,6 @@
 
 /** Test NcmlNew enhancement */
 public class TestEnhance {
-  private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
   private static String dataDir = TestDir.cdmLocalTestDataDir + "ncml/enhance/";
 
   @Test