diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index 685958025a..7385e1e5a9 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -168,6 +168,8 @@ protected VariableDS(VariableDS vds, boolean isCopy) { this.unsignedConversion = vds.unsignedConversion; this.scaleOffset = vds.scaleOffset; this.convertMissing = vds.convertMissing; + this.fillValue = vds.getFillValue(); + this.hasFillValue = vds.hasFillValue(); // Add this so that old VariableDS units agrees with new VariableDS units. String units = vds.getUnitsString(); @@ -556,8 +558,8 @@ public Array getMissingDataArray(int[] shape) { } Array array = Array.factoryConstant(getDataType(), shape, storage); - if (convertMissing != null && convertMissing.hasFillValue()) { - array.setObject(0, convertMissing.getFillValue()); + if (hasFillValue) { + array.setObject(0, fillValue); } return array; } @@ -661,7 +663,7 @@ public boolean isMissing(double val) { if (Double.isNaN(val)) { return true; } - return convertMissing != null && convertMissing.isMissing(val); + return convertMissing != null && convertMissing.isMissing(packVal(val)); } @Override @@ -671,32 +673,46 @@ public boolean hasValidData() { @Override public double getValidMin() { - return convertMissing != null ? convertMissing.getValidMin() : -Double.MAX_VALUE; + if (convertMissing == null) { + return -Double.MAX_VALUE; + } + if (scaleOffset == null) { + return convertMissing.getValidMin(); + } + return scaleOffset.getScaleFactor() < 0 ? unpackVal(convertMissing.getValidMax()) + : unpackVal(convertMissing.getValidMin()); } @Override public double getValidMax() { - return convertMissing != null ? convertMissing.getValidMax() : Double.MAX_VALUE; + if (convertMissing == null) { + return Double.MAX_VALUE; + } + if (scaleOffset == null) { + return convertMissing.getValidMax(); + } + return scaleOffset.getScaleFactor() < 0 ? unpackVal(convertMissing.getValidMin()) + : unpackVal(convertMissing.getValidMax()); } @Override public boolean isInvalidData(double val) { - return convertMissing != null && convertMissing.isInvalidData(val); + return convertMissing != null && convertMissing.isInvalidData(packVal(val)); } @Override public boolean hasFillValue() { - return convertMissing != null && convertMissing.hasFillValue(); + return hasFillValue; } @Override public double getFillValue() { - return convertMissing != null ? convertMissing.getFillValue() : Double.NaN; + return fillValue; } @Override public boolean isFillValue(double val) { - return convertMissing != null && convertMissing.isFillValue(val); + return convertMissing != null && convertMissing.isFillValue(packVal(val)); } @Override @@ -706,12 +722,28 @@ public boolean hasMissingValue() { @Override public double[] getMissingValues() { - return convertMissing != null ? convertMissing.getMissingValues() : new double[] {0}; + if (convertMissing == null) { + return new double[] {0}; + } + double[] missingVals = convertMissing.getMissingValues(); + double[] vals = new double[missingVals.length]; + for (int i = 0; i < missingVals.length; i++) { + vals[i] = unpackVal(missingVals[i]); + } + return vals; } @Override public boolean isMissingValue(double val) { - return convertMissing != null && convertMissing.isMissingValue(val); + return convertMissing != null && convertMissing.isMissingValue(packVal(val)); + } + + private double unpackVal(double val) { + return scaleOffset != null && !Double.isNaN(val) ? scaleOffset.convert(val) : val; + } + + private double packVal(double val) { + return scaleOffset != null && !Double.isNaN(val) ? scaleOffset.applyScaleOffset(val) : val; } /** @deprecated Use NetcdfDataset.builder() */ @@ -831,6 +863,8 @@ public Array convert(Array in, boolean convertUnsigned, boolean applyScaleOffset protected DataType orgDataType; // keep separate for the case where there is no orgVar. protected String orgName; // in case Variable was renamed, and we need to keep track of the original name String orgFileTypeId; // the original fileTypeId. + private boolean hasFillValue = false; + private double fillValue = Double.MAX_VALUE; protected VariableDS(Builder builder, Group parentGroup) { super(builder, parentGroup); @@ -857,6 +891,13 @@ protected VariableDS(Builder builder, Group parentGroup) { this.scaleOffset = builder.scaleOffset; createEnhancements(); + if (convertMissing != null) { + fillValue = unpackVal(convertMissing.getFillValue()); + hasFillValue = true; + } else { + fillValue = ConvertMissing.getFillValueOrDefault(this); + hasFillValue = !Double.isNaN(fillValue); + } // We have to complete this after the NetcdfDataset is built. this.coordSysNames = builder.coordSysNames; diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java index 86db6e9e25..d985e72984 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java @@ -62,26 +62,8 @@ public static ConvertMissing createFromVariable(VariableDS var) { } /// fill_value - boolean hasFillValue = var.hasFillValue(); - double fillValue = var.getFillValue(); - // need fill value info before convertMissing - Attribute fillValueAtt = var.findAttribute(CDM.FILL_VALUE); - if (fillValueAtt != null && !fillValueAtt.isString()) { - DataType fillType = FilterHelpers.getAttributeDataType(fillValueAtt, var.getSignedness()); - fillValue = var.convertUnsigned(fillValueAtt.getNumericValue(), fillType).doubleValue(); - hasFillValue = true; - } else { - // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value. - String ncfileId = var.getNetcdfFile() == null ? null : var.getNetcdfFile().getFileTypeId(); - if (DataFormatType.NETCDF.getDescription().equals(ncfileId) - || DataFormatType.NETCDF4.getDescription().equals(ncfileId)) { - DataType fillType = var.getDataType(); - if (fillType.isNumeric()) { - fillValue = var.convertUnsigned(N3iosp.getFillValueDefault(fillType), fillType).doubleValue(); - hasFillValue = true; - } - } - } + double fillValue = getFillValueOrDefault(var); + boolean hasFillValue = !Double.isNaN(fillValue); /// missing_value double[] missingValue = null; @@ -267,4 +249,23 @@ public Array convertMissing(Array in) { return out; } + + public static double getFillValueOrDefault(VariableDS var) { + // need fill value info before convertMissing + Attribute fillValueAtt = var.findAttribute(CDM.FILL_VALUE); + if (fillValueAtt != null && !fillValueAtt.isString()) { + DataType fillType = FilterHelpers.getAttributeDataType(fillValueAtt, var.getSignedness()); + return var.convertUnsigned(fillValueAtt.getNumericValue(), fillType).doubleValue(); + } + // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value. + String ncfileId = var.getNetcdfFile() == null ? null : var.getNetcdfFile().getFileTypeId(); + if (DataFormatType.NETCDF.getDescription().equals(ncfileId) + || DataFormatType.NETCDF4.getDescription().equals(ncfileId)) { + DataType fillType = var.getDataType(); + if (fillType.isNumeric()) { + return var.convertUnsigned(N3iosp.getFillValueDefault(fillType), fillType).doubleValue(); + } + } + return Double.NaN; + } }