From 212ee250ab11dbde44b2ca80b587bb144b949f75 Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Mon, 3 Oct 2022 14:33:16 -0700 Subject: [PATCH 01/24] add default values for scale and offset params --- .../src/main/java/ucar/nc2/filter/ScaleOffset.java | 4 ++-- .../src/test/java/ucar/nc2/filter/TestFilters.java | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java index 4f7aad01d9..369ad2f447 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java @@ -56,8 +56,8 @@ public class ScaleOffset extends Filter { public ScaleOffset(Map properties) { // get offset and scale parameters - offset = ((Number) properties.get("offset")).doubleValue(); - scale = (int) properties.get("scale"); + offset = ((Number) properties.getOrDefault("offset", 0)).doubleValue(); + scale = (int) properties.getOrDefault("scale", 1); // input data type String type = (String) properties.get("dtype"); diff --git a/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java b/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java index d7e6943178..51c67f55bc 100644 --- a/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java +++ b/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java @@ -96,6 +96,16 @@ public void testScaleOffset() throws IOException { // test decode byte[] decoded = filter.decode(encoded); assertThat(decoded).isEqualTo(input); + + // test empty props + Map props2 = new HashMap<>(); + props2.put("id", "fixedscaleoffset"); + props2.put("dtype", " Date: Fri, 23 Dec 2022 15:55:41 -0800 Subject: [PATCH 02/24] refactor scale, missing, and unsigned implementations --- .../src/main/java/ucar/nc2/NetcdfFiles.java | 4 +- .../dataset/EnhanceScaleMissingUnsigned.java | 4 +- .../java/ucar/nc2/dataset/NetcdfDataset.java | 2 +- .../java/ucar/nc2/dataset/VariableDS.java | 151 +++++----- .../java/ucar/nc2/filter/ConvertMissing.java | 264 ++++++++++++++++++ .../src/main/java/ucar/nc2/filter/Filter.java | 3 + .../java/ucar/nc2/filter/FilterHelpers.java | 225 +++++++++++++++ .../java/ucar/nc2/filter/ScaleOffset.java | 224 +++++++-------- .../ucar/nc2/filter/UnsignedConversion.java | 68 +++++ .../nc2/internal/iosp/hdf5/H5iospNew.java | 2 +- 10 files changed, 752 insertions(+), 195 deletions(-) create mode 100644 cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java create mode 100644 cdm/core/src/main/java/ucar/nc2/filter/FilterHelpers.java create mode 100644 cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java diff --git a/cdm/core/src/main/java/ucar/nc2/NetcdfFiles.java b/cdm/core/src/main/java/ucar/nc2/NetcdfFiles.java index 9aa9c3dba2..e89ef3a94a 100644 --- a/cdm/core/src/main/java/ucar/nc2/NetcdfFiles.java +++ b/cdm/core/src/main/java/ucar/nc2/NetcdfFiles.java @@ -265,7 +265,7 @@ public static NetcdfFile open(String location, int buffer_size, ucar.nc2.util.Ca * @param buffer_size RandomAccessFile buffer size, if <= 0, use default size * @param cancelTask allow task to be cancelled; may be null. * @param iospMessage special iosp tweaking (sent before open is called), may be null - * @return NetcdfFile object, or null if cant find IOServiceProver + * @return NetcdfFile object, or null if can't find IOServiceProver * @throws IOException if error */ public static NetcdfFile open(String location, int buffer_size, ucar.nc2.util.CancelTask cancelTask, @@ -290,7 +290,7 @@ public static NetcdfFile open(String location, int buffer_size, ucar.nc2.util.Ca * @param iospMessage special iosp tweaking (sent before open is called), may be null * @return NetcdfFile object, or null if cant find IOServiceProver * @throws IOException if read error - * @throws ClassNotFoundException cannat find iospClassName in thye class path + * @throws ClassNotFoundException cannot find iospClassName in the class path * @throws InstantiationException if class cannot be instantiated * @throws IllegalAccessException if class is not accessible */ diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsigned.java b/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsigned.java index 16fc11a629..2a255435d2 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsigned.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsigned.java @@ -31,7 +31,7 @@ *
  • Values will be {@link DataType#widenNumber widened}, which effectively reinterprets signed data as unsigned * data.
    • *
  • To accommodate the unsigned conversion, the variable's data type will be changed to the - * {@link EnhanceScaleMissingUnsignedImpl#nextLarger(DataType) next larger type}.
    • + * {@link ucar.nc2.filter.FilterHelpers#nextLarger(DataType) next larger type}. * * *

    Implementation rules for scale/offset

    @@ -117,7 +117,9 @@ * * @author caron * @author cwardgar + * @deprecated use implementations in Filter package */ +@Deprecated public interface EnhanceScaleMissingUnsigned extends IsMissingEvaluator { /** true if Variable data will be converted using scale and offset */ boolean hasScaleOffset(); diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/NetcdfDataset.java b/cdm/core/src/main/java/ucar/nc2/dataset/NetcdfDataset.java index c87fd7036d..4b78cab001 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/NetcdfDataset.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/NetcdfDataset.java @@ -98,7 +98,7 @@ public enum Enhance { * Convert unsigned values to signed values. * For {@link ucar.nc2.constants.CDM#UNSIGNED} variables, reinterpret the bit patterns of any * negative values as unsigned. The result will be positive values that must be stored in a - * {@link EnhanceScaleMissingUnsignedImpl#nextLarger larger data type}. + * {@link ucar.nc2.filter.FilterHelpers#nextLarger larger data type}. */ ConvertUnsigned, /** Apply scale and offset to values, promoting the data type if needed. */ diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index cdfa726f9f..a30e5a3fc2 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -10,6 +10,10 @@ import ucar.nc2.*; import ucar.nc2.constants.CDM; import ucar.nc2.dataset.NetcdfDataset.Enhance; +import ucar.nc2.filter.ConvertMissing; +import ucar.nc2.filter.FilterHelpers; +import ucar.nc2.filter.ScaleOffset; +import ucar.nc2.filter.UnsignedConversion; import ucar.nc2.internal.dataset.CoordinatesHelper; import ucar.nc2.util.CancelTask; import javax.annotation.Nullable; @@ -163,7 +167,9 @@ protected VariableDS(VariableDS vds, boolean isCopy) { this.orgName = vds.orgName; this.enhanceProxy = new EnhancementsImpl(this); // decouple coordinate systems - this.scaleMissingUnsignedProxy = vds.scaleMissingUnsignedProxy; + this.unsignedConversion = vds.unsignedConversion; + this.scaleOffset = vds.scaleOffset; + this.convertMissing = vds.convertMissing; // Add this so that old VariableDS units agrees with new VariableDS units. String units = vds.getUnitsString(); @@ -225,16 +231,16 @@ public void enhance(Set enhancements) { setDataType(orgDataType); } - // Initialize EnhanceScaleMissingUnsignedImpl. We can't do this in the constructors because this object may not - // contain all of the relevant attributes at that time. NcMLReader is an example of this: the VariableDS is - // constructed first, and then Attributes are added to it later. - this.scaleMissingUnsignedProxy = new EnhanceScaleMissingUnsignedImpl(this, this.enhanceMode); - if (this.enhanceMode.contains(Enhance.ConvertEnums) && dataType.isEnum()) { setDataType(DataType.STRING); // LOOK promote data type to STRING ???? return; // We can return here, because the other conversions don't apply to STRING. } + // Initialize EnhanceScaleMissingUnsignedImpl. We can't do this in the constructors because this object may not + // contain all of the relevant attributes at that time. NcMLReader is an example of this: the VariableDS is + // constructed first, and then Attributes are added to it later. + this.scaleMissingUnsignedProxy = new EnhanceScaleMissingUnsignedImpl(this, this.enhanceMode); + if (this.enhanceMode.contains(Enhance.ConvertUnsigned)) { // We may need a larger data type to hold the results of the unsigned conversion. setDataType(scaleMissingUnsignedProxy.getUnsignedConversionType()); @@ -529,8 +535,8 @@ public Array getMissingDataArray(int[] shape) { } Array array = Array.factoryConstant(getDataType(), shape, storage); - if (scaleMissingUnsignedProxy.hasFillValue()) { - array.setObject(0, scaleMissingUnsignedProxy.getFillValue()); + if (convertMissing.hasFillValue()) { + array.setObject(0, convertMissing.getFillValue()); } return array; } @@ -541,26 +547,25 @@ public Array getMissingDataArray(int[] shape) { * @param f put info here */ public void showScaleMissingProxy(Formatter f) { - f.format("has missing = %s%n", scaleMissingUnsignedProxy.hasMissing()); - if (scaleMissingUnsignedProxy.hasMissing()) { - if (scaleMissingUnsignedProxy.hasMissingValue()) { + f.format("has missing = %s%n", convertMissing.hasMissing()); + if (convertMissing.hasMissing()) { + if (convertMissing.hasMissingValue()) { f.format(" missing value(s) = "); - for (double d : scaleMissingUnsignedProxy.getMissingValues()) + for (double d : convertMissing.getMissingValues()) f.format(" %f", d); f.format("%n"); } - if (scaleMissingUnsignedProxy.hasFillValue()) - f.format(" fillValue = %f%n", scaleMissingUnsignedProxy.getFillValue()); - if (scaleMissingUnsignedProxy.hasValidData()) - f.format(" valid min/max = [%f,%f]%n", scaleMissingUnsignedProxy.getValidMin(), - scaleMissingUnsignedProxy.getValidMax()); - } - f.format("FillValue or default = %s%n", scaleMissingUnsignedProxy.getFillValue()); - - f.format("%nhas scale/offset = %s%n", scaleMissingUnsignedProxy.hasScaleOffset()); - if (scaleMissingUnsignedProxy.hasScaleOffset()) { - double offset = scaleMissingUnsignedProxy.applyScaleOffset(0.0); - double scale = scaleMissingUnsignedProxy.applyScaleOffset(1.0) - offset; + if (convertMissing.hasFillValue()) + f.format(" fillValue = %f%n", convertMissing.getFillValue()); + if (convertMissing.hasValidData()) + f.format(" valid min/max = [%f,%f]%n", convertMissing.getValidMin(), convertMissing.getValidMax()); + } + f.format("FillValue or default = %s%n", convertMissing.getFillValue()); + + f.format("%nhas scale/offset = %s%n", scaleOffset != null); + if (scaleOffset != null) { + double offset = scaleOffset.applyScaleOffset(0.0); + double scale = scaleOffset.applyScaleOffset(1.0) - offset; f.format(" scale_factor = %f add_offset = %f%n", scale, offset); } f.format("original data type = %s%n", orgDataType); @@ -609,77 +614,78 @@ public void removeCoordinateSystem(CoordinateSystem cs) { @Override public boolean hasScaleOffset() { - return scaleMissingUnsignedProxy.hasScaleOffset(); + return scaleOffset != null; } @Override public double getScaleFactor() { - return scaleMissingUnsignedProxy.getScaleFactor(); + return scaleOffset != null ? scaleOffset.getScaleFactor() : 1; } @Override public double getOffset() { - return scaleMissingUnsignedProxy.getOffset(); + return scaleOffset != null ? scaleOffset.getOffset() : 0; } @Override public boolean hasMissing() { - return scaleMissingUnsignedProxy.hasMissing(); + // TODO: null or all false values? + return convertMissing.hasMissing(); } @Override public boolean isMissing(double val) { - return scaleMissingUnsignedProxy.isMissing(val); + return convertMissing.isMissing(val); } @Override public boolean hasValidData() { - return scaleMissingUnsignedProxy.hasValidData(); + return convertMissing.hasMissingValue(); } @Override public double getValidMin() { - return scaleMissingUnsignedProxy.getValidMin(); + return convertMissing.getValidMin(); } @Override public double getValidMax() { - return scaleMissingUnsignedProxy.getValidMax(); + return convertMissing.getValidMax(); } @Override public boolean isInvalidData(double val) { - return scaleMissingUnsignedProxy.isInvalidData(val); + return convertMissing.isInvalidData(val); } @Override public boolean hasFillValue() { - return scaleMissingUnsignedProxy.hasFillValue(); + return convertMissing.hasFillValue(); } @Override public double getFillValue() { - return scaleMissingUnsignedProxy.getFillValue(); + return convertMissing.getFillValue(); } @Override public boolean isFillValue(double val) { - return scaleMissingUnsignedProxy.isFillValue(val); + return convertMissing.isFillValue(val); } @Override public boolean hasMissingValue() { - return scaleMissingUnsignedProxy.hasMissingValue(); + return convertMissing.hasMissingValue(); } @Override public double[] getMissingValues() { - return scaleMissingUnsignedProxy.getMissingValues(); + return convertMissing.getMissingValues(); } @Override public boolean isMissingValue(double val) { - return scaleMissingUnsignedProxy.isMissingValue(val); + return convertMissing.isMissingValue(val); } /** @deprecated Use NetcdfDataset.builder() */ @@ -696,6 +702,18 @@ public void setInvalidDataIsMissing(boolean b) { scaleMissingUnsignedProxy.setInvalidDataIsMissing(b); } + public boolean missingDataIsMissing() { + return builder().missingDataIsMissing; + } + + public boolean fillValueIsMissing() { + return builder().fillValueIsMissing; + } + + public boolean invalidDataIsMissing() { + return builder().invalidDataIsMissing; + } + /** @deprecated Use NetcdfDataset.builder() */ @Deprecated @Override @@ -706,50 +724,54 @@ public void setMissingDataIsMissing(boolean b) { @Nullable @Override public DataType getScaledOffsetType() { - return scaleMissingUnsignedProxy.getScaledOffsetType(); + return scaleOffset.getScaledOffsetType(); } @Override public DataType getUnsignedConversionType() { - return scaleMissingUnsignedProxy.getUnsignedConversionType(); + return unsignedConversion != null ? unsignedConversion.getOutType() : dataType; } @Override public DataType.Signedness getSignedness() { - return scaleMissingUnsignedProxy.getSignedness(); + return unsignedConversion != null ? DataType.Signedness.SIGNED : dataType.getSignedness(); } @Override public double applyScaleOffset(Number value) { - return scaleMissingUnsignedProxy.applyScaleOffset(value); + return scaleOffset != null ? scaleOffset.removeScaleOffset(value) : value.doubleValue(); } @Override public Array applyScaleOffset(Array data) { - return scaleMissingUnsignedProxy.applyScaleOffset(data); + return scaleOffset != null ? scaleOffset.removeScaleOffset(data) : data; } @Override public Number convertUnsigned(Number value) { - return scaleMissingUnsignedProxy.convertUnsigned(value); + return unsignedConversion != null ? unsignedConversion.convertUnsigned(value) : value; } @Override public Array convertUnsigned(Array in) { - return scaleMissingUnsignedProxy.convertUnsigned(in); + return unsignedConversion != null ? unsignedConversion.convertUnsigned(in) : in; } @Override public Number convertMissing(Number value) { - return scaleMissingUnsignedProxy.convertMissing(value); + return convertMissing.convertMissing(value); } @Override public Array convertMissing(Array in) { - return scaleMissingUnsignedProxy.convertMissing(in); + return convertMissing.convertMissing(in); } + /** + * @deprecated use implementations in filters package + */ @Override + @Deprecated public Array convert(Array in, boolean convertUnsigned, boolean applyScaleOffset, boolean convertMissing) { return scaleMissingUnsignedProxy.convert(in, convertUnsigned, applyScaleOffset, convertMissing); } @@ -758,9 +780,12 @@ public Array convert(Array in, boolean convertUnsigned, boolean applyScaleOffset // TODO remove in version 6. private EnhancementsImpl enhanceProxy; private List coordSysNames; - - // TODO make these final and immutable in 6. private EnhanceScaleMissingUnsignedImpl scaleMissingUnsignedProxy = new EnhanceScaleMissingUnsignedImpl(); + + // TODO make immutable in version 6 + private UnsignedConversion unsignedConversion; + private ScaleOffset scaleOffset; + private ConvertMissing convertMissing; private Set enhanceMode = EnumSet.noneOf(Enhance.class); // The set of enhancements that were made. protected Variable orgVar; // wrap this Variable : use it for the I/O @@ -787,23 +812,21 @@ protected VariableDS(Builder builder, Group parentGroup) { this.orgFileTypeId = builder.orgFileTypeId; this.enhanceProxy = new EnhancementsImpl(this, builder.units, builder.getDescription()); - this.scaleMissingUnsignedProxy = new EnhanceScaleMissingUnsignedImpl(this, this.enhanceMode); - this.scaleMissingUnsignedProxy.setFillValueIsMissing(builder.fillValueIsMissing); - this.scaleMissingUnsignedProxy.setInvalidDataIsMissing(builder.invalidDataIsMissing); - this.scaleMissingUnsignedProxy.setMissingDataIsMissing(builder.missingDataIsMissing); if (this.enhanceMode.contains(Enhance.ConvertEnums) && dataType.isEnum()) { this.dataType = DataType.STRING; // LOOK promote enum data type to STRING ???? - } - - if (this.enhanceMode.contains(Enhance.ConvertUnsigned) && !dataType.isEnum()) { - // We may need a larger data type to hold the results of the unsigned conversion. - this.dataType = scaleMissingUnsignedProxy.getUnsignedConversionType(); - } - - if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR) - && scaleMissingUnsignedProxy.hasScaleOffset()) { - this.dataType = scaleMissingUnsignedProxy.getScaledOffsetType(); + } else { + if (this.enhanceMode.contains(Enhance.ConvertUnsigned) && !dataType.isEnum()) { + this.unsignedConversion = UnsignedConversion.createFromVar(this); + this.dataType = unsignedConversion.getOutType(); + } + if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR)) { + this.scaleOffset = ScaleOffset.createFromVariable(this); + this.dataType = scaleOffset.getScaledOffsetType(); + } + if (this.enhanceMode.contains(Enhance.ConvertMissing)) { + this.convertMissing = ConvertMissing.createFromVariable(this); + } } // We have to complete this after the NetcdfDataset is built. diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java new file mode 100644 index 0000000000..1a52e75564 --- /dev/null +++ b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java @@ -0,0 +1,264 @@ +package ucar.nc2.filter; + +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.IndexIterator; +import ucar.nc2.Attribute; +import ucar.nc2.constants.CDM; +import ucar.nc2.constants.DataFormatType; +import ucar.nc2.dataset.VariableDS; +import ucar.nc2.iosp.netcdf3.N3iosp; +import ucar.nc2.util.Misc; + +import java.util.Arrays; + +public class ConvertMissing { + + private boolean hasValidMin, hasValidMax; + private double validMin, validMax; + + private boolean hasFillValue; + private double fillValue; // LOOK: making it double not really correct. What about CHAR? + + private boolean hasMissingValue; + private double[] missingValue; // LOOK: also wrong to make double, for the same reason. + + // defaults from NetcdfDataset modes + private boolean invalidDataIsMissing; + private boolean fillValueIsMissing; + private boolean missingDataIsMissing; + + public static ConvertMissing createFromVariable(VariableDS var) { + // valid min and max + DataType.Signedness signedness = var.getSignedness(); + double validMin = -Double.MAX_VALUE, validMax = Double.MAX_VALUE; + boolean hasValidMin = false, hasValidMax = false; + // assume here its in units of unpacked data. correct this below + Attribute validRangeAtt = var.findAttribute(CDM.VALID_RANGE); + DataType validType = null; + if (validRangeAtt != null && !validRangeAtt.isString() && validRangeAtt.getLength() > 1) { + validType = FilterHelpers.getAttributeDataType(validRangeAtt, signedness); + validMin = var.convertUnsigned(validRangeAtt.getNumericValue(0)).doubleValue(); + validMax = var.convertUnsigned(validRangeAtt.getNumericValue(1)).doubleValue(); + hasValidMin = true; + hasValidMax = true; + } + + Attribute validMinAtt = var.findAttribute(CDM.VALID_MIN); + Attribute validMaxAtt = var.findAttribute(CDM.VALID_MAX); + + // Only process the valid_min and valid_max attributes if valid_range isn't present. + if (!hasValidMin) { + if (validMinAtt != null && !validMinAtt.isString()) { + validType = FilterHelpers.getAttributeDataType(validMinAtt, signedness); + validMin = var.convertUnsigned(validMinAtt.getNumericValue()).doubleValue(); + hasValidMin = true; + } + + if (validMaxAtt != null && !validMaxAtt.isString()) { + validType = FilterHelpers.largestOf(validType, FilterHelpers.getAttributeDataType(validMaxAtt, signedness)); + validMax = var.convertUnsigned(validMaxAtt.getNumericValue()).doubleValue(); + hasValidMax = true; + } + } + + // check if validData values are stored packed or unpacked + if (hasValidMin || hasValidMax) { + if (FilterHelpers.rank(validType) == FilterHelpers.rank(var.getScaledOffsetType()) + && FilterHelpers.rank(validType) > FilterHelpers.rank(var.getUnsignedConversionType())) { + // If valid_range is the same type as the wider of scale_factor and add_offset, PLUS + // it is wider than the (packed) data, we know that the valid_range values were stored as unpacked. + // We already assumed that this was the case when we first read the attribute values, so there's + // nothing for us to do here. + } else { + // Otherwise, the valid_range values were stored as packed. So now we must unpack them. + if (hasValidMin) { + validMin = var.applyScaleOffset(validMin); + } + if (hasValidMax) { + validMax = var.applyScaleOffset(validMax); + } + } + // During the scaling process, it is possible that the valid minimum and maximum values have effectively been + // swapped (for example, when the scale value is negative). Go ahead and check to make sure the valid min is + // actually less than the valid max, and if not, fix it. See https://github.com/Unidata/netcdf-java/issues/572. + if (validMin > validMax) { + double tmp = validMin; + validMin = validMax; + validMax = tmp; + } + } + + /// _FillValue + double fillValue = Double.MAX_VALUE; + boolean hasFillValue = false; + Attribute fillValueAtt = var.findAttribute(CDM.FILL_VALUE); + if (fillValueAtt != null && !fillValueAtt.isString()) { + fillValue = var.convertUnsigned(fillValueAtt.getNumericValue()).doubleValue(); + fillValue = var.applyScaleOffset(fillValue); // This will fail when _FillValue is CHAR. + hasFillValue = true; + } else { + // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value. + String fileTypeId = var.getFileTypeId(); + boolean isNetcdfIosp = DataFormatType.NETCDF.getDescription().equals(fileTypeId) + || DataFormatType.NETCDF4.getDescription().equals(fileTypeId); + + if (isNetcdfIosp) { + DataType unsignedConversionType = var.getUnsignedConversionType(); + if (unsignedConversionType.isNumeric()) { + fillValue = var.applyScaleOffset(N3iosp.getFillValueDefault(unsignedConversionType)); + hasFillValue = true; + } + } + } + + /// missing_value + double[] missingValue = null; + boolean hasMissingValue = false; + Attribute missingValueAtt = var.findAttribute(CDM.MISSING_VALUE); + if (missingValueAtt != null) { + if (missingValueAtt.isString()) { + String svalue = missingValueAtt.getStringValue(); + if (var.getOriginalDataType() == DataType.CHAR) { + missingValue = new double[1]; + if (svalue.isEmpty()) { + missingValue[0] = 0; + } else { + missingValue[0] = svalue.charAt(0); + } + + hasMissingValue = true; + } else { // not a CHAR - try to fix problem where they use a numeric value as a String attribute + try { + missingValue = new double[1]; + missingValue[0] = Double.parseDouble(svalue); + hasMissingValue = true; + } catch (NumberFormatException ex) { + // TODO add logger + } + } + } else { // not a string + missingValue = new double[missingValueAtt.getLength()]; + for (int i = 0; i < missingValue.length; i++) { + missingValue[i] = var.convertUnsigned(missingValueAtt.getNumericValue(i)).doubleValue(); + missingValue[i] = var.applyScaleOffset(missingValue[i]); + } + + for (double mv : missingValue) { + if (!Double.isNaN(mv)) { + hasMissingValue = true; // dont need to do anything if it's already a NaN + break; + } + } + } + } + return new ConvertMissing(var.fillValueIsMissing(), var.invalidDataIsMissing(), var.missingDataIsMissing(), + hasValidMin, hasValidMax, validMin, validMax, hasFillValue, fillValue, hasMissingValue, missingValue); + } + + + public ConvertMissing(boolean fillValueIsMissing, boolean invalidDataIsMissing, boolean missingDataIsMissing, + boolean hasValidMin, boolean hasValidMax, double validMin, double validMax, boolean hasFillValue, + double fillValue, boolean hasMissingValue, double[] missingValue) { + this.fillValueIsMissing = fillValueIsMissing; + this.invalidDataIsMissing = invalidDataIsMissing; + this.missingDataIsMissing = missingDataIsMissing; + this.hasValidMin = hasValidMin; + this.hasValidMax = hasValidMax; + this.validMin = validMin; + this.validMax = validMax; + this.hasFillValue = hasFillValue; + this.fillValue = fillValue; + this.hasMissingValue = hasMissingValue; + this.missingValue = missingValue; + } + + public boolean hasValidData() { + return hasValidMin || hasValidMax; + } + + public double getValidMin() { + return validMin; + } + + public double getValidMax() { + return validMax; + } + + public boolean isInvalidData(double val) { + // valid_min and valid_max may have been multiplied by scale_factor, which could be a float, not a double. + // That potential loss of precision means that we cannot do the nearlyEquals() comparison with + // Misc.defaultMaxRelativeDiffDouble. + boolean greaterThanOrEqualToValidMin = + Misc.nearlyEquals(val, validMin, Misc.defaultMaxRelativeDiffFloat) || val > validMin; + boolean lessThanOrEqualToValidMax = + Misc.nearlyEquals(val, validMax, Misc.defaultMaxRelativeDiffFloat) || val < validMax; + + return (hasValidMin && !greaterThanOrEqualToValidMin) || (hasValidMax && !lessThanOrEqualToValidMax); + } + + public boolean hasFillValue() { + return hasFillValue; + } + + public boolean isFillValue(double val) { + return hasFillValue && Misc.nearlyEquals(val, fillValue, Misc.defaultMaxRelativeDiffFloat); + } + + public double getFillValue() { + return fillValue; + } + + public boolean isMissingValue(double val) { + if (!hasMissingValue) { + return false; + } + for (double aMissingValue : missingValue) { + if (Misc.nearlyEquals(val, aMissingValue, Misc.defaultMaxRelativeDiffFloat)) { + return true; + } + } + return false; + } + + public double[] getMissingValues() { + return missingValue; + } + + public boolean hasMissingValue() { + return hasMissingValue; + } + + public boolean hasMissing() { + return (invalidDataIsMissing && hasValidData()) || (fillValueIsMissing && hasFillValue()) + || (missingDataIsMissing && hasMissingValue()); + } + + public boolean isMissing(double val) { + if (Double.isNaN(val)) { + return true; + } else { + return (missingDataIsMissing && isMissingValue(val)) || (fillValueIsMissing && isFillValue(val)) + || (invalidDataIsMissing && isInvalidData(val)); + } + } + + public Number convertMissing(Number value) { + return isMissing(value.doubleValue()) ? Double.NaN : value; + } + + public Array convertMissing(Array in) { + Array out = Array.factory(in.getDataType(), in.getShape()); + IndexIterator iterIn = in.getIndexIterator(); + IndexIterator iterOut = out.getIndexIterator(); + + // iterate and convert elements + while (iterIn.hasNext()) { + Number value = (Number) iterIn.getObjectNext(); + value = convertMissing(value); + iterOut.setObjectNext(value); + } + + return out; + } +} diff --git a/cdm/core/src/main/java/ucar/nc2/filter/Filter.java b/cdm/core/src/main/java/ucar/nc2/filter/Filter.java index e21e28605a..ce21551332 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/Filter.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/Filter.java @@ -10,6 +10,9 @@ import java.util.Formatter; import java.util.Map; +/** + * A class implementing a conversion to be applied on large chunk of data + */ public abstract class Filter { public abstract String getName(); diff --git a/cdm/core/src/main/java/ucar/nc2/filter/FilterHelpers.java b/cdm/core/src/main/java/ucar/nc2/filter/FilterHelpers.java new file mode 100644 index 0000000000..82fdd49a80 --- /dev/null +++ b/cdm/core/src/main/java/ucar/nc2/filter/FilterHelpers.java @@ -0,0 +1,225 @@ +package ucar.nc2.filter; + +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.IndexIterator; +import ucar.nc2.Attribute; + +import java.nio.*; + +import static ucar.ma2.DataType.*; + +public class FilterHelpers { + + public static Array bytesToArray(byte[] in, DataType type, ByteOrder order) { + return Array.factory(type, new int[] {in.length / type.getSize()}, convertToType(in, type, order)); + } + + public static byte[] arrayToBytes(Array arr, DataType type, ByteOrder order) { + ByteBuffer bb = ByteBuffer.allocate((int) arr.getSize() * type.getSize()); + bb.order(order); + + IndexIterator ii = arr.getIndexIterator(); + while (ii.hasNext()) { + switch (type) { + case BYTE: + case UBYTE: + bb.put(ii.getByteNext()); + break; + case SHORT: + case USHORT: + bb.putShort(ii.getShortNext()); + break; + case INT: + case UINT: + bb.putInt(ii.getIntNext()); + break; + case LONG: + case ULONG: + bb.putLong(ii.getLongNext()); + break; + case FLOAT: + bb.putFloat(ii.getFloatNext()); + break; + case DOUBLE: + bb.putDouble(ii.getDoubleNext()); + break; + } + } + return bb.array(); + } + + private static Object convertToType(byte[] dataIn, DataType wantType, ByteOrder bo) { + if (wantType.getSize() == 1) { + return dataIn; + } // no need for conversion + + ByteBuffer bb = ByteBuffer.wrap(dataIn); + bb.order(bo); + switch (wantType) { + case SHORT: + case USHORT: + ShortBuffer sb = bb.asShortBuffer(); + short[] shortArray = new short[sb.limit()]; + sb.get(shortArray); + return shortArray; + case INT: + case UINT: + IntBuffer ib = bb.asIntBuffer(); + int[] intArray = new int[ib.limit()]; + ib.get(intArray); + return intArray; + case LONG: + case ULONG: + LongBuffer lb = bb.asLongBuffer(); + long[] longArray = new long[lb.limit()]; + lb.get(longArray); + return longArray; + case FLOAT: + FloatBuffer fb = bb.asFloatBuffer(); + float[] floatArray = new float[fb.limit()]; + fb.get(floatArray); + return floatArray; + case DOUBLE: + DoubleBuffer db = bb.asDoubleBuffer(); + double[] doubleArray = new double[db.limit()]; + db.get(doubleArray); + return doubleArray; + default: + return bb.array(); + } + } + + /** + * Returns the smallest numeric data type that: + *
      + *
    1. can hold a larger integer than {@code dataType} can
      2. + *
    2. if integral, has the same signedness as {@code dataType}
      3. + *
    + *

    + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
    ArgumentResult
    BYTESHORT
    UBYTEUSHORT
    SHORTINT
    USHORTUINT
    INTLONG
    UINTULONG
    LONGDOUBLE
    ULONGDOUBLE
    Any other data typeJust return argument
    + *

    + * The returned type is intended to be just big enough to hold the result of performing an unsigned conversion of a + * value of the smaller type. For example, the {@code byte} value {@code -106} equals {@code 150} when interpreted + * as unsigned. That won't fit in a (signed) {@code byte}, but it will fit in a {@code short}. + * + * @param dataType an integral data type. + * @return the next larger type. + */ + public static DataType nextLarger(DataType dataType) { + switch (dataType) { + case BYTE: + return SHORT; + case UBYTE: + return USHORT; + case SHORT: + return INT; + case USHORT: + return UINT; + case INT: + return LONG; + case UINT: + return ULONG; + case LONG: + case ULONG: + return DOUBLE; + default: + return dataType; + } + } + + // Get the data type of an attribute. Make it unsigned if the variable is unsigned. + public static DataType getAttributeDataType(Attribute attribute, DataType.Signedness signedness) { + DataType dataType = attribute.getDataType(); + if (signedness == Signedness.UNSIGNED) { + // If variable is unsigned, make its integral attributes unsigned too. + dataType = dataType.withSignedness(signedness); + } + return dataType; + } + + + public static int rank(DataType dataType) { + if (dataType == null) { + return -1; + } + + switch (dataType) { + case BYTE: + return 0; + case UBYTE: + return 1; + case SHORT: + return 2; + case USHORT: + return 3; + case INT: + return 4; + case UINT: + return 5; + case LONG: + return 6; + case ULONG: + return 7; + case FLOAT: + return 8; + case DOUBLE: + return 9; + default: + return -1; + } + } + + public static DataType largestOf(DataType... dataTypes) { + DataType widest = null; + for (DataType dataType : dataTypes) { + if (widest == null) { + widest = dataType; + } else if (rank(dataType) > rank(widest)) { + widest = dataType; + } + } + return widest; + } +} diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java index 369ad2f447..4ff1b98cc0 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java @@ -8,6 +8,9 @@ import ucar.ma2.Array; import ucar.ma2.DataType; import ucar.ma2.IndexIterator; +import ucar.nc2.Attribute; +import ucar.nc2.constants.CDM; +import ucar.nc2.dataset.VariableDS; import java.nio.*; import java.util.HashMap; @@ -25,6 +28,13 @@ public class ScaleOffset extends Filter { private static final int id = 6; + public static class Keys { + public static final String OFFSET_KEY = "offset"; + public static final String SCALE_KEY = "scale"; + public static final String DTYPE_KEY = "dtype"; + public static final String ASTYPE_KEY = "astype"; + } + // maps numeric zarr datatypes to CDM datatypes private static Map dTypeMap; @@ -43,7 +53,7 @@ public class ScaleOffset extends Filter { } private final double offset; - private final int scale; + private final double scale; // type information for original data type private final ByteOrder dtypeOrder; @@ -53,24 +63,73 @@ public class ScaleOffset extends Filter { private final DataType astype; private final ByteOrder astypeOrder; + public static ScaleOffset createFromVariable(VariableDS var) { + + DataType scaleType = null, offsetType = null; + double scale = 1.0, offset = 0; + + DataType origDataType = var.getDataType(); + DataType.Signedness signedness = var.getSignedness(); + + Attribute scaleAtt = var.findAttribute(CDM.SCALE_FACTOR); + if (scaleAtt != null && !scaleAtt.isString()) { + scaleType = FilterHelpers.getAttributeDataType(scaleAtt, signedness); + scale = var.convertUnsigned(scaleAtt.getNumericValue()).doubleValue(); + } + + Attribute offsetAtt = var.findAttribute(CDM.ADD_OFFSET); + if (offsetAtt != null && !offsetAtt.isString()) { + offsetType = FilterHelpers.getAttributeDataType(offsetAtt, signedness); + offset = var.convertUnsigned(offsetAtt.getNumericValue()).doubleValue(); + } + if (scale != 1.0 || offset != 0) { + DataType scaledOffsetType = + FilterHelpers.largestOf(var.getUnsignedConversionType(), scaleType, offsetType).withSignedness(signedness); + + Map scaleOffsetProps = new HashMap<>(); + scaleOffsetProps.put(ScaleOffset.Keys.OFFSET_KEY, offset); + scaleOffsetProps.put(ScaleOffset.Keys.SCALE_KEY, scale); + scaleOffsetProps.put(ScaleOffset.Keys.DTYPE_KEY, origDataType); + scaleOffsetProps.put(ScaleOffset.Keys.ASTYPE_KEY, scaledOffsetType); + return new ScaleOffset(scaleOffsetProps); + } + return null; + } public ScaleOffset(Map properties) { // get offset and scale parameters - offset = ((Number) properties.getOrDefault("offset", 0)).doubleValue(); - scale = (int) properties.getOrDefault("scale", 1); + offset = ((Number) properties.getOrDefault(Keys.OFFSET_KEY, 0)).doubleValue(); + scale = (int) properties.getOrDefault(Keys.SCALE_KEY, 1); // input data type - String type = (String) properties.get("dtype"); - dtype = parseDataType(type); - if (dtype == null) { + Object typeProp = properties.get(Keys.DTYPE_KEY); + if (typeProp instanceof String) { + String type = (String) typeProp; + dtype = parseDataType(type); + if (dtype == null) { + throw new RuntimeException("ScaleOffset error: could not parse dtype"); + } + dtypeOrder = parseByteOrder(type, ByteOrder.LITTLE_ENDIAN); + } else if (typeProp instanceof DataType) { + dtype = (DataType) typeProp; + dtypeOrder = ByteOrder.LITTLE_ENDIAN; + } else { throw new RuntimeException("ScaleOffset error: could not parse dtype"); } - dtypeOrder = parseByteOrder(type, ByteOrder.LITTLE_ENDIAN); // get storage type, if exists, or default to dtype - String aType = (String) properties.getOrDefault("astype", type); - astype = parseDataType(aType); - astypeOrder = parseByteOrder(aType, dtypeOrder); + Object aTypeProp = properties.getOrDefault(Keys.ASTYPE_KEY, null); + if (aTypeProp instanceof String) { + String aType = (String) aTypeProp; + astype = parseDataType(aType); + astypeOrder = parseByteOrder(aType, dtypeOrder); + } else if (aTypeProp instanceof DataType) { + astype = (DataType) aTypeProp; + astypeOrder = ByteOrder.LITTLE_ENDIAN; + } else { + astype = dtype; + astypeOrder = dtypeOrder; + } } @Override @@ -83,27 +142,36 @@ public int getId() { return id; } + public double getScaleFactor() { + return this.scale; + } + + public double getOffset() { + return this.offset; + } + + public DataType getScaledOffsetType() { + return this.astype; + } + @Override public byte[] encode(byte[] dataIn) { - Array in = - Array.factory(dtype, new int[] {dataIn.length / dtype.getSize()}, convertToType(dataIn, dtype, dtypeOrder)); - Array out = applyScaleOffset(in); - return arrayToBytes(out, astype, astypeOrder); + Array out = applyScaleOffset(FilterHelpers.bytesToArray(dataIn, dtype, dtypeOrder)); + return FilterHelpers.arrayToBytes(out, astype, astypeOrder); } @Override public byte[] decode(byte[] dataIn) { - Array in = - Array.factory(astype, new int[] {dataIn.length / astype.getSize()}, convertToType(dataIn, astype, astypeOrder)); - Array out = removeScaleOffset(in); - return arrayToBytes(out, dtype, dtypeOrder); + Array out = removeScaleOffset(FilterHelpers.bytesToArray(dataIn, astype, astypeOrder)); + return FilterHelpers.arrayToBytes(out, dtype, dtypeOrder); } - private Array applyScaleOffset(Array in) { + // not used anywhere yet + public Array applyScaleOffset(Array in) { // use wider datatype if unsigned DataType outType = astype; if (astype.getSignedness() == Signedness.UNSIGNED) { - outType = nextLarger(astype).withSignedness(DataType.Signedness.UNSIGNED); + outType = FilterHelpers.nextLarger(astype).withSignedness(DataType.Signedness.UNSIGNED); } // create conversion array @@ -122,11 +190,11 @@ private Array applyScaleOffset(Array in) { return out; } - private Array removeScaleOffset(Array in) { + public Array removeScaleOffset(Array in) { // use wider datatype if unsigned DataType outType = dtype; if (dtype.getSignedness() == Signedness.UNSIGNED) { - outType = nextLarger(dtype).withSignedness(DataType.Signedness.UNSIGNED); + outType = FilterHelpers.nextLarger(dtype).withSignedness(DataType.Signedness.UNSIGNED); } // create conversion array @@ -166,26 +234,14 @@ private static ByteOrder parseByteOrder(String dtype, ByteOrder defaultOrder) { return defaultOrder; } - private DataType nextLarger(DataType dataType) { - switch (dataType) { - case BYTE: - return SHORT; - case UBYTE: - return USHORT; - case SHORT: - return INT; - case USHORT: - return UINT; - case INT: - return LONG; - case UINT: - return ULONG; - case LONG: - case ULONG: - return DOUBLE; - default: - return dataType; - } + public int applyScaleOffset(Number value) { + double convertedValue = value.doubleValue(); + return (int) Math.round((convertedValue - offset) * scale); + } + + public double removeScaleOffset(Number value) { + double convertedValue = value.doubleValue(); + return convertedValue / scale + offset; } private Number convertUnsigned(Number value, Signedness signedness) { @@ -197,90 +253,6 @@ private Number convertUnsigned(Number value, Signedness signedness) { } } - private Object convertToType(byte[] dataIn, DataType wantType, ByteOrder bo) { - if (wantType.getSize() == 1) { - return dataIn; - } // no need for conversion - - ByteBuffer bb = ByteBuffer.wrap(dataIn); - bb.order(bo); - switch (wantType) { - case SHORT: - case USHORT: - ShortBuffer sb = bb.asShortBuffer(); - short[] shortArray = new short[sb.limit()]; - sb.get(shortArray); - return shortArray; - case INT: - case UINT: - IntBuffer ib = bb.asIntBuffer(); - int[] intArray = new int[ib.limit()]; - ib.get(intArray); - return intArray; - case LONG: - case ULONG: - LongBuffer lb = bb.asLongBuffer(); - long[] longArray = new long[lb.limit()]; - lb.get(longArray); - return longArray; - case FLOAT: - FloatBuffer fb = bb.asFloatBuffer(); - float[] floatArray = new float[fb.limit()]; - fb.get(floatArray); - return floatArray; - case DOUBLE: - DoubleBuffer db = bb.asDoubleBuffer(); - double[] doubleArray = new double[db.limit()]; - db.get(doubleArray); - return doubleArray; - default: - return bb.array(); - } - } - - private int applyScaleOffset(Number value) { - double convertedValue = value.doubleValue(); - return (int) Math.round((convertedValue - offset) * scale); - } - - private double removeScaleOffset(Number value) { - double convertedValue = value.doubleValue(); - return convertedValue / scale + offset; - } - - private byte[] arrayToBytes(Array arr, DataType type, ByteOrder order) { - ByteBuffer bb = ByteBuffer.allocate((int) arr.getSize() * type.getSize()); - bb.order(order); - - IndexIterator ii = arr.getIndexIterator(); - while (ii.hasNext()) { - switch (type) { - case BYTE: - case UBYTE: - bb.put(ii.getByteNext()); - break; - case SHORT: - case USHORT: - bb.putShort(ii.getShortNext()); - break; - case INT: - case UINT: - bb.putInt(ii.getIntNext()); - break; - case LONG: - case ULONG: - bb.putLong(ii.getLongNext()); - break; - case FLOAT: - bb.putFloat(ii.getFloatNext()); - break; - case DOUBLE: - bb.putDouble(ii.getDoubleNext()); - break; - } - } - return bb.array(); - } public static class Provider implements FilterProvider { diff --git a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java new file mode 100644 index 0000000000..ddba9a8220 --- /dev/null +++ b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java @@ -0,0 +1,68 @@ +package ucar.nc2.filter; + +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.IndexIterator; +import ucar.nc2.Variable; +import ucar.nc2.constants.CDM; +import ucar.nc2.dataset.VariableDS; + +public class UnsignedConversion { + + private DataType outType; + + public static UnsignedConversion createFromVar(VariableDS var) { + DataType origDataType = var.getDataType(); + DataType unsignedConversionType = origDataType; + + // unsignedConversionType is initialized to origDataType, and origDataType may be a non-integral type that doesn't + // have an "unsigned flavor" (such as FLOAT and DOUBLE). Furthermore, unsignedConversionType may start out as + // integral, but then be widened to non-integral (i.e. LONG -> DOUBLE). For these reasons, we cannot rely upon + // unsignedConversionType to store the signedness of the variable. We need a separate field. + DataType.Signedness signedness = origDataType.getSignedness(); + + // In the event of conflict, "unsigned" wins. Potential conflicts include: + // 1. origDataType is unsigned, but variable has "_Unsigned == false" attribute. + // 2. origDataType is signed, but variable has "_Unsigned == true" attribute. + if (signedness == DataType.Signedness.SIGNED) { + String unsignedAtt = var.attributes().findAttributeString(CDM.UNSIGNED, null); + if (unsignedAtt != null && unsignedAtt.equalsIgnoreCase("true")) { + signedness = DataType.Signedness.UNSIGNED; + } + } + + if (signedness == DataType.Signedness.UNSIGNED) { + // We may need a larger data type to hold the results of the unsigned conversion. + unsignedConversionType = FilterHelpers.nextLarger(origDataType).withSignedness(DataType.Signedness.UNSIGNED); + } + return new UnsignedConversion(unsignedConversionType); + } + + public UnsignedConversion(DataType outType) { + this.outType = outType; + } + + public DataType getOutType() { + return this.outType; + } + + public Number convertUnsigned(Number value) { + return DataType.widenNumberIfNegative(value); + } + + + public Array convertUnsigned(Array in) { + Array out = Array.factory(outType, in.getShape()); + IndexIterator iterIn = in.getIndexIterator(); + IndexIterator iterOut = out.getIndexIterator(); + + // iterate and convert elements + while (iterIn.hasNext()) { + Number value = (Number) iterIn.getObjectNext(); + value = convertUnsigned(value); + iterOut.setObjectNext(value); + } + + return out; + } +} diff --git a/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5iospNew.java b/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5iospNew.java index e881ee9103..137033093f 100644 --- a/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5iospNew.java +++ b/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5iospNew.java @@ -509,7 +509,7 @@ void convertHeap(ArrayStructureBB asbb, int pos, StructureMembers sm) throws IOE // vlenarray extracts the i'th vlen contents (struct not supported). Array vlenArray = header.readHeapVlen(bb, destPos, m.getDataType(), endian); fieldarray[i] = vlenArray; - destPos += VLEN_T_SIZE; // Apparentlly no way to compute VLEN_T_SIZE on the fly + destPos += VLEN_T_SIZE; // Apparently no way to compute VLEN_T_SIZE on the fly } Array result; if (prefixrank == 0) // if scalar, return just the singleton vlen array From a2d9fe25c474b3e1a554f27a91dc0aba328110d9 Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Mon, 27 Mar 2023 13:58:06 -0700 Subject: [PATCH 03/24] add tests --- .../java/ucar/nc2/dataset/VariableDS.java | 16 +- .../java/ucar/nc2/filter/ScaleOffset.java | 19 ++- .../ucar/nc2/filter/UnsignedConversion.java | 1 + .../ucar/nc2/filter/TestEnhancements.java | 148 ++++++++++++++++++ .../java/ucar/nc2/filter/TestFilters.java | 2 +- 5 files changed, 175 insertions(+), 11 deletions(-) create mode 100644 cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index a30e5a3fc2..362c91d22d 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -272,8 +272,16 @@ Array convert(Array data, Set enhancements) { if (this.isVariableLength) { return data; } - return scaleMissingUnsignedProxy.convert(data, enhancements.contains(Enhance.ConvertUnsigned), - enhancements.contains(Enhance.ApplyScaleOffset), enhancements.contains(Enhance.ConvertMissing)); + if (enhancements.contains(Enhance.ConvertUnsigned)) { + data = unsignedConversion.convertUnsigned(data); + } + if (enhancements.contains(Enhance.ApplyScaleOffset) && scaleOffset != null) { + data = scaleOffset.removeScaleOffset(data); + } + if (enhancements.contains(Enhance.ConvertMissing)) { + data = convertMissing.convertMissing(data); + } + return data; } } @@ -724,7 +732,7 @@ public void setMissingDataIsMissing(boolean b) { @Nullable @Override public DataType getScaledOffsetType() { - return scaleOffset.getScaledOffsetType(); + return scaleOffset != null ? scaleOffset.getScaledOffsetType() : dataType; } @Override @@ -822,7 +830,7 @@ protected VariableDS(Builder builder, Group parentGroup) { } if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR)) { this.scaleOffset = ScaleOffset.createFromVariable(this); - this.dataType = scaleOffset.getScaledOffsetType(); + this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType; } if (this.enhanceMode.contains(Enhance.ConvertMissing)) { this.convertMissing = ConvertMissing.createFromVariable(this); diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java index 4ff1b98cc0..dcfefb94d4 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java @@ -55,6 +55,9 @@ public static class Keys { private final double offset; private final double scale; + private static final double DEFAULT_OFFSET = 0.0; + private static final double DEFAULT_SCALE = 1.0; + // type information for original data type private final ByteOrder dtypeOrder; private final DataType dtype; @@ -66,7 +69,7 @@ public static class Keys { public static ScaleOffset createFromVariable(VariableDS var) { DataType scaleType = null, offsetType = null; - double scale = 1.0, offset = 0; + double scale = DEFAULT_SCALE, offset = DEFAULT_OFFSET; DataType origDataType = var.getDataType(); DataType.Signedness signedness = var.getSignedness(); @@ -82,7 +85,7 @@ public static ScaleOffset createFromVariable(VariableDS var) { offsetType = FilterHelpers.getAttributeDataType(offsetAtt, signedness); offset = var.convertUnsigned(offsetAtt.getNumericValue()).doubleValue(); } - if (scale != 1.0 || offset != 0) { + if (scale != DEFAULT_SCALE || offset != DEFAULT_OFFSET) { DataType scaledOffsetType = FilterHelpers.largestOf(var.getUnsignedConversionType(), scaleType, offsetType).withSignedness(signedness); @@ -98,8 +101,8 @@ public static ScaleOffset createFromVariable(VariableDS var) { public ScaleOffset(Map properties) { // get offset and scale parameters - offset = ((Number) properties.getOrDefault(Keys.OFFSET_KEY, 0)).doubleValue(); - scale = (int) properties.getOrDefault(Keys.SCALE_KEY, 1); + offset = ((Number) properties.getOrDefault(Keys.OFFSET_KEY, DEFAULT_OFFSET)).doubleValue(); + scale = ((Number) properties.getOrDefault(Keys.SCALE_KEY, DEFAULT_SCALE)).doubleValue(); // input data type Object typeProp = properties.get(Keys.DTYPE_KEY); @@ -156,18 +159,21 @@ public DataType getScaledOffsetType() { @Override public byte[] encode(byte[] dataIn) { + if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return dataIn; } Array out = applyScaleOffset(FilterHelpers.bytesToArray(dataIn, dtype, dtypeOrder)); return FilterHelpers.arrayToBytes(out, astype, astypeOrder); } @Override public byte[] decode(byte[] dataIn) { + if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return dataIn; } Array out = removeScaleOffset(FilterHelpers.bytesToArray(dataIn, astype, astypeOrder)); return FilterHelpers.arrayToBytes(out, dtype, dtypeOrder); } // not used anywhere yet public Array applyScaleOffset(Array in) { + if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return in; } // use wider datatype if unsigned DataType outType = astype; if (astype.getSignedness() == Signedness.UNSIGNED) { @@ -191,6 +197,7 @@ public Array applyScaleOffset(Array in) { } public Array removeScaleOffset(Array in) { + if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return in; } // use wider datatype if unsigned DataType outType = dtype; if (dtype.getSignedness() == Signedness.UNSIGNED) { @@ -234,9 +241,9 @@ private static ByteOrder parseByteOrder(String dtype, ByteOrder defaultOrder) { return defaultOrder; } - public int applyScaleOffset(Number value) { + public double applyScaleOffset(Number value) { double convertedValue = value.doubleValue(); - return (int) Math.round((convertedValue - offset) * scale); + return Math.round((convertedValue - offset) * scale); } public double removeScaleOffset(Number value) { diff --git a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java index ddba9a8220..8c1da9ea3a 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java @@ -47,6 +47,7 @@ public DataType getOutType() { } public Number convertUnsigned(Number value) { + Number newVal = DataType.widenNumberIfNegative(value); return DataType.widenNumberIfNegative(value); } diff --git a/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java new file mode 100644 index 0000000000..e54256d4a2 --- /dev/null +++ b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java @@ -0,0 +1,148 @@ +package ucar.nc2.filter; + +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Attribute; +import ucar.nc2.Dimension; +import ucar.nc2.Variable; +import ucar.nc2.constants.CDM; +import ucar.nc2.dataset.NetcdfDataset; +import ucar.nc2.dataset.NetcdfDatasets; +import ucar.nc2.write.NetcdfFormatWriter; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.IOException; + +public class TestEnhancements { + + private static NetcdfDataset ncd; + + private static short[] signedShorts = new short[] {123, 124, 125, 126, 127, -128, -127, -126, -125, -124}; + + private static final float VALID_MIN = 100; + private static final float VALID_MAX = 200; + private static final float FILL_VALUE = 150; + private static float[] missingData = new float[]{90, 100, Float.NaN, 120, 130, 140, 150, 190, 200, 210}; + + private static final Short SIGNED_SCALED_MAX = -126; + private static final Short SIGNED_SCALED_FILL_VALUE = -128; + + @ClassRule + public static final TemporaryFolder tempFolder = new TemporaryFolder(); + + @BeforeClass + public static void setUp() throws IOException , InvalidRangeException { + final int data_len = 10; + String filePath = tempFolder.newFile().getAbsolutePath(); + NetcdfFormatWriter.Builder builder = NetcdfFormatWriter.createNewNetcdf3(filePath); + Dimension dim = builder.addDimension("dim", data_len); + + Array signedData = Array.factory(DataType.SHORT, new int[]{data_len}, signedShorts); + // signed shorts + builder.addVariable("signedVar", DataType.SHORT, "dim"); + // unsigned shorts + builder.addVariable("unsignedVar", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.UNSIGNED, "true")); + + // scaled and offset data + builder.addVariable("scaleOffsetVar", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.SCALE_FACTOR, .1)) + .addAttribute(new Attribute(CDM.ADD_OFFSET, 10)); + + Array missingDataArray = Array.factory(DataType.FLOAT, new int[]{data_len}, missingData); + // Data with min + builder.addVariable("validMin", DataType.FLOAT, "dim").addAttribute(new Attribute(CDM.VALID_MIN, VALID_MIN)); + // Data with min and max + builder.addVariable("validMinMax", DataType.FLOAT, "dim") + .addAttribute(new Attribute(CDM.VALID_MIN, VALID_MIN)) + .addAttribute(new Attribute(CDM.VALID_MAX, VALID_MAX)); + // Data with range and fill value + Array range = Array.factory(DataType.FLOAT, new int[]{2}, new float[]{VALID_MIN, VALID_MAX}); + builder.addVariable("validRange", DataType.FLOAT, "dim") + .addAttribute(Attribute.builder(CDM.VALID_RANGE).setValues(range).build()) + .addAttribute(Attribute.builder(CDM.FILL_VALUE).setNumericValue(FILL_VALUE, true).build()); + + // unsigned, scaled/offset, and missing value + Array enhanceAllArray = Array.factory(DataType.SHORT, new int[]{data_len}, signedShorts); + builder.addVariable("enhanceAll", DataType.SHORT, "dim") + .addAttribute(new Attribute(CDM.UNSIGNED, "true")) + .addAttribute(new Attribute(CDM.SCALE_FACTOR, .1)) + .addAttribute(new Attribute(CDM.ADD_OFFSET, 10)) + .addAttribute(new Attribute(CDM.VALID_MAX, SIGNED_SCALED_MAX)) + .addAttribute(new Attribute(CDM.FILL_VALUE, SIGNED_SCALED_FILL_VALUE)); + + // write data + NetcdfFormatWriter writer = builder.build(); + writer.write(writer.findVariable("signedVar"), new int[1], signedData); + writer.write(writer.findVariable("unsignedVar"), new int[1], signedData); + writer.write(writer.findVariable("scaleOffsetVar"), new int[1], signedData); + writer.write(writer.findVariable("validMin"), new int[1], missingDataArray); + writer.write(writer.findVariable("validMinMax"), new int[1], missingDataArray); + writer.write(writer.findVariable("validRange"), new int[1], missingDataArray); + writer.write(writer.findVariable("enhanceAll"), new int[1], enhanceAllArray); + writer.close(); + ncd = NetcdfDatasets.openDataset(filePath); + } + + @Test + public void testUnsignedConversion() throws IOException { + final int[] unsignedValues = new int[]{123, 124, 125, 126, 127, -128, -127, -126, -125, -126}; + // signed var + Variable v = ncd.findVariable("signedVar"); + Array data = v.read(); + assertThat(data.isUnsigned()).isFalse(); + assertThat(data.getDataType()).isEqualTo(DataType.SHORT); + assertThat((short[])data.copyTo1DJavaArray()).isEqualTo(signedShorts); + + // var with unsigned data type + v = ncd.findVariable("unsignedVar"); + data = v.read(); + assertThat(data.isUnsigned()).isTrue(); + assertThat(data.getDataType()).isEqualTo(DataType.UINT); + assertThat((int[])data.copyTo1DJavaArray()).isEqualTo(unsignedValues); + } + + @Test + public void testScaleOffset() throws IOException { + final double[] expected = new double[]{1240, 1250, 1260, 1270, 1280, -1270, -1260, -1250, -1240, -1230}; + // signed var + Variable v = ncd.findVariable("scaleOffsetVar"); + Array data = v.read(); + assertThat(data.isUnsigned()).isFalse(); + assertThat(data.getDataType()).isEqualTo(DataType.SHORT); + assertThat((double[])data.copyTo1DJavaArray()).isEqualTo(expected); + } + + @Test + public void testConvertMissing() throws IOException { + // var with valid min + float[] expected = new float[]{Float.NaN, 100, Float.NaN, 120, 130, 140, 150, 190, 200, 210}; + Variable v = ncd.findVariable("validMin"); + Array data = v.read(); + assertThat((float[])data.copyTo1DJavaArray()).isEqualTo(expected); + + // var with valid min and max + expected = new float[]{Float.NaN, 100, Float.NaN, 120, 130, 140, 150, 190, 200, Float.NaN}; + v = ncd.findVariable("validMinMax"); + data = v.read(); + assertThat((float[])data.copyTo1DJavaArray()).isEqualTo(expected); + + // var with valid range and fill value + expected = new float[]{Float.NaN, 100, Float.NaN, 120, 130, 140, Float.NaN, 190, 200, Float.NaN}; + v = ncd.findVariable("validRange"); + data = v.read(); + assertThat((float[])data.copyTo1DJavaArray()).isEqualTo(expected); + } + + @Test + public void testCombinedEnhancements() throws IOException { + int[] expected = new int[]{1240, 1250, 1260, 1270, 1280, 0, 1278, 1260, 0, 0}; + Variable v = ncd.findVariable("enhanceAll"); + Array data = v.read(); + assertThat((int[])data.copyTo1DJavaArray()).isEqualTo(expected); + } +} diff --git a/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java b/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java index 51c67f55bc..b98298efdf 100644 --- a/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java +++ b/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java @@ -100,7 +100,7 @@ public void testScaleOffset() throws IOException { // test empty props Map props2 = new HashMap<>(); props2.put("id", "fixedscaleoffset"); - props2.put("dtype", " Date: Tue, 28 Mar 2023 12:13:42 -0700 Subject: [PATCH 04/24] fix bugs --- .../src/test/java/ucar/nc2/TestSequence.java | 5 - .../nc2/ft/point/TestCfDocDsgExamples.java | 4 - .../EnhanceScaleMissingUnsignedImpl.java | 628 ------------------ .../java/ucar/nc2/dataset/VariableDS.java | 129 ++-- .../java/ucar/nc2/filter/ConvertMissing.java | 24 +- .../java/ucar/nc2/filter/ScaleOffset.java | 24 +- .../ucar/nc2/filter/UnsignedConversion.java | 2 - .../ucar/nc2/filter/TestEnhancements.java | 246 ++++--- .../src/test/java/ucar/nc2/TestSequence.java | 11 - 9 files changed, 230 insertions(+), 843 deletions(-) delete mode 100644 cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsignedImpl.java diff --git a/cdm-test/src/test/java/ucar/nc2/TestSequence.java b/cdm-test/src/test/java/ucar/nc2/TestSequence.java index 6e6780a5d6..69cbbf1c8b 100644 --- a/cdm-test/src/test/java/ucar/nc2/TestSequence.java +++ b/cdm-test/src/test/java/ucar/nc2/TestSequence.java @@ -36,16 +36,12 @@ import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import ucar.ma2.*; -import ucar.nc2.dataset.NetcdfDataset; import ucar.nc2.dataset.NetcdfDatasets; import ucar.unidata.util.test.category.NeedsCdmUnitTest; import ucar.unidata.util.test.TestDir; import java.io.File; import java.io.IOException; -import java.lang.invoke.MethodHandles; import java.util.Arrays; import java.util.List; @@ -56,7 +52,6 @@ * @since Nov 10, 2009 */ public class TestSequence { - private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @Test @Category(NeedsCdmUnitTest.class) diff --git a/cdm-test/src/test/java/ucar/nc2/ft/point/TestCfDocDsgExamples.java b/cdm-test/src/test/java/ucar/nc2/ft/point/TestCfDocDsgExamples.java index c3f29ef9a7..0e7422ec8e 100644 --- a/cdm-test/src/test/java/ucar/nc2/ft/point/TestCfDocDsgExamples.java +++ b/cdm-test/src/test/java/ucar/nc2/ft/point/TestCfDocDsgExamples.java @@ -4,8 +4,6 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import ucar.nc2.NetcdfFile; import ucar.nc2.constants.FeatureType; import ucar.nc2.dataset.NetcdfDatasets; @@ -16,7 +14,6 @@ import java.io.File; import java.io.FilenameFilter; import java.io.IOException; -import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; @@ -28,7 +25,6 @@ */ @RunWith(Parameterized.class) public class TestCfDocDsgExamples { - private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String cfDocDsgExamplesDir = TestDir.cdmLocalFromTestDataDir + "cfDocDsgExamples/"; private static List getPointDatasets() { diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsignedImpl.java b/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsignedImpl.java deleted file mode 100644 index 7a296e4b90..0000000000 --- a/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsignedImpl.java +++ /dev/null @@ -1,628 +0,0 @@ -/* - * Copyright (c) 1998-2020 John Caron and University Corporation for Atmospheric Research/Unidata - * See LICENSE for license information. - */ -package ucar.nc2.dataset; - -import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import ucar.ma2.*; -import ucar.nc2.*; -import ucar.nc2.constants.CDM; -import ucar.nc2.constants.DataFormatType; -import ucar.nc2.dataset.NetcdfDataset.Enhance; -import ucar.nc2.iosp.netcdf3.N3iosp; -import ucar.nc2.util.Misc; -import javax.annotation.Nonnull; -import java.lang.invoke.MethodHandles; -import java.util.Arrays; -import static ucar.ma2.DataType.*; - -/** - * Implementation of EnhanceScaleMissingUnsigned for unsigned data, scale/offset packed data, and missing data. - * - * @author caron - * @author cwardgar - * @see EnhanceScaleMissingUnsigned - */ -class EnhanceScaleMissingUnsignedImpl implements EnhanceScaleMissingUnsigned { - private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private DataType origDataType, unsignedConversionType, scaledOffsetType; - - // defaults from NetcdfDataset modes - private boolean invalidDataIsMissing = NetcdfDataset.invalidDataIsMissing; - private boolean fillValueIsMissing = NetcdfDataset.fillValueIsMissing; - private boolean missingDataIsMissing = NetcdfDataset.missingDataIsMissing; - - private boolean useScaleOffset; - private double scale = 1.0, offset; - - private boolean hasValidRange, hasValidMin, hasValidMax; - private double validMin = -Double.MAX_VALUE, validMax = Double.MAX_VALUE; - - private boolean hasFillValue; - private double fillValue; // LOOK: making it double not really correct. What about CHAR? - - private boolean hasMissingValue; - private double[] missingValue; // LOOK: also wrong to make double, for the same reason. - - private DataType.Signedness signedness; - - - /** - * Constructor, when you dont want anything done. - */ - EnhanceScaleMissingUnsignedImpl() {} - - /** - * Constructor, default values. - * - * @param forVar the Variable to decorate. - */ - EnhanceScaleMissingUnsignedImpl(VariableDS forVar, Set enhancements) { - this(forVar, enhancements, NetcdfDataset.fillValueIsMissing, NetcdfDataset.invalidDataIsMissing, - NetcdfDataset.missingDataIsMissing); - } - - /** - * Constructor. - * If scale/offset attributes are found, remove them from the decorated variable. - * - * @param forVar the Variable to decorate. - * @param fillValueIsMissing use _FillValue for isMissing() - * @param invalidDataIsMissing use valid_range for isMissing() - * @param missingDataIsMissing use missing_value for isMissing() - */ - EnhanceScaleMissingUnsignedImpl(VariableDS forVar, Set enhancements, boolean fillValueIsMissing, - boolean invalidDataIsMissing, boolean missingDataIsMissing) { - this.fillValueIsMissing = fillValueIsMissing; - this.invalidDataIsMissing = invalidDataIsMissing; - this.missingDataIsMissing = missingDataIsMissing; - - this.origDataType = forVar.getDataType(); - this.unsignedConversionType = origDataType; - - // unsignedConversionType is initialized to origDataType, and origDataType may be a non-integral type that doesn't - // have an "unsigned flavor" (such as FLOAT and DOUBLE). Furthermore, unsignedConversionType may start out as - // integral, but then be widened to non-integral (i.e. LONG -> DOUBLE). For these reasons, we cannot rely upon - // unsignedConversionType to store the signedness of the variable. We need a separate field. - this.signedness = origDataType.getSignedness(); - - // In the event of conflict, "unsigned" wins. Potential conflicts include: - // 1. origDataType is unsigned, but variable has "_Unsigned == false" attribute. - // 2. origDataType is signed, but variable has "_Unsigned == true" attribute. - if (signedness == Signedness.SIGNED) { - String unsignedAtt = forVar.attributes().findAttributeString(CDM.UNSIGNED, null); - if (unsignedAtt != null && unsignedAtt.equalsIgnoreCase("true")) { - this.signedness = Signedness.UNSIGNED; - } - } - - if (signedness == Signedness.UNSIGNED) { - // We may need a larger data type to hold the results of the unsigned conversion. - this.unsignedConversionType = nextLarger(origDataType).withSignedness(Signedness.UNSIGNED); - logger.debug("assign unsignedConversionType = {}", unsignedConversionType); - } - - DataType scaleType = null, offsetType = null, validType = null; - logger.debug("{} for Variable = {}", getClass().getSimpleName(), forVar.getShortName()); - - Attribute scaleAtt = forVar.findAttribute(CDM.SCALE_FACTOR); - if (scaleAtt != null && !scaleAtt.isString()) { - scaleType = getAttributeDataType(scaleAtt); - scale = convertUnsigned(scaleAtt.getNumericValue(), scaleType).doubleValue(); - useScaleOffset = enhancements.contains(Enhance.ApplyScaleOffset); - logger.debug("scale = {} type = {}", scale, scaleType); - } - - Attribute offsetAtt = forVar.findAttribute(CDM.ADD_OFFSET); - if (offsetAtt != null && !offsetAtt.isString()) { - offsetType = getAttributeDataType(offsetAtt); - offset = convertUnsigned(offsetAtt.getNumericValue(), offsetType).doubleValue(); - useScaleOffset = enhancements.contains(Enhance.ApplyScaleOffset); - logger.debug("offset = {}", offset); - } - - ////// missing data : valid_range. assume here its in units of unpacked data. correct this below - Attribute validRangeAtt = forVar.findAttribute(CDM.VALID_RANGE); - if (validRangeAtt != null && !validRangeAtt.isString() && validRangeAtt.getLength() > 1) { - validType = getAttributeDataType(validRangeAtt); - validMin = convertUnsigned(validRangeAtt.getNumericValue(0), validType).doubleValue(); - validMax = convertUnsigned(validRangeAtt.getNumericValue(1), validType).doubleValue(); - hasValidRange = true; - logger.debug("valid_range = {} {}", validMin, validMax); - } - - Attribute validMinAtt = forVar.findAttribute(CDM.VALID_MIN); - Attribute validMaxAtt = forVar.findAttribute(CDM.VALID_MAX); - - // Only process the valid_min and valid_max attributes if valid_range isn't present. - if (!hasValidRange) { - if (validMinAtt != null && !validMinAtt.isString()) { - validType = getAttributeDataType(validMinAtt); - validMin = convertUnsigned(validMinAtt.getNumericValue(), validType).doubleValue(); - hasValidMin = true; - logger.debug("valid_min = {}", validMin); - } - - if (validMaxAtt != null && !validMaxAtt.isString()) { - validType = largestOf(validType, getAttributeDataType(validMaxAtt)); - validMax = convertUnsigned(validMaxAtt.getNumericValue(), validType).doubleValue(); - hasValidMax = true; - logger.debug("valid_min = {}", validMax); - } - - if (hasValidMin && hasValidMax) { - hasValidRange = true; - } - } - - /// _FillValue - Attribute fillValueAtt = forVar.findAttribute(CDM.FILL_VALUE); - if (fillValueAtt != null && !fillValueAtt.isString()) { - DataType fillType = getAttributeDataType(fillValueAtt); - fillValue = convertUnsigned(fillValueAtt.getNumericValue(), fillType).doubleValue(); - fillValue = applyScaleOffset(fillValue); // This will fail when _FillValue is CHAR. - hasFillValue = true; - } else { - // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value. - String fileTypeId = forVar.orgFileTypeId; - boolean isNetcdfIosp = DataFormatType.NETCDF.getDescription().equals(fileTypeId) - || DataFormatType.NETCDF4.getDescription().equals(fileTypeId); - - if (isNetcdfIosp) { - if (unsignedConversionType.isNumeric()) { - fillValue = applyScaleOffset(N3iosp.getFillValueDefault(unsignedConversionType)); - hasFillValue = true; - } - } - } - - /// missing_value - Attribute missingValueAtt = forVar.findAttribute(CDM.MISSING_VALUE); - if (missingValueAtt != null) { - if (missingValueAtt.isString()) { - String svalue = missingValueAtt.getStringValue(); - if (origDataType == DataType.CHAR) { - missingValue = new double[1]; - if (svalue.isEmpty()) { - missingValue[0] = 0; - } else { - missingValue[0] = svalue.charAt(0); - } - - hasMissingValue = true; - } else { // not a CHAR - try to fix problem where they use a numeric value as a String attribute - try { - missingValue = new double[1]; - missingValue[0] = Double.parseDouble(svalue); - hasMissingValue = true; - } catch (NumberFormatException ex) { - logger.debug("String missing_value not parseable as double = {}", missingValueAtt.getStringValue()); - } - } - } else { // not a string - DataType missType = getAttributeDataType(missingValueAtt); - - missingValue = new double[missingValueAtt.getLength()]; - for (int i = 0; i < missingValue.length; i++) { - missingValue[i] = convertUnsigned(missingValueAtt.getNumericValue(i), missType).doubleValue(); - missingValue[i] = applyScaleOffset(missingValue[i]); - } - logger.debug("missing_data: {}", Arrays.toString(missingValue)); - - for (double mv : missingValue) { - if (!Double.isNaN(mv)) { - hasMissingValue = true; // dont need to do anything if its already a NaN - break; - } - } - } - } - - /// assign convertedDataType if needed - if (useScaleOffset) { - scaledOffsetType = largestOf(unsignedConversionType, scaleType, offsetType).withSignedness(signedness); - logger.debug("assign scaledOffsetType = {}", scaledOffsetType); - - // validData may be packed or unpacked - if (hasValidData()) { - if (rank(validType) == rank(largestOf(scaleType, offsetType)) - && rank(validType) > rank(unsignedConversionType)) { - // If valid_range is the same type as the wider of scale_factor and add_offset, PLUS - // it is wider than the (packed) data, we know that the valid_range values were stored as unpacked. - // We already assumed that this was the case when we first read the attribute values, so there's - // nothing for us to do here. - } else { - // Otherwise, the valid_range values were stored as packed. So now we must unpack them. - if (hasValidRange || hasValidMin) { - validMin = applyScaleOffset(validMin); - } - if (hasValidRange || hasValidMax) { - validMax = applyScaleOffset(validMax); - } - } - // During the scaling process, it is possible that the valid minimum and maximum values have effectively been - // swapped (for example, when the scale value is negative). Go ahead and check to make sure the valid min is - // actually less than the valid max, and if not, fix it. See https://github.com/Unidata/netcdf-java/issues/572. - if (validMin > validMax) { - double tmp = validMin; - validMin = validMax; - validMax = tmp; - } - } - } - } - - // Get the data type of an attribute. Make it unsigned if the variable is unsigned. - private DataType getAttributeDataType(Attribute attribute) { - DataType dataType = attribute.getDataType(); - if (signedness == Signedness.UNSIGNED) { - // If variable is unsigned, make its integral attributes unsigned too. - dataType = dataType.withSignedness(signedness); - } - return dataType; - } - - /** - * Returns a distinct integer for each of the {@link DataType#isNumeric() numeric} data types that can be used to - * (roughly) order them by the range of the DataType. {@code BYTE < UBYTE < SHORT < USHORT < INT < UINT < - * LONG < ULONG < FLOAT < DOUBLE}. {@code -1} will be returned for all non-numeric data types. - * - * @param dataType a numeric data type. - * @return a distinct integer for each of the numeric data types that can be used to (roughly) order them by size. - */ - public static int rank(DataType dataType) { - if (dataType == null) { - return -1; - } - - switch (dataType) { - case BYTE: - return 0; - case UBYTE: - return 1; - case SHORT: - return 2; - case USHORT: - return 3; - case INT: - return 4; - case UINT: - return 5; - case LONG: - return 6; - case ULONG: - return 7; - case FLOAT: - return 8; - case DOUBLE: - return 9; - default: - return -1; - } - } - - /** - * Returns the data type that is the largest among the arguments. Relative sizes of data types are determined via - * {@link #rank(DataType)}. - * - * @param dataTypes an array of numeric data types. - * @return the data type that is the largest among the arguments. - */ - public static DataType largestOf(DataType... dataTypes) { - DataType widest = null; - for (DataType dataType : dataTypes) { - if (widest == null) { - widest = dataType; - } else if (rank(dataType) > rank(widest)) { - widest = dataType; - } - } - return widest; - } - - /** - * Returns the smallest numeric data type that: - *

      - *
    1. can hold a larger integer than {@code dataType} can
      2. - *
    2. if integral, has the same signedness as {@code dataType}
      3. - *
    - * The relative sizes of data types are determined in a manner consistent with {@link #rank(DataType)}. - *

    - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
    ArgumentResult
    BYTESHORT
    UBYTEUSHORT
    SHORTINT
    USHORTUINT
    INTLONG
    UINTULONG
    LONGDOUBLE
    ULONGDOUBLE
    Any other data typeJust return argument
    - *

    - * The returned type is intended to be just big enough to hold the result of performing an unsigned conversion of a - * value of the smaller type. For example, the {@code byte} value {@code -106} equals {@code 150} when interpreted - * as unsigned. That won't fit in a (signed) {@code byte}, but it will fit in a {@code short}. - * - * @param dataType an integral data type. - * @return the next larger type. - */ - public static DataType nextLarger(DataType dataType) { - switch (dataType) { - case BYTE: - return SHORT; - case UBYTE: - return USHORT; - case SHORT: - return INT; - case USHORT: - return UINT; - case INT: - return LONG; - case UINT: - return ULONG; - case LONG: - case ULONG: - return DOUBLE; - default: - return dataType; - } - } - - @Override - public double getScaleFactor() { - return scale; - } - - @Override - public double getOffset() { - return offset; - } - - @Override - public Signedness getSignedness() { - return signedness; - } - - @Override - public DataType getScaledOffsetType() { - return scaledOffsetType; - } - - @Nonnull - @Override - public DataType getUnsignedConversionType() { - return unsignedConversionType; - } - - @Override - public boolean hasValidData() { - return hasValidRange || hasValidMin || hasValidMax; - } - - @Override - public double getValidMin() { - return validMin; - } - - @Override - public double getValidMax() { - return validMax; - } - - @Override - public boolean isInvalidData(double val) { - // valid_min and valid_max may have been multiplied by scale_factor, which could be a float, not a double. - // That potential loss of precision means that we cannot do the nearlyEquals() comparison with - // Misc.defaultMaxRelativeDiffDouble. - boolean greaterThanOrEqualToValidMin = - Misc.nearlyEquals(val, validMin, Misc.defaultMaxRelativeDiffFloat) || val > validMin; - boolean lessThanOrEqualToValidMax = - Misc.nearlyEquals(val, validMax, Misc.defaultMaxRelativeDiffFloat) || val < validMax; - - return (hasValidRange && !(greaterThanOrEqualToValidMin && lessThanOrEqualToValidMax)) - || (hasValidMin && !greaterThanOrEqualToValidMin) || (hasValidMax && !lessThanOrEqualToValidMax); - } - - @Override - public boolean hasFillValue() { - return hasFillValue; - } - - @Override - public boolean isFillValue(double val) { - return hasFillValue && Misc.nearlyEquals(val, fillValue, Misc.defaultMaxRelativeDiffFloat); - } - - @Override - public double getFillValue() { - return fillValue; - } - - @Override - public boolean hasScaleOffset() { - return useScaleOffset; - } - - @Override - public boolean hasMissingValue() { - return hasMissingValue; - } - - @Override - public boolean isMissingValue(double val) { - if (!hasMissingValue) { - return false; - } - for (double aMissingValue : missingValue) { - if (Misc.nearlyEquals(val, aMissingValue, Misc.defaultMaxRelativeDiffFloat)) { - return true; - } - } - return false; - } - - @Override - public double[] getMissingValues() { - return missingValue; - } - - @Override - public void setFillValueIsMissing(boolean b) { - this.fillValueIsMissing = b; - } - - @Override - public void setInvalidDataIsMissing(boolean b) { - this.invalidDataIsMissing = b; - } - - @Override - public void setMissingDataIsMissing(boolean b) { - this.missingDataIsMissing = b; - } - - @Override - public boolean hasMissing() { - return (invalidDataIsMissing && hasValidData()) || (fillValueIsMissing && hasFillValue()) - || (missingDataIsMissing && hasMissingValue()); - } - - @Override - public boolean isMissing(double val) { - if (Double.isNaN(val)) { - return true; - } else { - return (missingDataIsMissing && isMissingValue(val)) || (fillValueIsMissing && isFillValue(val)) - || (invalidDataIsMissing && isInvalidData(val)); - } - } - - - @Override - public Number convertUnsigned(Number value) { - return convertUnsigned(value, signedness); - } - - private static Number convertUnsigned(Number value, DataType dataType) { - return convertUnsigned(value, dataType.getSignedness()); - } - - private static Number convertUnsigned(Number value, Signedness signedness) { - if (signedness == Signedness.UNSIGNED) { - // Handle integral types that should be treated as unsigned by widening them if necessary. - return DataType.widenNumberIfNegative(value); - } else { - return value; - } - } - - @Override - public Array convertUnsigned(Array in) { - return convert(in, true, false, false); - } - - @Override - public double applyScaleOffset(Number value) { - double convertedValue = value.doubleValue(); - return useScaleOffset ? scale * convertedValue + offset : convertedValue; - } - - @Override - public Array applyScaleOffset(Array in) { - return convert(in, false, true, false); - } - - @Override - public Number convertMissing(Number value) { - return isMissing(value.doubleValue()) ? Double.NaN : value; - } - - @Override - public Array convertMissing(Array in) { - return convert(in, false, false, true); - } - - @Override - public Array convert(Array in, boolean convertUnsigned, boolean applyScaleOffset, boolean convertMissing) { - if (!in.getDataType().isNumeric() || (!convertUnsigned && !applyScaleOffset && !convertMissing)) { - return in; // Nothing to do! - } - - if (getSignedness() == Signedness.SIGNED) { - convertUnsigned = false; - } - if (!hasScaleOffset()) { - applyScaleOffset = false; - } - - DataType outType = origDataType; - if (convertUnsigned) { - outType = getUnsignedConversionType(); - } - if (applyScaleOffset) { - outType = getScaledOffsetType(); - } - - if (outType != DataType.FLOAT && outType != DataType.DOUBLE) { - convertMissing = false; - } - - Array out = Array.factory(outType, in.getShape()); - IndexIterator iterIn = in.getIndexIterator(); - IndexIterator iterOut = out.getIndexIterator(); - - while (iterIn.hasNext()) { - Number value = (Number) iterIn.getObjectNext(); - - if (convertUnsigned) { - value = convertUnsigned(value); - } - if (applyScaleOffset) { - value = applyScaleOffset(value); - } - if (convertMissing) { - value = convertMissing(value); - } - - iterOut.setObjectNext(value); - } - - return out; - } -} diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index 362c91d22d..44bc84b1e0 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -233,22 +233,21 @@ public void enhance(Set enhancements) { if (this.enhanceMode.contains(Enhance.ConvertEnums) && dataType.isEnum()) { setDataType(DataType.STRING); // LOOK promote data type to STRING ???? - return; // We can return here, because the other conversions don't apply to STRING. } // Initialize EnhanceScaleMissingUnsignedImpl. We can't do this in the constructors because this object may not // contain all of the relevant attributes at that time. NcMLReader is an example of this: the VariableDS is // constructed first, and then Attributes are added to it later. - this.scaleMissingUnsignedProxy = new EnhanceScaleMissingUnsignedImpl(this, this.enhanceMode); - - if (this.enhanceMode.contains(Enhance.ConvertUnsigned)) { - // We may need a larger data type to hold the results of the unsigned conversion. - setDataType(scaleMissingUnsignedProxy.getUnsignedConversionType()); + if (this.enhanceMode.contains(Enhance.ConvertUnsigned) && !dataType.isString()) { + this.unsignedConversion = UnsignedConversion.createFromVar(this); + this.dataType = unsignedConversion.getOutType(); } - - if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR) - && scaleMissingUnsignedProxy.hasScaleOffset()) { - setDataType(scaleMissingUnsignedProxy.getScaledOffsetType()); + if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR)) { + this.scaleOffset = ScaleOffset.createFromVariable(this); + this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType; + } + if (this.enhanceMode.contains(Enhance.ConvertMissing)) { + this.convertMissing = ConvertMissing.createFromVariable(this); } } @@ -272,13 +271,13 @@ Array convert(Array data, Set enhancements) { if (this.isVariableLength) { return data; } - if (enhancements.contains(Enhance.ConvertUnsigned)) { + if (enhancements.contains(Enhance.ConvertUnsigned) && unsignedConversion != null) { data = unsignedConversion.convertUnsigned(data); } if (enhancements.contains(Enhance.ApplyScaleOffset) && scaleOffset != null) { data = scaleOffset.removeScaleOffset(data); } - if (enhancements.contains(Enhance.ConvertMissing)) { + if (enhancements.contains(Enhance.ConvertMissing) && convertMissing != null) { data = convertMissing.convertMissing(data); } return data; @@ -543,7 +542,7 @@ public Array getMissingDataArray(int[] shape) { } Array array = Array.factoryConstant(getDataType(), shape, storage); - if (convertMissing.hasFillValue()) { + if (convertMissing != null && convertMissing.hasFillValue()) { array.setObject(0, convertMissing.getFillValue()); } return array; @@ -555,20 +554,23 @@ public Array getMissingDataArray(int[] shape) { * @param f put info here */ public void showScaleMissingProxy(Formatter f) { - f.format("has missing = %s%n", convertMissing.hasMissing()); - if (convertMissing.hasMissing()) { - if (convertMissing.hasMissingValue()) { - f.format(" missing value(s) = "); - for (double d : convertMissing.getMissingValues()) - f.format(" %f", d); - f.format("%n"); + f.format("has missing = %s%n", convertMissing != null); + if (convertMissing != null) { + if (convertMissing.hasMissing()) { + if (convertMissing.hasMissingValue()) { + f.format(" missing value(s) = "); + for (double d : convertMissing.getMissingValues()) + f.format(" %f", d); + f.format("%n"); + } + if (convertMissing.hasFillValue()) + f.format(" fillValue = %f%n", convertMissing.getFillValue()); + if (convertMissing.hasValidData()) + f.format(" valid min/max = [%f,%f]%n", convertMissing.getValidMin(), convertMissing.getValidMax()); } - if (convertMissing.hasFillValue()) - f.format(" fillValue = %f%n", convertMissing.getFillValue()); - if (convertMissing.hasValidData()) - f.format(" valid min/max = [%f,%f]%n", convertMissing.getValidMin(), convertMissing.getValidMax()); + f.format("FillValue or default = %s%n", convertMissing.getFillValue()); } - f.format("FillValue or default = %s%n", convertMissing.getFillValue()); + f.format("%nhas scale/offset = %s%n", scaleOffset != null); if (scaleOffset != null) { @@ -637,77 +639,83 @@ public double getOffset() { @Override public boolean hasMissing() { - // TODO: null or all false values? - return convertMissing.hasMissing(); + return convertMissing != null ? convertMissing.hasMissing() : false; } @Override public boolean isMissing(double val) { - return convertMissing.isMissing(val); + if (Double.isNaN(val)) { + return true; + } + return convertMissing != null ? convertMissing.isMissing(val) : false; } @Override public boolean hasValidData() { - return convertMissing.hasMissingValue(); + return convertMissing != null ? convertMissing.hasMissingValue() : false; } @Override public double getValidMin() { - return convertMissing.getValidMin(); + return convertMissing != null ? convertMissing.getValidMin() : -Double.MAX_VALUE; } @Override public double getValidMax() { - return convertMissing.getValidMax(); + return convertMissing != null ? convertMissing.getValidMax() : Double.MAX_VALUE; } @Override public boolean isInvalidData(double val) { - return convertMissing.isInvalidData(val); + return convertMissing != null ? convertMissing.isInvalidData(val) : false; } @Override public boolean hasFillValue() { - return convertMissing.hasFillValue(); + return convertMissing != null ? convertMissing.hasFillValue() : false; } @Override public double getFillValue() { - return convertMissing.getFillValue(); + return convertMissing != null ? convertMissing.getFillValue() : Double.MAX_VALUE; } @Override public boolean isFillValue(double val) { - return convertMissing.isFillValue(val); + return convertMissing != null ? convertMissing.isFillValue(val) : false; } @Override public boolean hasMissingValue() { - return convertMissing.hasMissingValue(); + return convertMissing != null ? convertMissing.hasMissingValue() : false; } @Override public double[] getMissingValues() { - return convertMissing.getMissingValues(); + return convertMissing != null ? convertMissing.getMissingValues() : new double[] {0}; } @Override public boolean isMissingValue(double val) { - return convertMissing.isMissingValue(val); + return convertMissing != null ? convertMissing.isMissingValue(val) : false; } /** @deprecated Use NetcdfDataset.builder() */ @Deprecated @Override public void setFillValueIsMissing(boolean b) { - scaleMissingUnsignedProxy.setFillValueIsMissing(b); + if (convertMissing != null) { + convertMissing.setFillValueIsMissing(b); + } } /** @deprecated Use NetcdfDataset.builder() */ @Deprecated @Override public void setInvalidDataIsMissing(boolean b) { - scaleMissingUnsignedProxy.setInvalidDataIsMissing(b); + if (convertMissing != null) { + convertMissing.setInvalidDataIsMissing(b); + } } public boolean missingDataIsMissing() { @@ -726,7 +734,9 @@ public boolean invalidDataIsMissing() { @Deprecated @Override public void setMissingDataIsMissing(boolean b) { - scaleMissingUnsignedProxy.setMissingDataIsMissing(b); + if (convertMissing != null) { + convertMissing.setMissingDataIsMissing(b); + } } @Nullable @@ -767,12 +777,12 @@ public Array convertUnsigned(Array in) { @Override public Number convertMissing(Number value) { - return convertMissing.convertMissing(value); + return convertMissing != null ? convertMissing.convertMissing(value) : value; } @Override public Array convertMissing(Array in) { - return convertMissing.convertMissing(in); + return convertMissing != null ? convertMissing.convertMissing(in) : in; } /** @@ -781,14 +791,19 @@ public Array convertMissing(Array in) { @Override @Deprecated public Array convert(Array in, boolean convertUnsigned, boolean applyScaleOffset, boolean convertMissing) { - return scaleMissingUnsignedProxy.convert(in, convertUnsigned, applyScaleOffset, convertMissing); + if (this.unsignedConversion != null) { + in = unsignedConversion.convertUnsigned(in); + } + if (this.scaleOffset != null) { + in = scaleOffset.removeScaleOffset(in); + } + return convertMissing(in); } //////////////////////////////////////////////////////////////////////////////////////////// // TODO remove in version 6. private EnhancementsImpl enhanceProxy; private List coordSysNames; - private EnhanceScaleMissingUnsignedImpl scaleMissingUnsignedProxy = new EnhanceScaleMissingUnsignedImpl(); // TODO make immutable in version 6 private UnsignedConversion unsignedConversion; @@ -823,18 +838,18 @@ protected VariableDS(Builder builder, Group parentGroup) { if (this.enhanceMode.contains(Enhance.ConvertEnums) && dataType.isEnum()) { this.dataType = DataType.STRING; // LOOK promote enum data type to STRING ???? - } else { - if (this.enhanceMode.contains(Enhance.ConvertUnsigned) && !dataType.isEnum()) { - this.unsignedConversion = UnsignedConversion.createFromVar(this); - this.dataType = unsignedConversion.getOutType(); - } - if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR)) { - this.scaleOffset = ScaleOffset.createFromVariable(this); - this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType; - } - if (this.enhanceMode.contains(Enhance.ConvertMissing)) { - this.convertMissing = ConvertMissing.createFromVariable(this); - } + } + + if (this.enhanceMode.contains(Enhance.ConvertUnsigned) && !dataType.isString()) { + this.unsignedConversion = UnsignedConversion.createFromVar(this); + this.dataType = unsignedConversion.getOutType(); + } + if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR)) { + this.scaleOffset = ScaleOffset.createFromVariable(this); + this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType; + } + if (this.enhanceMode.contains(Enhance.ConvertMissing)) { + this.convertMissing = ConvertMissing.createFromVariable(this); } // We have to complete this after the NetcdfDataset is built. diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java index 1a52e75564..602becfc25 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java @@ -10,8 +10,6 @@ import ucar.nc2.iosp.netcdf3.N3iosp; import ucar.nc2.util.Misc; -import java.util.Arrays; - public class ConvertMissing { private boolean hasValidMin, hasValidMax; @@ -243,12 +241,32 @@ public boolean isMissing(double val) { } } + @Deprecated + public void setFillValueIsMissing(boolean b) { + this.fillValueIsMissing = b; + } + + @Deprecated + public void setInvalidDataIsMissing(boolean b) { + this.invalidDataIsMissing = b; + } + + @Deprecated + public void setMissingDataIsMissing(boolean b) { + this.missingDataIsMissing = b; + } + public Number convertMissing(Number value) { return isMissing(value.doubleValue()) ? Double.NaN : value; } public Array convertMissing(Array in) { - Array out = Array.factory(in.getDataType(), in.getShape()); + DataType type = in.getDataType(); + if (!type.isNumeric()) { + return in; + } + + Array out = Array.factory(type, in.getShape()); IndexIterator iterIn = in.getIndexIterator(); IndexIterator iterOut = out.getIndexIterator(); diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java index dcfefb94d4..acca827b7d 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java @@ -77,7 +77,7 @@ public static ScaleOffset createFromVariable(VariableDS var) { Attribute scaleAtt = var.findAttribute(CDM.SCALE_FACTOR); if (scaleAtt != null && !scaleAtt.isString()) { scaleType = FilterHelpers.getAttributeDataType(scaleAtt, signedness); - scale = var.convertUnsigned(scaleAtt.getNumericValue()).doubleValue(); + scale = 1 / var.convertUnsigned(scaleAtt.getNumericValue()).doubleValue(); } Attribute offsetAtt = var.findAttribute(CDM.ADD_OFFSET); @@ -92,8 +92,8 @@ public static ScaleOffset createFromVariable(VariableDS var) { Map scaleOffsetProps = new HashMap<>(); scaleOffsetProps.put(ScaleOffset.Keys.OFFSET_KEY, offset); scaleOffsetProps.put(ScaleOffset.Keys.SCALE_KEY, scale); - scaleOffsetProps.put(ScaleOffset.Keys.DTYPE_KEY, origDataType); - scaleOffsetProps.put(ScaleOffset.Keys.ASTYPE_KEY, scaledOffsetType); + scaleOffsetProps.put(ScaleOffset.Keys.DTYPE_KEY, scaledOffsetType); + scaleOffsetProps.put(ScaleOffset.Keys.ASTYPE_KEY, origDataType); return new ScaleOffset(scaleOffsetProps); } return null; @@ -154,26 +154,32 @@ public double getOffset() { } public DataType getScaledOffsetType() { - return this.astype; + return this.dtype; } @Override public byte[] encode(byte[] dataIn) { - if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return dataIn; } + if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { + return dataIn; + } Array out = applyScaleOffset(FilterHelpers.bytesToArray(dataIn, dtype, dtypeOrder)); return FilterHelpers.arrayToBytes(out, astype, astypeOrder); } @Override public byte[] decode(byte[] dataIn) { - if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return dataIn; } + if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { + return dataIn; + } Array out = removeScaleOffset(FilterHelpers.bytesToArray(dataIn, astype, astypeOrder)); return FilterHelpers.arrayToBytes(out, dtype, dtypeOrder); } // not used anywhere yet public Array applyScaleOffset(Array in) { - if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return in; } + if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { + return in; + } // use wider datatype if unsigned DataType outType = astype; if (astype.getSignedness() == Signedness.UNSIGNED) { @@ -197,7 +203,9 @@ public Array applyScaleOffset(Array in) { } public Array removeScaleOffset(Array in) { - if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return in; } + if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { + return in; + } // use wider datatype if unsigned DataType outType = dtype; if (dtype.getSignedness() == Signedness.UNSIGNED) { diff --git a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java index 8c1da9ea3a..8d71981736 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java @@ -3,7 +3,6 @@ import ucar.ma2.Array; import ucar.ma2.DataType; import ucar.ma2.IndexIterator; -import ucar.nc2.Variable; import ucar.nc2.constants.CDM; import ucar.nc2.dataset.VariableDS; @@ -47,7 +46,6 @@ public DataType getOutType() { } public Number convertUnsigned(Number value) { - Number newVal = DataType.widenNumberIfNegative(value); return DataType.widenNumberIfNegative(value); } diff --git a/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java index e54256d4a2..b141629128 100644 --- a/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java +++ b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java @@ -8,7 +8,6 @@ import ucar.ma2.DataType; import ucar.ma2.InvalidRangeException; import ucar.nc2.Attribute; -import ucar.nc2.Dimension; import ucar.nc2.Variable; import ucar.nc2.constants.CDM; import ucar.nc2.dataset.NetcdfDataset; @@ -21,128 +20,125 @@ public class TestEnhancements { - private static NetcdfDataset ncd; - - private static short[] signedShorts = new short[] {123, 124, 125, 126, 127, -128, -127, -126, -125, -124}; - - private static final float VALID_MIN = 100; - private static final float VALID_MAX = 200; - private static final float FILL_VALUE = 150; - private static float[] missingData = new float[]{90, 100, Float.NaN, 120, 130, 140, 150, 190, 200, 210}; - - private static final Short SIGNED_SCALED_MAX = -126; - private static final Short SIGNED_SCALED_FILL_VALUE = -128; - - @ClassRule - public static final TemporaryFolder tempFolder = new TemporaryFolder(); - - @BeforeClass - public static void setUp() throws IOException , InvalidRangeException { - final int data_len = 10; - String filePath = tempFolder.newFile().getAbsolutePath(); - NetcdfFormatWriter.Builder builder = NetcdfFormatWriter.createNewNetcdf3(filePath); - Dimension dim = builder.addDimension("dim", data_len); - - Array signedData = Array.factory(DataType.SHORT, new int[]{data_len}, signedShorts); - // signed shorts - builder.addVariable("signedVar", DataType.SHORT, "dim"); - // unsigned shorts - builder.addVariable("unsignedVar", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.UNSIGNED, "true")); - - // scaled and offset data - builder.addVariable("scaleOffsetVar", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.SCALE_FACTOR, .1)) - .addAttribute(new Attribute(CDM.ADD_OFFSET, 10)); - - Array missingDataArray = Array.factory(DataType.FLOAT, new int[]{data_len}, missingData); - // Data with min - builder.addVariable("validMin", DataType.FLOAT, "dim").addAttribute(new Attribute(CDM.VALID_MIN, VALID_MIN)); - // Data with min and max - builder.addVariable("validMinMax", DataType.FLOAT, "dim") - .addAttribute(new Attribute(CDM.VALID_MIN, VALID_MIN)) - .addAttribute(new Attribute(CDM.VALID_MAX, VALID_MAX)); - // Data with range and fill value - Array range = Array.factory(DataType.FLOAT, new int[]{2}, new float[]{VALID_MIN, VALID_MAX}); - builder.addVariable("validRange", DataType.FLOAT, "dim") - .addAttribute(Attribute.builder(CDM.VALID_RANGE).setValues(range).build()) - .addAttribute(Attribute.builder(CDM.FILL_VALUE).setNumericValue(FILL_VALUE, true).build()); - - // unsigned, scaled/offset, and missing value - Array enhanceAllArray = Array.factory(DataType.SHORT, new int[]{data_len}, signedShorts); - builder.addVariable("enhanceAll", DataType.SHORT, "dim") - .addAttribute(new Attribute(CDM.UNSIGNED, "true")) - .addAttribute(new Attribute(CDM.SCALE_FACTOR, .1)) - .addAttribute(new Attribute(CDM.ADD_OFFSET, 10)) - .addAttribute(new Attribute(CDM.VALID_MAX, SIGNED_SCALED_MAX)) - .addAttribute(new Attribute(CDM.FILL_VALUE, SIGNED_SCALED_FILL_VALUE)); - - // write data - NetcdfFormatWriter writer = builder.build(); - writer.write(writer.findVariable("signedVar"), new int[1], signedData); - writer.write(writer.findVariable("unsignedVar"), new int[1], signedData); - writer.write(writer.findVariable("scaleOffsetVar"), new int[1], signedData); - writer.write(writer.findVariable("validMin"), new int[1], missingDataArray); - writer.write(writer.findVariable("validMinMax"), new int[1], missingDataArray); - writer.write(writer.findVariable("validRange"), new int[1], missingDataArray); - writer.write(writer.findVariable("enhanceAll"), new int[1], enhanceAllArray); - writer.close(); - ncd = NetcdfDatasets.openDataset(filePath); - } - - @Test - public void testUnsignedConversion() throws IOException { - final int[] unsignedValues = new int[]{123, 124, 125, 126, 127, -128, -127, -126, -125, -126}; - // signed var - Variable v = ncd.findVariable("signedVar"); - Array data = v.read(); - assertThat(data.isUnsigned()).isFalse(); - assertThat(data.getDataType()).isEqualTo(DataType.SHORT); - assertThat((short[])data.copyTo1DJavaArray()).isEqualTo(signedShorts); - - // var with unsigned data type - v = ncd.findVariable("unsignedVar"); - data = v.read(); - assertThat(data.isUnsigned()).isTrue(); - assertThat(data.getDataType()).isEqualTo(DataType.UINT); - assertThat((int[])data.copyTo1DJavaArray()).isEqualTo(unsignedValues); - } - - @Test - public void testScaleOffset() throws IOException { - final double[] expected = new double[]{1240, 1250, 1260, 1270, 1280, -1270, -1260, -1250, -1240, -1230}; - // signed var - Variable v = ncd.findVariable("scaleOffsetVar"); - Array data = v.read(); - assertThat(data.isUnsigned()).isFalse(); - assertThat(data.getDataType()).isEqualTo(DataType.SHORT); - assertThat((double[])data.copyTo1DJavaArray()).isEqualTo(expected); - } - - @Test - public void testConvertMissing() throws IOException { - // var with valid min - float[] expected = new float[]{Float.NaN, 100, Float.NaN, 120, 130, 140, 150, 190, 200, 210}; - Variable v = ncd.findVariable("validMin"); - Array data = v.read(); - assertThat((float[])data.copyTo1DJavaArray()).isEqualTo(expected); - - // var with valid min and max - expected = new float[]{Float.NaN, 100, Float.NaN, 120, 130, 140, 150, 190, 200, Float.NaN}; - v = ncd.findVariable("validMinMax"); - data = v.read(); - assertThat((float[])data.copyTo1DJavaArray()).isEqualTo(expected); - - // var with valid range and fill value - expected = new float[]{Float.NaN, 100, Float.NaN, 120, 130, 140, Float.NaN, 190, 200, Float.NaN}; - v = ncd.findVariable("validRange"); - data = v.read(); - assertThat((float[])data.copyTo1DJavaArray()).isEqualTo(expected); - } - - @Test - public void testCombinedEnhancements() throws IOException { - int[] expected = new int[]{1240, 1250, 1260, 1270, 1280, 0, 1278, 1260, 0, 0}; - Variable v = ncd.findVariable("enhanceAll"); - Array data = v.read(); - assertThat((int[])data.copyTo1DJavaArray()).isEqualTo(expected); - } + private static NetcdfDataset ncd; + + private static short[] signedShorts = new short[] {-5, -4, -3, -2, -1, 0, 1, 2, 3, 4}; + + private static final float VALID_MIN = 100; + private static final float VALID_MAX = 200; + private static final float FILL_VALUE = 150; + private static float[] missingData = new float[] {90, 100, Float.NaN, 120, 130, 140, 150, 190, 200, 210}; + + private static final Short SIGNED_SCALED_MAX = -2; + private static final Short SIGNED_SCALED_FILL_VALUE = -4; + + @ClassRule + public static final TemporaryFolder tempFolder = new TemporaryFolder(); + + @BeforeClass + public static void setUp() throws IOException, InvalidRangeException { + final int data_len = 10; + String filePath = tempFolder.newFile().getAbsolutePath(); + NetcdfFormatWriter.Builder builder = NetcdfFormatWriter.createNewNetcdf3(filePath); + builder.addDimension("dim", data_len); + + Array signedData = Array.factory(DataType.SHORT, new int[] {data_len}, signedShorts); + // signed shorts + builder.addVariable("signedVar", DataType.SHORT, "dim"); + // unsigned shorts + builder.addVariable("unsignedVar", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.UNSIGNED, "true")); + + // scaled and offset data + builder.addVariable("scaleOffsetVar", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.SCALE_FACTOR, 10)) + .addAttribute(new Attribute(CDM.ADD_OFFSET, 10)); + + Array missingDataArray = Array.factory(DataType.FLOAT, new int[] {data_len}, missingData); + // Data with min + builder.addVariable("validMin", DataType.FLOAT, "dim").addAttribute(new Attribute(CDM.VALID_MIN, VALID_MIN)); + // Data with min and max + builder.addVariable("validMinMax", DataType.FLOAT, "dim").addAttribute(new Attribute(CDM.VALID_MIN, VALID_MIN)) + .addAttribute(new Attribute(CDM.VALID_MAX, VALID_MAX)); + // Data with range and fill value + Array range = Array.factory(DataType.FLOAT, new int[] {2}, new float[] {VALID_MIN, VALID_MAX}); + builder.addVariable("validRange", DataType.FLOAT, "dim") + .addAttribute(Attribute.builder(CDM.VALID_RANGE).setValues(range).build()) + .addAttribute(Attribute.builder(CDM.FILL_VALUE).setNumericValue(FILL_VALUE, true).build()); + + // unsigned, scaled/offset, and missing value + Array enhanceAllArray = Array.factory(DataType.SHORT, new int[] {data_len}, signedShorts); + builder.addVariable("enhanceAll", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.UNSIGNED, "true")) + .addAttribute(new Attribute(CDM.SCALE_FACTOR, 10)).addAttribute(new Attribute(CDM.ADD_OFFSET, 10)) + .addAttribute(new Attribute(CDM.VALID_MAX, SIGNED_SCALED_MAX)) + .addAttribute(new Attribute(CDM.FILL_VALUE, SIGNED_SCALED_FILL_VALUE)); + + // write data + NetcdfFormatWriter writer = builder.build(); + writer.write(writer.findVariable("signedVar"), new int[1], signedData); + writer.write(writer.findVariable("unsignedVar"), new int[1], signedData); + writer.write(writer.findVariable("scaleOffsetVar"), new int[1], signedData); + writer.write(writer.findVariable("validMin"), new int[1], missingDataArray); + writer.write(writer.findVariable("validMinMax"), new int[1], missingDataArray); + writer.write(writer.findVariable("validRange"), new int[1], missingDataArray); + writer.write(writer.findVariable("enhanceAll"), new int[1], enhanceAllArray); + writer.close(); + ncd = NetcdfDatasets.openDataset(filePath); + } + + @Test + public void testUnsignedConversion() throws IOException { + final int[] unsignedValues = new int[] {65531, 65532, 65533, 65534, 65535, 0, 1, 2, 3, 4}; + // signed var + Variable v = ncd.findVariable("signedVar"); + Array data = v.read(); + assertThat(data.isUnsigned()).isFalse(); + assertThat(data.getDataType()).isEqualTo(DataType.SHORT); + assertThat((short[]) data.copyTo1DJavaArray()).isEqualTo(signedShorts); + + // var with unsigned data type + v = ncd.findVariable("unsignedVar"); + data = v.read(); + assertThat(data.isUnsigned()).isTrue(); + assertThat(data.getDataType()).isEqualTo(DataType.UINT); + assertThat((int[]) data.copyTo1DJavaArray()).isEqualTo(unsignedValues); + } + + @Test + public void testScaleOffset() throws IOException { + final int[] expected = new int[] {-40, -30, -20, -10, 0, 10, 20, 30, 40, 50}; + // signed var + Variable v = ncd.findVariable("scaleOffsetVar"); + Array data = v.read(); + assertThat(data.isUnsigned()).isFalse(); + assertThat(data.getDataType()).isEqualTo(DataType.INT); + assertThat((int[]) data.copyTo1DJavaArray()).isEqualTo(expected); + } + + @Test + public void testConvertMissing() throws IOException { + // var with valid min + float[] expected = new float[] {Float.NaN, 100, Float.NaN, 120, 130, 140, 150, 190, 200, 210}; + Variable v = ncd.findVariable("validMin"); + Array data = v.read(); + assertThat((float[]) data.copyTo1DJavaArray()).isEqualTo(expected); + + // var with valid min and max + expected = new float[] {Float.NaN, 100, Float.NaN, 120, 130, 140, 150, 190, 200, Float.NaN}; + v = ncd.findVariable("validMinMax"); + data = v.read(); + assertThat((float[]) data.copyTo1DJavaArray()).isEqualTo(expected); + + // var with valid range and fill value + expected = new float[] {Float.NaN, 100, Float.NaN, 120, 130, 140, Float.NaN, 190, 200, Float.NaN}; + v = ncd.findVariable("validRange"); + data = v.read(); + assertThat((float[]) data.copyTo1DJavaArray()).isEqualTo(expected); + } + + @Test + public void testCombinedEnhancements() throws IOException { + int[] expected = new int[] {655320, 0, 655340, 655350, 0, 10, 20, 30, 40, 50}; + Variable v = ncd.findVariable("enhanceAll"); + Array data = v.read(); + assertThat((int[]) data.copyTo1DJavaArray()).isEqualTo(expected); + } } diff --git a/cdm/misc/src/test/java/ucar/nc2/TestSequence.java b/cdm/misc/src/test/java/ucar/nc2/TestSequence.java index 41093ce757..2e1aa5ecd3 100644 --- a/cdm/misc/src/test/java/ucar/nc2/TestSequence.java +++ b/cdm/misc/src/test/java/ucar/nc2/TestSequence.java @@ -6,30 +6,19 @@ package ucar.nc2; import com.google.common.collect.Sets; -import java.io.File; import java.io.IOException; -import java.lang.invoke.MethodHandles; import java.util.Arrays; import java.util.List; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import ucar.ma2.Array; -import ucar.ma2.ArrayFloat; -import ucar.ma2.ArraySequence; -import ucar.ma2.ArrayStructure; -import ucar.ma2.MAMath; import ucar.ma2.StructureData; import ucar.ma2.StructureDataIterator; -import ucar.nc2.dataset.NetcdfDatasets; import ucar.unidata.util.test.TestDir; import ucar.unidata.util.test.category.NeedsCdmUnitTest; /** Test Sequences constructed when reading NLDN datasets. */ public class TestSequence { - private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @Test @Category(NeedsCdmUnitTest.class) From bd4124cf4822b853080b0853b8a96c6a4863007d Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Fri, 28 Apr 2023 15:39:50 -0700 Subject: [PATCH 05/24] apply fill value without using enhance modes --- .../java/ucar/nc2/dataset/VariableDS.java | 56 +++++++++++++++++-- .../java/ucar/nc2/filter/ConvertMissing.java | 25 +-------- .../test/java/ucar/nc2/ncml/TestEnhance.java | 4 -- 3 files changed, 55 insertions(+), 30 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index 44bc84b1e0..30a6f1f4e7 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -9,12 +9,14 @@ import ucar.ma2.*; import ucar.nc2.*; import ucar.nc2.constants.CDM; +import ucar.nc2.constants.DataFormatType; import ucar.nc2.dataset.NetcdfDataset.Enhance; import ucar.nc2.filter.ConvertMissing; import ucar.nc2.filter.FilterHelpers; import ucar.nc2.filter.ScaleOffset; import ucar.nc2.filter.UnsignedConversion; import ucar.nc2.internal.dataset.CoordinatesHelper; +import ucar.nc2.iosp.netcdf3.N3iosp; import ucar.nc2.util.CancelTask; import javax.annotation.Nullable; import java.io.IOException; @@ -170,6 +172,8 @@ protected VariableDS(VariableDS vds, boolean isCopy) { this.unsignedConversion = vds.unsignedConversion; this.scaleOffset = vds.scaleOffset; this.convertMissing = vds.convertMissing; + this.fillValue = vds.getFillValue(); + this.hasFillValue = vds.hasFillValue(); // Add this so that old VariableDS units agrees with new VariableDS units. String units = vds.getUnitsString(); @@ -242,6 +246,26 @@ public void enhance(Set enhancements) { this.unsignedConversion = UnsignedConversion.createFromVar(this); this.dataType = unsignedConversion.getOutType(); } + // need fill value info before convertMissing + Attribute fillValueAtt = findAttribute(CDM.FILL_VALUE); + if (fillValueAtt != null && !fillValueAtt.isString()) { + fillValue = convertUnsigned(fillValueAtt.getNumericValue()).doubleValue(); + fillValue = applyScaleOffset(fillValue); // This will fail when _FillValue is CHAR. + hasFillValue = true; + } else { + // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value. + String fileTypeId = getFileTypeId(); + boolean isNetcdfIosp = DataFormatType.NETCDF.getDescription().equals(fileTypeId) + || DataFormatType.NETCDF4.getDescription().equals(fileTypeId); + + if (isNetcdfIosp) { + DataType unsignedConversionType = getUnsignedConversionType(); + if (unsignedConversionType.isNumeric()) { + fillValue = applyScaleOffset(N3iosp.getFillValueDefault(unsignedConversionType)); + hasFillValue = true; + } + } + } if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR)) { this.scaleOffset = ScaleOffset.createFromVariable(this); this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType; @@ -542,8 +566,8 @@ public Array getMissingDataArray(int[] shape) { } Array array = Array.factoryConstant(getDataType(), shape, storage); - if (convertMissing != null && convertMissing.hasFillValue()) { - array.setObject(0, convertMissing.getFillValue()); + if (hasFillValue) { + array.setObject(0, fillValue); } return array; } @@ -672,12 +696,12 @@ public boolean isInvalidData(double val) { @Override public boolean hasFillValue() { - return convertMissing != null ? convertMissing.hasFillValue() : false; + return hasFillValue; } @Override public double getFillValue() { - return convertMissing != null ? convertMissing.getFillValue() : Double.MAX_VALUE; + return fillValue; } @Override @@ -816,6 +840,9 @@ public Array convert(Array in, boolean convertUnsigned, boolean applyScaleOffset protected String orgName; // in case Variable was renamed, and we need to keep track of the original name String orgFileTypeId; // the original fileTypeId. + private boolean hasFillValue = false; + private double fillValue = Double.MAX_VALUE; + protected VariableDS(Builder builder, Group parentGroup) { super(builder, parentGroup); @@ -848,6 +875,27 @@ protected VariableDS(Builder builder, Group parentGroup) { this.scaleOffset = ScaleOffset.createFromVariable(this); this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType; } + + // need fill value info before convertMissing + Attribute fillValueAtt = findAttribute(CDM.FILL_VALUE); + if (fillValueAtt != null && !fillValueAtt.isString()) { + fillValue = convertUnsigned(fillValueAtt.getNumericValue()).doubleValue(); + fillValue = applyScaleOffset(fillValue); // This will fail when _FillValue is CHAR. + hasFillValue = true; + } else { + // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value. + String fileTypeId = getFileTypeId(); + boolean isNetcdfIosp = DataFormatType.NETCDF.getDescription().equals(fileTypeId) + || DataFormatType.NETCDF4.getDescription().equals(fileTypeId); + + if (isNetcdfIosp) { + DataType unsignedConversionType = getUnsignedConversionType(); + if (unsignedConversionType.isNumeric()) { + fillValue = applyScaleOffset(N3iosp.getFillValueDefault(unsignedConversionType)); + hasFillValue = true; + } + } + } if (this.enhanceMode.contains(Enhance.ConvertMissing)) { this.convertMissing = ConvertMissing.createFromVariable(this); } diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java index 602becfc25..9ebeba8f2d 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java @@ -87,28 +87,9 @@ public static ConvertMissing createFromVariable(VariableDS var) { } } - /// _FillValue - double fillValue = Double.MAX_VALUE; - boolean hasFillValue = false; - Attribute fillValueAtt = var.findAttribute(CDM.FILL_VALUE); - if (fillValueAtt != null && !fillValueAtt.isString()) { - fillValue = var.convertUnsigned(fillValueAtt.getNumericValue()).doubleValue(); - fillValue = var.applyScaleOffset(fillValue); // This will fail when _FillValue is CHAR. - hasFillValue = true; - } else { - // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value. - String fileTypeId = var.getFileTypeId(); - boolean isNetcdfIosp = DataFormatType.NETCDF.getDescription().equals(fileTypeId) - || DataFormatType.NETCDF4.getDescription().equals(fileTypeId); - - if (isNetcdfIosp) { - DataType unsignedConversionType = var.getUnsignedConversionType(); - if (unsignedConversionType.isNumeric()) { - fillValue = var.applyScaleOffset(N3iosp.getFillValueDefault(unsignedConversionType)); - hasFillValue = true; - } - } - } + /// fill_value + boolean hasFillValue = var.hasFillValue(); + double fillValue = var.getFillValue(); /// missing_value double[] missingValue = null; diff --git a/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhance.java b/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhance.java index bffe5e574a..4d60a7987d 100644 --- a/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhance.java +++ b/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhance.java @@ -6,10 +6,7 @@ import static com.google.common.truth.Truth.assertThat; import java.io.IOException; -import java.lang.invoke.MethodHandles; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import ucar.ma2.DataType; import ucar.nc2.NetcdfFile; import ucar.nc2.Variable; @@ -18,7 +15,6 @@ /** Test NcmlNew enhancement */ public class TestEnhance { - private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static String dataDir = TestDir.cdmLocalTestDataDir + "ncml/enhance/"; @Test From 6e9e909150f1f26cb547d10ef05b3febae20b73d Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Mon, 3 Oct 2022 14:33:16 -0700 Subject: [PATCH 06/24] add default values for scale and offset params --- .../src/main/java/ucar/nc2/filter/ScaleOffset.java | 4 ++-- .../src/test/java/ucar/nc2/filter/TestFilters.java | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java index 4f7aad01d9..369ad2f447 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java @@ -56,8 +56,8 @@ public class ScaleOffset extends Filter { public ScaleOffset(Map properties) { // get offset and scale parameters - offset = ((Number) properties.get("offset")).doubleValue(); - scale = (int) properties.get("scale"); + offset = ((Number) properties.getOrDefault("offset", 0)).doubleValue(); + scale = (int) properties.getOrDefault("scale", 1); // input data type String type = (String) properties.get("dtype"); diff --git a/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java b/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java index d7e6943178..51c67f55bc 100644 --- a/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java +++ b/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java @@ -96,6 +96,16 @@ public void testScaleOffset() throws IOException { // test decode byte[] decoded = filter.decode(encoded); assertThat(decoded).isEqualTo(input); + + // test empty props + Map props2 = new HashMap<>(); + props2.put("id", "fixedscaleoffset"); + props2.put("dtype", " Date: Fri, 23 Dec 2022 15:55:41 -0800 Subject: [PATCH 07/24] refactor scale, missing, and unsigned implementations --- .../src/main/java/ucar/nc2/NetcdfFiles.java | 4 +- .../dataset/EnhanceScaleMissingUnsigned.java | 4 +- .../java/ucar/nc2/dataset/NetcdfDataset.java | 2 +- .../java/ucar/nc2/dataset/VariableDS.java | 151 +++++----- .../java/ucar/nc2/filter/ConvertMissing.java | 264 ++++++++++++++++++ .../src/main/java/ucar/nc2/filter/Filter.java | 3 + .../java/ucar/nc2/filter/FilterHelpers.java | 225 +++++++++++++++ .../java/ucar/nc2/filter/ScaleOffset.java | 224 +++++++-------- .../ucar/nc2/filter/UnsignedConversion.java | 68 +++++ .../nc2/internal/iosp/hdf5/H5iospNew.java | 2 +- 10 files changed, 752 insertions(+), 195 deletions(-) create mode 100644 cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java create mode 100644 cdm/core/src/main/java/ucar/nc2/filter/FilterHelpers.java create mode 100644 cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java diff --git a/cdm/core/src/main/java/ucar/nc2/NetcdfFiles.java b/cdm/core/src/main/java/ucar/nc2/NetcdfFiles.java index 9aa9c3dba2..e89ef3a94a 100644 --- a/cdm/core/src/main/java/ucar/nc2/NetcdfFiles.java +++ b/cdm/core/src/main/java/ucar/nc2/NetcdfFiles.java @@ -265,7 +265,7 @@ public static NetcdfFile open(String location, int buffer_size, ucar.nc2.util.Ca * @param buffer_size RandomAccessFile buffer size, if <= 0, use default size * @param cancelTask allow task to be cancelled; may be null. * @param iospMessage special iosp tweaking (sent before open is called), may be null - * @return NetcdfFile object, or null if cant find IOServiceProver + * @return NetcdfFile object, or null if can't find IOServiceProver * @throws IOException if error */ public static NetcdfFile open(String location, int buffer_size, ucar.nc2.util.CancelTask cancelTask, @@ -290,7 +290,7 @@ public static NetcdfFile open(String location, int buffer_size, ucar.nc2.util.Ca * @param iospMessage special iosp tweaking (sent before open is called), may be null * @return NetcdfFile object, or null if cant find IOServiceProver * @throws IOException if read error - * @throws ClassNotFoundException cannat find iospClassName in thye class path + * @throws ClassNotFoundException cannot find iospClassName in the class path * @throws InstantiationException if class cannot be instantiated * @throws IllegalAccessException if class is not accessible */ diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsigned.java b/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsigned.java index 16fc11a629..2a255435d2 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsigned.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsigned.java @@ -31,7 +31,7 @@ *

  • Values will be {@link DataType#widenNumber widened}, which effectively reinterprets signed data as unsigned * data.
    • *
  • To accommodate the unsigned conversion, the variable's data type will be changed to the - * {@link EnhanceScaleMissingUnsignedImpl#nextLarger(DataType) next larger type}.
    • + * {@link ucar.nc2.filter.FilterHelpers#nextLarger(DataType) next larger type}. * * *

    Implementation rules for scale/offset

    @@ -117,7 +117,9 @@ * * @author caron * @author cwardgar + * @deprecated use implementations in Filter package */ +@Deprecated public interface EnhanceScaleMissingUnsigned extends IsMissingEvaluator { /** true if Variable data will be converted using scale and offset */ boolean hasScaleOffset(); diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/NetcdfDataset.java b/cdm/core/src/main/java/ucar/nc2/dataset/NetcdfDataset.java index c87fd7036d..4b78cab001 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/NetcdfDataset.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/NetcdfDataset.java @@ -98,7 +98,7 @@ public enum Enhance { * Convert unsigned values to signed values. * For {@link ucar.nc2.constants.CDM#UNSIGNED} variables, reinterpret the bit patterns of any * negative values as unsigned. The result will be positive values that must be stored in a - * {@link EnhanceScaleMissingUnsignedImpl#nextLarger larger data type}. + * {@link ucar.nc2.filter.FilterHelpers#nextLarger larger data type}. */ ConvertUnsigned, /** Apply scale and offset to values, promoting the data type if needed. */ diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index cdfa726f9f..a30e5a3fc2 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -10,6 +10,10 @@ import ucar.nc2.*; import ucar.nc2.constants.CDM; import ucar.nc2.dataset.NetcdfDataset.Enhance; +import ucar.nc2.filter.ConvertMissing; +import ucar.nc2.filter.FilterHelpers; +import ucar.nc2.filter.ScaleOffset; +import ucar.nc2.filter.UnsignedConversion; import ucar.nc2.internal.dataset.CoordinatesHelper; import ucar.nc2.util.CancelTask; import javax.annotation.Nullable; @@ -163,7 +167,9 @@ protected VariableDS(VariableDS vds, boolean isCopy) { this.orgName = vds.orgName; this.enhanceProxy = new EnhancementsImpl(this); // decouple coordinate systems - this.scaleMissingUnsignedProxy = vds.scaleMissingUnsignedProxy; + this.unsignedConversion = vds.unsignedConversion; + this.scaleOffset = vds.scaleOffset; + this.convertMissing = vds.convertMissing; // Add this so that old VariableDS units agrees with new VariableDS units. String units = vds.getUnitsString(); @@ -225,16 +231,16 @@ public void enhance(Set enhancements) { setDataType(orgDataType); } - // Initialize EnhanceScaleMissingUnsignedImpl. We can't do this in the constructors because this object may not - // contain all of the relevant attributes at that time. NcMLReader is an example of this: the VariableDS is - // constructed first, and then Attributes are added to it later. - this.scaleMissingUnsignedProxy = new EnhanceScaleMissingUnsignedImpl(this, this.enhanceMode); - if (this.enhanceMode.contains(Enhance.ConvertEnums) && dataType.isEnum()) { setDataType(DataType.STRING); // LOOK promote data type to STRING ???? return; // We can return here, because the other conversions don't apply to STRING. } + // Initialize EnhanceScaleMissingUnsignedImpl. We can't do this in the constructors because this object may not + // contain all of the relevant attributes at that time. NcMLReader is an example of this: the VariableDS is + // constructed first, and then Attributes are added to it later. + this.scaleMissingUnsignedProxy = new EnhanceScaleMissingUnsignedImpl(this, this.enhanceMode); + if (this.enhanceMode.contains(Enhance.ConvertUnsigned)) { // We may need a larger data type to hold the results of the unsigned conversion. setDataType(scaleMissingUnsignedProxy.getUnsignedConversionType()); @@ -529,8 +535,8 @@ public Array getMissingDataArray(int[] shape) { } Array array = Array.factoryConstant(getDataType(), shape, storage); - if (scaleMissingUnsignedProxy.hasFillValue()) { - array.setObject(0, scaleMissingUnsignedProxy.getFillValue()); + if (convertMissing.hasFillValue()) { + array.setObject(0, convertMissing.getFillValue()); } return array; } @@ -541,26 +547,25 @@ public Array getMissingDataArray(int[] shape) { * @param f put info here */ public void showScaleMissingProxy(Formatter f) { - f.format("has missing = %s%n", scaleMissingUnsignedProxy.hasMissing()); - if (scaleMissingUnsignedProxy.hasMissing()) { - if (scaleMissingUnsignedProxy.hasMissingValue()) { + f.format("has missing = %s%n", convertMissing.hasMissing()); + if (convertMissing.hasMissing()) { + if (convertMissing.hasMissingValue()) { f.format(" missing value(s) = "); - for (double d : scaleMissingUnsignedProxy.getMissingValues()) + for (double d : convertMissing.getMissingValues()) f.format(" %f", d); f.format("%n"); } - if (scaleMissingUnsignedProxy.hasFillValue()) - f.format(" fillValue = %f%n", scaleMissingUnsignedProxy.getFillValue()); - if (scaleMissingUnsignedProxy.hasValidData()) - f.format(" valid min/max = [%f,%f]%n", scaleMissingUnsignedProxy.getValidMin(), - scaleMissingUnsignedProxy.getValidMax()); - } - f.format("FillValue or default = %s%n", scaleMissingUnsignedProxy.getFillValue()); - - f.format("%nhas scale/offset = %s%n", scaleMissingUnsignedProxy.hasScaleOffset()); - if (scaleMissingUnsignedProxy.hasScaleOffset()) { - double offset = scaleMissingUnsignedProxy.applyScaleOffset(0.0); - double scale = scaleMissingUnsignedProxy.applyScaleOffset(1.0) - offset; + if (convertMissing.hasFillValue()) + f.format(" fillValue = %f%n", convertMissing.getFillValue()); + if (convertMissing.hasValidData()) + f.format(" valid min/max = [%f,%f]%n", convertMissing.getValidMin(), convertMissing.getValidMax()); + } + f.format("FillValue or default = %s%n", convertMissing.getFillValue()); + + f.format("%nhas scale/offset = %s%n", scaleOffset != null); + if (scaleOffset != null) { + double offset = scaleOffset.applyScaleOffset(0.0); + double scale = scaleOffset.applyScaleOffset(1.0) - offset; f.format(" scale_factor = %f add_offset = %f%n", scale, offset); } f.format("original data type = %s%n", orgDataType); @@ -609,77 +614,78 @@ public void removeCoordinateSystem(CoordinateSystem cs) { @Override public boolean hasScaleOffset() { - return scaleMissingUnsignedProxy.hasScaleOffset(); + return scaleOffset != null; } @Override public double getScaleFactor() { - return scaleMissingUnsignedProxy.getScaleFactor(); + return scaleOffset != null ? scaleOffset.getScaleFactor() : 1; } @Override public double getOffset() { - return scaleMissingUnsignedProxy.getOffset(); + return scaleOffset != null ? scaleOffset.getOffset() : 0; } @Override public boolean hasMissing() { - return scaleMissingUnsignedProxy.hasMissing(); + // TODO: null or all false values? + return convertMissing.hasMissing(); } @Override public boolean isMissing(double val) { - return scaleMissingUnsignedProxy.isMissing(val); + return convertMissing.isMissing(val); } @Override public boolean hasValidData() { - return scaleMissingUnsignedProxy.hasValidData(); + return convertMissing.hasMissingValue(); } @Override public double getValidMin() { - return scaleMissingUnsignedProxy.getValidMin(); + return convertMissing.getValidMin(); } @Override public double getValidMax() { - return scaleMissingUnsignedProxy.getValidMax(); + return convertMissing.getValidMax(); } @Override public boolean isInvalidData(double val) { - return scaleMissingUnsignedProxy.isInvalidData(val); + return convertMissing.isInvalidData(val); } @Override public boolean hasFillValue() { - return scaleMissingUnsignedProxy.hasFillValue(); + return convertMissing.hasFillValue(); } @Override public double getFillValue() { - return scaleMissingUnsignedProxy.getFillValue(); + return convertMissing.getFillValue(); } @Override public boolean isFillValue(double val) { - return scaleMissingUnsignedProxy.isFillValue(val); + return convertMissing.isFillValue(val); } @Override public boolean hasMissingValue() { - return scaleMissingUnsignedProxy.hasMissingValue(); + return convertMissing.hasMissingValue(); } @Override public double[] getMissingValues() { - return scaleMissingUnsignedProxy.getMissingValues(); + return convertMissing.getMissingValues(); } @Override public boolean isMissingValue(double val) { - return scaleMissingUnsignedProxy.isMissingValue(val); + return convertMissing.isMissingValue(val); } /** @deprecated Use NetcdfDataset.builder() */ @@ -696,6 +702,18 @@ public void setInvalidDataIsMissing(boolean b) { scaleMissingUnsignedProxy.setInvalidDataIsMissing(b); } + public boolean missingDataIsMissing() { + return builder().missingDataIsMissing; + } + + public boolean fillValueIsMissing() { + return builder().fillValueIsMissing; + } + + public boolean invalidDataIsMissing() { + return builder().invalidDataIsMissing; + } + /** @deprecated Use NetcdfDataset.builder() */ @Deprecated @Override @@ -706,50 +724,54 @@ public void setMissingDataIsMissing(boolean b) { @Nullable @Override public DataType getScaledOffsetType() { - return scaleMissingUnsignedProxy.getScaledOffsetType(); + return scaleOffset.getScaledOffsetType(); } @Override public DataType getUnsignedConversionType() { - return scaleMissingUnsignedProxy.getUnsignedConversionType(); + return unsignedConversion != null ? unsignedConversion.getOutType() : dataType; } @Override public DataType.Signedness getSignedness() { - return scaleMissingUnsignedProxy.getSignedness(); + return unsignedConversion != null ? DataType.Signedness.SIGNED : dataType.getSignedness(); } @Override public double applyScaleOffset(Number value) { - return scaleMissingUnsignedProxy.applyScaleOffset(value); + return scaleOffset != null ? scaleOffset.removeScaleOffset(value) : value.doubleValue(); } @Override public Array applyScaleOffset(Array data) { - return scaleMissingUnsignedProxy.applyScaleOffset(data); + return scaleOffset != null ? scaleOffset.removeScaleOffset(data) : data; } @Override public Number convertUnsigned(Number value) { - return scaleMissingUnsignedProxy.convertUnsigned(value); + return unsignedConversion != null ? unsignedConversion.convertUnsigned(value) : value; } @Override public Array convertUnsigned(Array in) { - return scaleMissingUnsignedProxy.convertUnsigned(in); + return unsignedConversion != null ? unsignedConversion.convertUnsigned(in) : in; } @Override public Number convertMissing(Number value) { - return scaleMissingUnsignedProxy.convertMissing(value); + return convertMissing.convertMissing(value); } @Override public Array convertMissing(Array in) { - return scaleMissingUnsignedProxy.convertMissing(in); + return convertMissing.convertMissing(in); } + /** + * @deprecated use implementations in filters package + */ @Override + @Deprecated public Array convert(Array in, boolean convertUnsigned, boolean applyScaleOffset, boolean convertMissing) { return scaleMissingUnsignedProxy.convert(in, convertUnsigned, applyScaleOffset, convertMissing); } @@ -758,9 +780,12 @@ public Array convert(Array in, boolean convertUnsigned, boolean applyScaleOffset // TODO remove in version 6. private EnhancementsImpl enhanceProxy; private List coordSysNames; - - // TODO make these final and immutable in 6. private EnhanceScaleMissingUnsignedImpl scaleMissingUnsignedProxy = new EnhanceScaleMissingUnsignedImpl(); + + // TODO make immutable in version 6 + private UnsignedConversion unsignedConversion; + private ScaleOffset scaleOffset; + private ConvertMissing convertMissing; private Set enhanceMode = EnumSet.noneOf(Enhance.class); // The set of enhancements that were made. protected Variable orgVar; // wrap this Variable : use it for the I/O @@ -787,23 +812,21 @@ protected VariableDS(Builder builder, Group parentGroup) { this.orgFileTypeId = builder.orgFileTypeId; this.enhanceProxy = new EnhancementsImpl(this, builder.units, builder.getDescription()); - this.scaleMissingUnsignedProxy = new EnhanceScaleMissingUnsignedImpl(this, this.enhanceMode); - this.scaleMissingUnsignedProxy.setFillValueIsMissing(builder.fillValueIsMissing); - this.scaleMissingUnsignedProxy.setInvalidDataIsMissing(builder.invalidDataIsMissing); - this.scaleMissingUnsignedProxy.setMissingDataIsMissing(builder.missingDataIsMissing); if (this.enhanceMode.contains(Enhance.ConvertEnums) && dataType.isEnum()) { this.dataType = DataType.STRING; // LOOK promote enum data type to STRING ???? - } - - if (this.enhanceMode.contains(Enhance.ConvertUnsigned) && !dataType.isEnum()) { - // We may need a larger data type to hold the results of the unsigned conversion. - this.dataType = scaleMissingUnsignedProxy.getUnsignedConversionType(); - } - - if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR) - && scaleMissingUnsignedProxy.hasScaleOffset()) { - this.dataType = scaleMissingUnsignedProxy.getScaledOffsetType(); + } else { + if (this.enhanceMode.contains(Enhance.ConvertUnsigned) && !dataType.isEnum()) { + this.unsignedConversion = UnsignedConversion.createFromVar(this); + this.dataType = unsignedConversion.getOutType(); + } + if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR)) { + this.scaleOffset = ScaleOffset.createFromVariable(this); + this.dataType = scaleOffset.getScaledOffsetType(); + } + if (this.enhanceMode.contains(Enhance.ConvertMissing)) { + this.convertMissing = ConvertMissing.createFromVariable(this); + } } // We have to complete this after the NetcdfDataset is built. diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java new file mode 100644 index 0000000000..1a52e75564 --- /dev/null +++ b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java @@ -0,0 +1,264 @@ +package ucar.nc2.filter; + +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.IndexIterator; +import ucar.nc2.Attribute; +import ucar.nc2.constants.CDM; +import ucar.nc2.constants.DataFormatType; +import ucar.nc2.dataset.VariableDS; +import ucar.nc2.iosp.netcdf3.N3iosp; +import ucar.nc2.util.Misc; + +import java.util.Arrays; + +public class ConvertMissing { + + private boolean hasValidMin, hasValidMax; + private double validMin, validMax; + + private boolean hasFillValue; + private double fillValue; // LOOK: making it double not really correct. What about CHAR? + + private boolean hasMissingValue; + private double[] missingValue; // LOOK: also wrong to make double, for the same reason. + + // defaults from NetcdfDataset modes + private boolean invalidDataIsMissing; + private boolean fillValueIsMissing; + private boolean missingDataIsMissing; + + public static ConvertMissing createFromVariable(VariableDS var) { + // valid min and max + DataType.Signedness signedness = var.getSignedness(); + double validMin = -Double.MAX_VALUE, validMax = Double.MAX_VALUE; + boolean hasValidMin = false, hasValidMax = false; + // assume here its in units of unpacked data. correct this below + Attribute validRangeAtt = var.findAttribute(CDM.VALID_RANGE); + DataType validType = null; + if (validRangeAtt != null && !validRangeAtt.isString() && validRangeAtt.getLength() > 1) { + validType = FilterHelpers.getAttributeDataType(validRangeAtt, signedness); + validMin = var.convertUnsigned(validRangeAtt.getNumericValue(0)).doubleValue(); + validMax = var.convertUnsigned(validRangeAtt.getNumericValue(1)).doubleValue(); + hasValidMin = true; + hasValidMax = true; + } + + Attribute validMinAtt = var.findAttribute(CDM.VALID_MIN); + Attribute validMaxAtt = var.findAttribute(CDM.VALID_MAX); + + // Only process the valid_min and valid_max attributes if valid_range isn't present. + if (!hasValidMin) { + if (validMinAtt != null && !validMinAtt.isString()) { + validType = FilterHelpers.getAttributeDataType(validMinAtt, signedness); + validMin = var.convertUnsigned(validMinAtt.getNumericValue()).doubleValue(); + hasValidMin = true; + } + + if (validMaxAtt != null && !validMaxAtt.isString()) { + validType = FilterHelpers.largestOf(validType, FilterHelpers.getAttributeDataType(validMaxAtt, signedness)); + validMax = var.convertUnsigned(validMaxAtt.getNumericValue()).doubleValue(); + hasValidMax = true; + } + } + + // check if validData values are stored packed or unpacked + if (hasValidMin || hasValidMax) { + if (FilterHelpers.rank(validType) == FilterHelpers.rank(var.getScaledOffsetType()) + && FilterHelpers.rank(validType) > FilterHelpers.rank(var.getUnsignedConversionType())) { + // If valid_range is the same type as the wider of scale_factor and add_offset, PLUS + // it is wider than the (packed) data, we know that the valid_range values were stored as unpacked. + // We already assumed that this was the case when we first read the attribute values, so there's + // nothing for us to do here. + } else { + // Otherwise, the valid_range values were stored as packed. So now we must unpack them. + if (hasValidMin) { + validMin = var.applyScaleOffset(validMin); + } + if (hasValidMax) { + validMax = var.applyScaleOffset(validMax); + } + } + // During the scaling process, it is possible that the valid minimum and maximum values have effectively been + // swapped (for example, when the scale value is negative). Go ahead and check to make sure the valid min is + // actually less than the valid max, and if not, fix it. See https://github.com/Unidata/netcdf-java/issues/572. + if (validMin > validMax) { + double tmp = validMin; + validMin = validMax; + validMax = tmp; + } + } + + /// _FillValue + double fillValue = Double.MAX_VALUE; + boolean hasFillValue = false; + Attribute fillValueAtt = var.findAttribute(CDM.FILL_VALUE); + if (fillValueAtt != null && !fillValueAtt.isString()) { + fillValue = var.convertUnsigned(fillValueAtt.getNumericValue()).doubleValue(); + fillValue = var.applyScaleOffset(fillValue); // This will fail when _FillValue is CHAR. + hasFillValue = true; + } else { + // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value. + String fileTypeId = var.getFileTypeId(); + boolean isNetcdfIosp = DataFormatType.NETCDF.getDescription().equals(fileTypeId) + || DataFormatType.NETCDF4.getDescription().equals(fileTypeId); + + if (isNetcdfIosp) { + DataType unsignedConversionType = var.getUnsignedConversionType(); + if (unsignedConversionType.isNumeric()) { + fillValue = var.applyScaleOffset(N3iosp.getFillValueDefault(unsignedConversionType)); + hasFillValue = true; + } + } + } + + /// missing_value + double[] missingValue = null; + boolean hasMissingValue = false; + Attribute missingValueAtt = var.findAttribute(CDM.MISSING_VALUE); + if (missingValueAtt != null) { + if (missingValueAtt.isString()) { + String svalue = missingValueAtt.getStringValue(); + if (var.getOriginalDataType() == DataType.CHAR) { + missingValue = new double[1]; + if (svalue.isEmpty()) { + missingValue[0] = 0; + } else { + missingValue[0] = svalue.charAt(0); + } + + hasMissingValue = true; + } else { // not a CHAR - try to fix problem where they use a numeric value as a String attribute + try { + missingValue = new double[1]; + missingValue[0] = Double.parseDouble(svalue); + hasMissingValue = true; + } catch (NumberFormatException ex) { + // TODO add logger + } + } + } else { // not a string + missingValue = new double[missingValueAtt.getLength()]; + for (int i = 0; i < missingValue.length; i++) { + missingValue[i] = var.convertUnsigned(missingValueAtt.getNumericValue(i)).doubleValue(); + missingValue[i] = var.applyScaleOffset(missingValue[i]); + } + + for (double mv : missingValue) { + if (!Double.isNaN(mv)) { + hasMissingValue = true; // dont need to do anything if it's already a NaN + break; + } + } + } + } + return new ConvertMissing(var.fillValueIsMissing(), var.invalidDataIsMissing(), var.missingDataIsMissing(), + hasValidMin, hasValidMax, validMin, validMax, hasFillValue, fillValue, hasMissingValue, missingValue); + } + + + public ConvertMissing(boolean fillValueIsMissing, boolean invalidDataIsMissing, boolean missingDataIsMissing, + boolean hasValidMin, boolean hasValidMax, double validMin, double validMax, boolean hasFillValue, + double fillValue, boolean hasMissingValue, double[] missingValue) { + this.fillValueIsMissing = fillValueIsMissing; + this.invalidDataIsMissing = invalidDataIsMissing; + this.missingDataIsMissing = missingDataIsMissing; + this.hasValidMin = hasValidMin; + this.hasValidMax = hasValidMax; + this.validMin = validMin; + this.validMax = validMax; + this.hasFillValue = hasFillValue; + this.fillValue = fillValue; + this.hasMissingValue = hasMissingValue; + this.missingValue = missingValue; + } + + public boolean hasValidData() { + return hasValidMin || hasValidMax; + } + + public double getValidMin() { + return validMin; + } + + public double getValidMax() { + return validMax; + } + + public boolean isInvalidData(double val) { + // valid_min and valid_max may have been multiplied by scale_factor, which could be a float, not a double. + // That potential loss of precision means that we cannot do the nearlyEquals() comparison with + // Misc.defaultMaxRelativeDiffDouble. + boolean greaterThanOrEqualToValidMin = + Misc.nearlyEquals(val, validMin, Misc.defaultMaxRelativeDiffFloat) || val > validMin; + boolean lessThanOrEqualToValidMax = + Misc.nearlyEquals(val, validMax, Misc.defaultMaxRelativeDiffFloat) || val < validMax; + + return (hasValidMin && !greaterThanOrEqualToValidMin) || (hasValidMax && !lessThanOrEqualToValidMax); + } + + public boolean hasFillValue() { + return hasFillValue; + } + + public boolean isFillValue(double val) { + return hasFillValue && Misc.nearlyEquals(val, fillValue, Misc.defaultMaxRelativeDiffFloat); + } + + public double getFillValue() { + return fillValue; + } + + public boolean isMissingValue(double val) { + if (!hasMissingValue) { + return false; + } + for (double aMissingValue : missingValue) { + if (Misc.nearlyEquals(val, aMissingValue, Misc.defaultMaxRelativeDiffFloat)) { + return true; + } + } + return false; + } + + public double[] getMissingValues() { + return missingValue; + } + + public boolean hasMissingValue() { + return hasMissingValue; + } + + public boolean hasMissing() { + return (invalidDataIsMissing && hasValidData()) || (fillValueIsMissing && hasFillValue()) + || (missingDataIsMissing && hasMissingValue()); + } + + public boolean isMissing(double val) { + if (Double.isNaN(val)) { + return true; + } else { + return (missingDataIsMissing && isMissingValue(val)) || (fillValueIsMissing && isFillValue(val)) + || (invalidDataIsMissing && isInvalidData(val)); + } + } + + public Number convertMissing(Number value) { + return isMissing(value.doubleValue()) ? Double.NaN : value; + } + + public Array convertMissing(Array in) { + Array out = Array.factory(in.getDataType(), in.getShape()); + IndexIterator iterIn = in.getIndexIterator(); + IndexIterator iterOut = out.getIndexIterator(); + + // iterate and convert elements + while (iterIn.hasNext()) { + Number value = (Number) iterIn.getObjectNext(); + value = convertMissing(value); + iterOut.setObjectNext(value); + } + + return out; + } +} diff --git a/cdm/core/src/main/java/ucar/nc2/filter/Filter.java b/cdm/core/src/main/java/ucar/nc2/filter/Filter.java index e21e28605a..ce21551332 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/Filter.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/Filter.java @@ -10,6 +10,9 @@ import java.util.Formatter; import java.util.Map; +/** + * A class implementing a conversion to be applied on large chunk of data + */ public abstract class Filter { public abstract String getName(); diff --git a/cdm/core/src/main/java/ucar/nc2/filter/FilterHelpers.java b/cdm/core/src/main/java/ucar/nc2/filter/FilterHelpers.java new file mode 100644 index 0000000000..82fdd49a80 --- /dev/null +++ b/cdm/core/src/main/java/ucar/nc2/filter/FilterHelpers.java @@ -0,0 +1,225 @@ +package ucar.nc2.filter; + +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.IndexIterator; +import ucar.nc2.Attribute; + +import java.nio.*; + +import static ucar.ma2.DataType.*; + +public class FilterHelpers { + + public static Array bytesToArray(byte[] in, DataType type, ByteOrder order) { + return Array.factory(type, new int[] {in.length / type.getSize()}, convertToType(in, type, order)); + } + + public static byte[] arrayToBytes(Array arr, DataType type, ByteOrder order) { + ByteBuffer bb = ByteBuffer.allocate((int) arr.getSize() * type.getSize()); + bb.order(order); + + IndexIterator ii = arr.getIndexIterator(); + while (ii.hasNext()) { + switch (type) { + case BYTE: + case UBYTE: + bb.put(ii.getByteNext()); + break; + case SHORT: + case USHORT: + bb.putShort(ii.getShortNext()); + break; + case INT: + case UINT: + bb.putInt(ii.getIntNext()); + break; + case LONG: + case ULONG: + bb.putLong(ii.getLongNext()); + break; + case FLOAT: + bb.putFloat(ii.getFloatNext()); + break; + case DOUBLE: + bb.putDouble(ii.getDoubleNext()); + break; + } + } + return bb.array(); + } + + private static Object convertToType(byte[] dataIn, DataType wantType, ByteOrder bo) { + if (wantType.getSize() == 1) { + return dataIn; + } // no need for conversion + + ByteBuffer bb = ByteBuffer.wrap(dataIn); + bb.order(bo); + switch (wantType) { + case SHORT: + case USHORT: + ShortBuffer sb = bb.asShortBuffer(); + short[] shortArray = new short[sb.limit()]; + sb.get(shortArray); + return shortArray; + case INT: + case UINT: + IntBuffer ib = bb.asIntBuffer(); + int[] intArray = new int[ib.limit()]; + ib.get(intArray); + return intArray; + case LONG: + case ULONG: + LongBuffer lb = bb.asLongBuffer(); + long[] longArray = new long[lb.limit()]; + lb.get(longArray); + return longArray; + case FLOAT: + FloatBuffer fb = bb.asFloatBuffer(); + float[] floatArray = new float[fb.limit()]; + fb.get(floatArray); + return floatArray; + case DOUBLE: + DoubleBuffer db = bb.asDoubleBuffer(); + double[] doubleArray = new double[db.limit()]; + db.get(doubleArray); + return doubleArray; + default: + return bb.array(); + } + } + + /** + * Returns the smallest numeric data type that: + *
      + *
    1. can hold a larger integer than {@code dataType} can
      2. + *
    2. if integral, has the same signedness as {@code dataType}
      3. + *
    + *

    + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
    ArgumentResult
    BYTESHORT
    UBYTEUSHORT
    SHORTINT
    USHORTUINT
    INTLONG
    UINTULONG
    LONGDOUBLE
    ULONGDOUBLE
    Any other data typeJust return argument
    + *

    + * The returned type is intended to be just big enough to hold the result of performing an unsigned conversion of a + * value of the smaller type. For example, the {@code byte} value {@code -106} equals {@code 150} when interpreted + * as unsigned. That won't fit in a (signed) {@code byte}, but it will fit in a {@code short}. + * + * @param dataType an integral data type. + * @return the next larger type. + */ + public static DataType nextLarger(DataType dataType) { + switch (dataType) { + case BYTE: + return SHORT; + case UBYTE: + return USHORT; + case SHORT: + return INT; + case USHORT: + return UINT; + case INT: + return LONG; + case UINT: + return ULONG; + case LONG: + case ULONG: + return DOUBLE; + default: + return dataType; + } + } + + // Get the data type of an attribute. Make it unsigned if the variable is unsigned. + public static DataType getAttributeDataType(Attribute attribute, DataType.Signedness signedness) { + DataType dataType = attribute.getDataType(); + if (signedness == Signedness.UNSIGNED) { + // If variable is unsigned, make its integral attributes unsigned too. + dataType = dataType.withSignedness(signedness); + } + return dataType; + } + + + public static int rank(DataType dataType) { + if (dataType == null) { + return -1; + } + + switch (dataType) { + case BYTE: + return 0; + case UBYTE: + return 1; + case SHORT: + return 2; + case USHORT: + return 3; + case INT: + return 4; + case UINT: + return 5; + case LONG: + return 6; + case ULONG: + return 7; + case FLOAT: + return 8; + case DOUBLE: + return 9; + default: + return -1; + } + } + + public static DataType largestOf(DataType... dataTypes) { + DataType widest = null; + for (DataType dataType : dataTypes) { + if (widest == null) { + widest = dataType; + } else if (rank(dataType) > rank(widest)) { + widest = dataType; + } + } + return widest; + } +} diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java index 369ad2f447..4ff1b98cc0 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java @@ -8,6 +8,9 @@ import ucar.ma2.Array; import ucar.ma2.DataType; import ucar.ma2.IndexIterator; +import ucar.nc2.Attribute; +import ucar.nc2.constants.CDM; +import ucar.nc2.dataset.VariableDS; import java.nio.*; import java.util.HashMap; @@ -25,6 +28,13 @@ public class ScaleOffset extends Filter { private static final int id = 6; + public static class Keys { + public static final String OFFSET_KEY = "offset"; + public static final String SCALE_KEY = "scale"; + public static final String DTYPE_KEY = "dtype"; + public static final String ASTYPE_KEY = "astype"; + } + // maps numeric zarr datatypes to CDM datatypes private static Map dTypeMap; @@ -43,7 +53,7 @@ public class ScaleOffset extends Filter { } private final double offset; - private final int scale; + private final double scale; // type information for original data type private final ByteOrder dtypeOrder; @@ -53,24 +63,73 @@ public class ScaleOffset extends Filter { private final DataType astype; private final ByteOrder astypeOrder; + public static ScaleOffset createFromVariable(VariableDS var) { + + DataType scaleType = null, offsetType = null; + double scale = 1.0, offset = 0; + + DataType origDataType = var.getDataType(); + DataType.Signedness signedness = var.getSignedness(); + + Attribute scaleAtt = var.findAttribute(CDM.SCALE_FACTOR); + if (scaleAtt != null && !scaleAtt.isString()) { + scaleType = FilterHelpers.getAttributeDataType(scaleAtt, signedness); + scale = var.convertUnsigned(scaleAtt.getNumericValue()).doubleValue(); + } + + Attribute offsetAtt = var.findAttribute(CDM.ADD_OFFSET); + if (offsetAtt != null && !offsetAtt.isString()) { + offsetType = FilterHelpers.getAttributeDataType(offsetAtt, signedness); + offset = var.convertUnsigned(offsetAtt.getNumericValue()).doubleValue(); + } + if (scale != 1.0 || offset != 0) { + DataType scaledOffsetType = + FilterHelpers.largestOf(var.getUnsignedConversionType(), scaleType, offsetType).withSignedness(signedness); + + Map scaleOffsetProps = new HashMap<>(); + scaleOffsetProps.put(ScaleOffset.Keys.OFFSET_KEY, offset); + scaleOffsetProps.put(ScaleOffset.Keys.SCALE_KEY, scale); + scaleOffsetProps.put(ScaleOffset.Keys.DTYPE_KEY, origDataType); + scaleOffsetProps.put(ScaleOffset.Keys.ASTYPE_KEY, scaledOffsetType); + return new ScaleOffset(scaleOffsetProps); + } + return null; + } public ScaleOffset(Map properties) { // get offset and scale parameters - offset = ((Number) properties.getOrDefault("offset", 0)).doubleValue(); - scale = (int) properties.getOrDefault("scale", 1); + offset = ((Number) properties.getOrDefault(Keys.OFFSET_KEY, 0)).doubleValue(); + scale = (int) properties.getOrDefault(Keys.SCALE_KEY, 1); // input data type - String type = (String) properties.get("dtype"); - dtype = parseDataType(type); - if (dtype == null) { + Object typeProp = properties.get(Keys.DTYPE_KEY); + if (typeProp instanceof String) { + String type = (String) typeProp; + dtype = parseDataType(type); + if (dtype == null) { + throw new RuntimeException("ScaleOffset error: could not parse dtype"); + } + dtypeOrder = parseByteOrder(type, ByteOrder.LITTLE_ENDIAN); + } else if (typeProp instanceof DataType) { + dtype = (DataType) typeProp; + dtypeOrder = ByteOrder.LITTLE_ENDIAN; + } else { throw new RuntimeException("ScaleOffset error: could not parse dtype"); } - dtypeOrder = parseByteOrder(type, ByteOrder.LITTLE_ENDIAN); // get storage type, if exists, or default to dtype - String aType = (String) properties.getOrDefault("astype", type); - astype = parseDataType(aType); - astypeOrder = parseByteOrder(aType, dtypeOrder); + Object aTypeProp = properties.getOrDefault(Keys.ASTYPE_KEY, null); + if (aTypeProp instanceof String) { + String aType = (String) aTypeProp; + astype = parseDataType(aType); + astypeOrder = parseByteOrder(aType, dtypeOrder); + } else if (aTypeProp instanceof DataType) { + astype = (DataType) aTypeProp; + astypeOrder = ByteOrder.LITTLE_ENDIAN; + } else { + astype = dtype; + astypeOrder = dtypeOrder; + } } @Override @@ -83,27 +142,36 @@ public int getId() { return id; } + public double getScaleFactor() { + return this.scale; + } + + public double getOffset() { + return this.offset; + } + + public DataType getScaledOffsetType() { + return this.astype; + } + @Override public byte[] encode(byte[] dataIn) { - Array in = - Array.factory(dtype, new int[] {dataIn.length / dtype.getSize()}, convertToType(dataIn, dtype, dtypeOrder)); - Array out = applyScaleOffset(in); - return arrayToBytes(out, astype, astypeOrder); + Array out = applyScaleOffset(FilterHelpers.bytesToArray(dataIn, dtype, dtypeOrder)); + return FilterHelpers.arrayToBytes(out, astype, astypeOrder); } @Override public byte[] decode(byte[] dataIn) { - Array in = - Array.factory(astype, new int[] {dataIn.length / astype.getSize()}, convertToType(dataIn, astype, astypeOrder)); - Array out = removeScaleOffset(in); - return arrayToBytes(out, dtype, dtypeOrder); + Array out = removeScaleOffset(FilterHelpers.bytesToArray(dataIn, astype, astypeOrder)); + return FilterHelpers.arrayToBytes(out, dtype, dtypeOrder); } - private Array applyScaleOffset(Array in) { + // not used anywhere yet + public Array applyScaleOffset(Array in) { // use wider datatype if unsigned DataType outType = astype; if (astype.getSignedness() == Signedness.UNSIGNED) { - outType = nextLarger(astype).withSignedness(DataType.Signedness.UNSIGNED); + outType = FilterHelpers.nextLarger(astype).withSignedness(DataType.Signedness.UNSIGNED); } // create conversion array @@ -122,11 +190,11 @@ private Array applyScaleOffset(Array in) { return out; } - private Array removeScaleOffset(Array in) { + public Array removeScaleOffset(Array in) { // use wider datatype if unsigned DataType outType = dtype; if (dtype.getSignedness() == Signedness.UNSIGNED) { - outType = nextLarger(dtype).withSignedness(DataType.Signedness.UNSIGNED); + outType = FilterHelpers.nextLarger(dtype).withSignedness(DataType.Signedness.UNSIGNED); } // create conversion array @@ -166,26 +234,14 @@ private static ByteOrder parseByteOrder(String dtype, ByteOrder defaultOrder) { return defaultOrder; } - private DataType nextLarger(DataType dataType) { - switch (dataType) { - case BYTE: - return SHORT; - case UBYTE: - return USHORT; - case SHORT: - return INT; - case USHORT: - return UINT; - case INT: - return LONG; - case UINT: - return ULONG; - case LONG: - case ULONG: - return DOUBLE; - default: - return dataType; - } + public int applyScaleOffset(Number value) { + double convertedValue = value.doubleValue(); + return (int) Math.round((convertedValue - offset) * scale); + } + + public double removeScaleOffset(Number value) { + double convertedValue = value.doubleValue(); + return convertedValue / scale + offset; } private Number convertUnsigned(Number value, Signedness signedness) { @@ -197,90 +253,6 @@ private Number convertUnsigned(Number value, Signedness signedness) { } } - private Object convertToType(byte[] dataIn, DataType wantType, ByteOrder bo) { - if (wantType.getSize() == 1) { - return dataIn; - } // no need for conversion - - ByteBuffer bb = ByteBuffer.wrap(dataIn); - bb.order(bo); - switch (wantType) { - case SHORT: - case USHORT: - ShortBuffer sb = bb.asShortBuffer(); - short[] shortArray = new short[sb.limit()]; - sb.get(shortArray); - return shortArray; - case INT: - case UINT: - IntBuffer ib = bb.asIntBuffer(); - int[] intArray = new int[ib.limit()]; - ib.get(intArray); - return intArray; - case LONG: - case ULONG: - LongBuffer lb = bb.asLongBuffer(); - long[] longArray = new long[lb.limit()]; - lb.get(longArray); - return longArray; - case FLOAT: - FloatBuffer fb = bb.asFloatBuffer(); - float[] floatArray = new float[fb.limit()]; - fb.get(floatArray); - return floatArray; - case DOUBLE: - DoubleBuffer db = bb.asDoubleBuffer(); - double[] doubleArray = new double[db.limit()]; - db.get(doubleArray); - return doubleArray; - default: - return bb.array(); - } - } - - private int applyScaleOffset(Number value) { - double convertedValue = value.doubleValue(); - return (int) Math.round((convertedValue - offset) * scale); - } - - private double removeScaleOffset(Number value) { - double convertedValue = value.doubleValue(); - return convertedValue / scale + offset; - } - - private byte[] arrayToBytes(Array arr, DataType type, ByteOrder order) { - ByteBuffer bb = ByteBuffer.allocate((int) arr.getSize() * type.getSize()); - bb.order(order); - - IndexIterator ii = arr.getIndexIterator(); - while (ii.hasNext()) { - switch (type) { - case BYTE: - case UBYTE: - bb.put(ii.getByteNext()); - break; - case SHORT: - case USHORT: - bb.putShort(ii.getShortNext()); - break; - case INT: - case UINT: - bb.putInt(ii.getIntNext()); - break; - case LONG: - case ULONG: - bb.putLong(ii.getLongNext()); - break; - case FLOAT: - bb.putFloat(ii.getFloatNext()); - break; - case DOUBLE: - bb.putDouble(ii.getDoubleNext()); - break; - } - } - return bb.array(); - } public static class Provider implements FilterProvider { diff --git a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java new file mode 100644 index 0000000000..ddba9a8220 --- /dev/null +++ b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java @@ -0,0 +1,68 @@ +package ucar.nc2.filter; + +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.IndexIterator; +import ucar.nc2.Variable; +import ucar.nc2.constants.CDM; +import ucar.nc2.dataset.VariableDS; + +public class UnsignedConversion { + + private DataType outType; + + public static UnsignedConversion createFromVar(VariableDS var) { + DataType origDataType = var.getDataType(); + DataType unsignedConversionType = origDataType; + + // unsignedConversionType is initialized to origDataType, and origDataType may be a non-integral type that doesn't + // have an "unsigned flavor" (such as FLOAT and DOUBLE). Furthermore, unsignedConversionType may start out as + // integral, but then be widened to non-integral (i.e. LONG -> DOUBLE). For these reasons, we cannot rely upon + // unsignedConversionType to store the signedness of the variable. We need a separate field. + DataType.Signedness signedness = origDataType.getSignedness(); + + // In the event of conflict, "unsigned" wins. Potential conflicts include: + // 1. origDataType is unsigned, but variable has "_Unsigned == false" attribute. + // 2. origDataType is signed, but variable has "_Unsigned == true" attribute. + if (signedness == DataType.Signedness.SIGNED) { + String unsignedAtt = var.attributes().findAttributeString(CDM.UNSIGNED, null); + if (unsignedAtt != null && unsignedAtt.equalsIgnoreCase("true")) { + signedness = DataType.Signedness.UNSIGNED; + } + } + + if (signedness == DataType.Signedness.UNSIGNED) { + // We may need a larger data type to hold the results of the unsigned conversion. + unsignedConversionType = FilterHelpers.nextLarger(origDataType).withSignedness(DataType.Signedness.UNSIGNED); + } + return new UnsignedConversion(unsignedConversionType); + } + + public UnsignedConversion(DataType outType) { + this.outType = outType; + } + + public DataType getOutType() { + return this.outType; + } + + public Number convertUnsigned(Number value) { + return DataType.widenNumberIfNegative(value); + } + + + public Array convertUnsigned(Array in) { + Array out = Array.factory(outType, in.getShape()); + IndexIterator iterIn = in.getIndexIterator(); + IndexIterator iterOut = out.getIndexIterator(); + + // iterate and convert elements + while (iterIn.hasNext()) { + Number value = (Number) iterIn.getObjectNext(); + value = convertUnsigned(value); + iterOut.setObjectNext(value); + } + + return out; + } +} diff --git a/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5iospNew.java b/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5iospNew.java index 771737de0e..73bf8e7999 100644 --- a/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5iospNew.java +++ b/cdm/core/src/main/java/ucar/nc2/internal/iosp/hdf5/H5iospNew.java @@ -509,7 +509,7 @@ void convertHeap(ArrayStructureBB asbb, int pos, StructureMembers sm) throws IOE // vlenarray extracts the i'th vlen contents (struct not supported). Array vlenArray = header.readHeapVlen(bb, destPos, m.getDataType(), endian); fieldarray[i] = vlenArray; - destPos += VLEN_T_SIZE; // Apparentlly no way to compute VLEN_T_SIZE on the fly + destPos += VLEN_T_SIZE; // Apparently no way to compute VLEN_T_SIZE on the fly } Array result; if (prefixrank == 0) // if scalar, return just the singleton vlen array From 65291f3a066f0089df80888673a484f5c8b33c6d Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Mon, 27 Mar 2023 13:58:06 -0700 Subject: [PATCH 08/24] add tests --- .../java/ucar/nc2/dataset/VariableDS.java | 16 +- .../java/ucar/nc2/filter/ScaleOffset.java | 19 ++- .../ucar/nc2/filter/UnsignedConversion.java | 1 + .../ucar/nc2/filter/TestEnhancements.java | 148 ++++++++++++++++++ .../java/ucar/nc2/filter/TestFilters.java | 2 +- 5 files changed, 175 insertions(+), 11 deletions(-) create mode 100644 cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index a30e5a3fc2..362c91d22d 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -272,8 +272,16 @@ Array convert(Array data, Set enhancements) { if (this.isVariableLength) { return data; } - return scaleMissingUnsignedProxy.convert(data, enhancements.contains(Enhance.ConvertUnsigned), - enhancements.contains(Enhance.ApplyScaleOffset), enhancements.contains(Enhance.ConvertMissing)); + if (enhancements.contains(Enhance.ConvertUnsigned)) { + data = unsignedConversion.convertUnsigned(data); + } + if (enhancements.contains(Enhance.ApplyScaleOffset) && scaleOffset != null) { + data = scaleOffset.removeScaleOffset(data); + } + if (enhancements.contains(Enhance.ConvertMissing)) { + data = convertMissing.convertMissing(data); + } + return data; } } @@ -724,7 +732,7 @@ public void setMissingDataIsMissing(boolean b) { @Nullable @Override public DataType getScaledOffsetType() { - return scaleOffset.getScaledOffsetType(); + return scaleOffset != null ? scaleOffset.getScaledOffsetType() : dataType; } @Override @@ -822,7 +830,7 @@ protected VariableDS(Builder builder, Group parentGroup) { } if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR)) { this.scaleOffset = ScaleOffset.createFromVariable(this); - this.dataType = scaleOffset.getScaledOffsetType(); + this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType; } if (this.enhanceMode.contains(Enhance.ConvertMissing)) { this.convertMissing = ConvertMissing.createFromVariable(this); diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java index 4ff1b98cc0..dcfefb94d4 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java @@ -55,6 +55,9 @@ public static class Keys { private final double offset; private final double scale; + private static final double DEFAULT_OFFSET = 0.0; + private static final double DEFAULT_SCALE = 1.0; + // type information for original data type private final ByteOrder dtypeOrder; private final DataType dtype; @@ -66,7 +69,7 @@ public static class Keys { public static ScaleOffset createFromVariable(VariableDS var) { DataType scaleType = null, offsetType = null; - double scale = 1.0, offset = 0; + double scale = DEFAULT_SCALE, offset = DEFAULT_OFFSET; DataType origDataType = var.getDataType(); DataType.Signedness signedness = var.getSignedness(); @@ -82,7 +85,7 @@ public static ScaleOffset createFromVariable(VariableDS var) { offsetType = FilterHelpers.getAttributeDataType(offsetAtt, signedness); offset = var.convertUnsigned(offsetAtt.getNumericValue()).doubleValue(); } - if (scale != 1.0 || offset != 0) { + if (scale != DEFAULT_SCALE || offset != DEFAULT_OFFSET) { DataType scaledOffsetType = FilterHelpers.largestOf(var.getUnsignedConversionType(), scaleType, offsetType).withSignedness(signedness); @@ -98,8 +101,8 @@ public static ScaleOffset createFromVariable(VariableDS var) { public ScaleOffset(Map properties) { // get offset and scale parameters - offset = ((Number) properties.getOrDefault(Keys.OFFSET_KEY, 0)).doubleValue(); - scale = (int) properties.getOrDefault(Keys.SCALE_KEY, 1); + offset = ((Number) properties.getOrDefault(Keys.OFFSET_KEY, DEFAULT_OFFSET)).doubleValue(); + scale = ((Number) properties.getOrDefault(Keys.SCALE_KEY, DEFAULT_SCALE)).doubleValue(); // input data type Object typeProp = properties.get(Keys.DTYPE_KEY); @@ -156,18 +159,21 @@ public DataType getScaledOffsetType() { @Override public byte[] encode(byte[] dataIn) { + if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return dataIn; } Array out = applyScaleOffset(FilterHelpers.bytesToArray(dataIn, dtype, dtypeOrder)); return FilterHelpers.arrayToBytes(out, astype, astypeOrder); } @Override public byte[] decode(byte[] dataIn) { + if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return dataIn; } Array out = removeScaleOffset(FilterHelpers.bytesToArray(dataIn, astype, astypeOrder)); return FilterHelpers.arrayToBytes(out, dtype, dtypeOrder); } // not used anywhere yet public Array applyScaleOffset(Array in) { + if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return in; } // use wider datatype if unsigned DataType outType = astype; if (astype.getSignedness() == Signedness.UNSIGNED) { @@ -191,6 +197,7 @@ public Array applyScaleOffset(Array in) { } public Array removeScaleOffset(Array in) { + if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return in; } // use wider datatype if unsigned DataType outType = dtype; if (dtype.getSignedness() == Signedness.UNSIGNED) { @@ -234,9 +241,9 @@ private static ByteOrder parseByteOrder(String dtype, ByteOrder defaultOrder) { return defaultOrder; } - public int applyScaleOffset(Number value) { + public double applyScaleOffset(Number value) { double convertedValue = value.doubleValue(); - return (int) Math.round((convertedValue - offset) * scale); + return Math.round((convertedValue - offset) * scale); } public double removeScaleOffset(Number value) { diff --git a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java index ddba9a8220..8c1da9ea3a 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java @@ -47,6 +47,7 @@ public DataType getOutType() { } public Number convertUnsigned(Number value) { + Number newVal = DataType.widenNumberIfNegative(value); return DataType.widenNumberIfNegative(value); } diff --git a/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java new file mode 100644 index 0000000000..e54256d4a2 --- /dev/null +++ b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java @@ -0,0 +1,148 @@ +package ucar.nc2.filter; + +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import ucar.ma2.Array; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.nc2.Attribute; +import ucar.nc2.Dimension; +import ucar.nc2.Variable; +import ucar.nc2.constants.CDM; +import ucar.nc2.dataset.NetcdfDataset; +import ucar.nc2.dataset.NetcdfDatasets; +import ucar.nc2.write.NetcdfFormatWriter; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.IOException; + +public class TestEnhancements { + + private static NetcdfDataset ncd; + + private static short[] signedShorts = new short[] {123, 124, 125, 126, 127, -128, -127, -126, -125, -124}; + + private static final float VALID_MIN = 100; + private static final float VALID_MAX = 200; + private static final float FILL_VALUE = 150; + private static float[] missingData = new float[]{90, 100, Float.NaN, 120, 130, 140, 150, 190, 200, 210}; + + private static final Short SIGNED_SCALED_MAX = -126; + private static final Short SIGNED_SCALED_FILL_VALUE = -128; + + @ClassRule + public static final TemporaryFolder tempFolder = new TemporaryFolder(); + + @BeforeClass + public static void setUp() throws IOException , InvalidRangeException { + final int data_len = 10; + String filePath = tempFolder.newFile().getAbsolutePath(); + NetcdfFormatWriter.Builder builder = NetcdfFormatWriter.createNewNetcdf3(filePath); + Dimension dim = builder.addDimension("dim", data_len); + + Array signedData = Array.factory(DataType.SHORT, new int[]{data_len}, signedShorts); + // signed shorts + builder.addVariable("signedVar", DataType.SHORT, "dim"); + // unsigned shorts + builder.addVariable("unsignedVar", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.UNSIGNED, "true")); + + // scaled and offset data + builder.addVariable("scaleOffsetVar", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.SCALE_FACTOR, .1)) + .addAttribute(new Attribute(CDM.ADD_OFFSET, 10)); + + Array missingDataArray = Array.factory(DataType.FLOAT, new int[]{data_len}, missingData); + // Data with min + builder.addVariable("validMin", DataType.FLOAT, "dim").addAttribute(new Attribute(CDM.VALID_MIN, VALID_MIN)); + // Data with min and max + builder.addVariable("validMinMax", DataType.FLOAT, "dim") + .addAttribute(new Attribute(CDM.VALID_MIN, VALID_MIN)) + .addAttribute(new Attribute(CDM.VALID_MAX, VALID_MAX)); + // Data with range and fill value + Array range = Array.factory(DataType.FLOAT, new int[]{2}, new float[]{VALID_MIN, VALID_MAX}); + builder.addVariable("validRange", DataType.FLOAT, "dim") + .addAttribute(Attribute.builder(CDM.VALID_RANGE).setValues(range).build()) + .addAttribute(Attribute.builder(CDM.FILL_VALUE).setNumericValue(FILL_VALUE, true).build()); + + // unsigned, scaled/offset, and missing value + Array enhanceAllArray = Array.factory(DataType.SHORT, new int[]{data_len}, signedShorts); + builder.addVariable("enhanceAll", DataType.SHORT, "dim") + .addAttribute(new Attribute(CDM.UNSIGNED, "true")) + .addAttribute(new Attribute(CDM.SCALE_FACTOR, .1)) + .addAttribute(new Attribute(CDM.ADD_OFFSET, 10)) + .addAttribute(new Attribute(CDM.VALID_MAX, SIGNED_SCALED_MAX)) + .addAttribute(new Attribute(CDM.FILL_VALUE, SIGNED_SCALED_FILL_VALUE)); + + // write data + NetcdfFormatWriter writer = builder.build(); + writer.write(writer.findVariable("signedVar"), new int[1], signedData); + writer.write(writer.findVariable("unsignedVar"), new int[1], signedData); + writer.write(writer.findVariable("scaleOffsetVar"), new int[1], signedData); + writer.write(writer.findVariable("validMin"), new int[1], missingDataArray); + writer.write(writer.findVariable("validMinMax"), new int[1], missingDataArray); + writer.write(writer.findVariable("validRange"), new int[1], missingDataArray); + writer.write(writer.findVariable("enhanceAll"), new int[1], enhanceAllArray); + writer.close(); + ncd = NetcdfDatasets.openDataset(filePath); + } + + @Test + public void testUnsignedConversion() throws IOException { + final int[] unsignedValues = new int[]{123, 124, 125, 126, 127, -128, -127, -126, -125, -126}; + // signed var + Variable v = ncd.findVariable("signedVar"); + Array data = v.read(); + assertThat(data.isUnsigned()).isFalse(); + assertThat(data.getDataType()).isEqualTo(DataType.SHORT); + assertThat((short[])data.copyTo1DJavaArray()).isEqualTo(signedShorts); + + // var with unsigned data type + v = ncd.findVariable("unsignedVar"); + data = v.read(); + assertThat(data.isUnsigned()).isTrue(); + assertThat(data.getDataType()).isEqualTo(DataType.UINT); + assertThat((int[])data.copyTo1DJavaArray()).isEqualTo(unsignedValues); + } + + @Test + public void testScaleOffset() throws IOException { + final double[] expected = new double[]{1240, 1250, 1260, 1270, 1280, -1270, -1260, -1250, -1240, -1230}; + // signed var + Variable v = ncd.findVariable("scaleOffsetVar"); + Array data = v.read(); + assertThat(data.isUnsigned()).isFalse(); + assertThat(data.getDataType()).isEqualTo(DataType.SHORT); + assertThat((double[])data.copyTo1DJavaArray()).isEqualTo(expected); + } + + @Test + public void testConvertMissing() throws IOException { + // var with valid min + float[] expected = new float[]{Float.NaN, 100, Float.NaN, 120, 130, 140, 150, 190, 200, 210}; + Variable v = ncd.findVariable("validMin"); + Array data = v.read(); + assertThat((float[])data.copyTo1DJavaArray()).isEqualTo(expected); + + // var with valid min and max + expected = new float[]{Float.NaN, 100, Float.NaN, 120, 130, 140, 150, 190, 200, Float.NaN}; + v = ncd.findVariable("validMinMax"); + data = v.read(); + assertThat((float[])data.copyTo1DJavaArray()).isEqualTo(expected); + + // var with valid range and fill value + expected = new float[]{Float.NaN, 100, Float.NaN, 120, 130, 140, Float.NaN, 190, 200, Float.NaN}; + v = ncd.findVariable("validRange"); + data = v.read(); + assertThat((float[])data.copyTo1DJavaArray()).isEqualTo(expected); + } + + @Test + public void testCombinedEnhancements() throws IOException { + int[] expected = new int[]{1240, 1250, 1260, 1270, 1280, 0, 1278, 1260, 0, 0}; + Variable v = ncd.findVariable("enhanceAll"); + Array data = v.read(); + assertThat((int[])data.copyTo1DJavaArray()).isEqualTo(expected); + } +} diff --git a/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java b/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java index 51c67f55bc..b98298efdf 100644 --- a/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java +++ b/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java @@ -100,7 +100,7 @@ public void testScaleOffset() throws IOException { // test empty props Map props2 = new HashMap<>(); props2.put("id", "fixedscaleoffset"); - props2.put("dtype", " Date: Tue, 28 Mar 2023 12:13:42 -0700 Subject: [PATCH 09/24] fix bugs --- .../src/test/java/ucar/nc2/TestSequence.java | 5 - .../nc2/ft/point/TestCfDocDsgExamples.java | 4 - .../EnhanceScaleMissingUnsignedImpl.java | 628 ------------------ .../java/ucar/nc2/dataset/VariableDS.java | 129 ++-- .../java/ucar/nc2/filter/ConvertMissing.java | 24 +- .../java/ucar/nc2/filter/ScaleOffset.java | 24 +- .../ucar/nc2/filter/UnsignedConversion.java | 2 - .../ucar/nc2/filter/TestEnhancements.java | 246 ++++--- .../src/test/java/ucar/nc2/TestSequence.java | 11 - 9 files changed, 230 insertions(+), 843 deletions(-) delete mode 100644 cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsignedImpl.java diff --git a/cdm-test/src/test/java/ucar/nc2/TestSequence.java b/cdm-test/src/test/java/ucar/nc2/TestSequence.java index 6e6780a5d6..69cbbf1c8b 100644 --- a/cdm-test/src/test/java/ucar/nc2/TestSequence.java +++ b/cdm-test/src/test/java/ucar/nc2/TestSequence.java @@ -36,16 +36,12 @@ import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import ucar.ma2.*; -import ucar.nc2.dataset.NetcdfDataset; import ucar.nc2.dataset.NetcdfDatasets; import ucar.unidata.util.test.category.NeedsCdmUnitTest; import ucar.unidata.util.test.TestDir; import java.io.File; import java.io.IOException; -import java.lang.invoke.MethodHandles; import java.util.Arrays; import java.util.List; @@ -56,7 +52,6 @@ * @since Nov 10, 2009 */ public class TestSequence { - private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @Test @Category(NeedsCdmUnitTest.class) diff --git a/cdm-test/src/test/java/ucar/nc2/ft/point/TestCfDocDsgExamples.java b/cdm-test/src/test/java/ucar/nc2/ft/point/TestCfDocDsgExamples.java index c3f29ef9a7..0e7422ec8e 100644 --- a/cdm-test/src/test/java/ucar/nc2/ft/point/TestCfDocDsgExamples.java +++ b/cdm-test/src/test/java/ucar/nc2/ft/point/TestCfDocDsgExamples.java @@ -4,8 +4,6 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import ucar.nc2.NetcdfFile; import ucar.nc2.constants.FeatureType; import ucar.nc2.dataset.NetcdfDatasets; @@ -16,7 +14,6 @@ import java.io.File; import java.io.FilenameFilter; import java.io.IOException; -import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; @@ -28,7 +25,6 @@ */ @RunWith(Parameterized.class) public class TestCfDocDsgExamples { - private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String cfDocDsgExamplesDir = TestDir.cdmLocalFromTestDataDir + "cfDocDsgExamples/"; private static List getPointDatasets() { diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsignedImpl.java b/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsignedImpl.java deleted file mode 100644 index 7a296e4b90..0000000000 --- a/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsignedImpl.java +++ /dev/null @@ -1,628 +0,0 @@ -/* - * Copyright (c) 1998-2020 John Caron and University Corporation for Atmospheric Research/Unidata - * See LICENSE for license information. - */ -package ucar.nc2.dataset; - -import java.util.Set; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import ucar.ma2.*; -import ucar.nc2.*; -import ucar.nc2.constants.CDM; -import ucar.nc2.constants.DataFormatType; -import ucar.nc2.dataset.NetcdfDataset.Enhance; -import ucar.nc2.iosp.netcdf3.N3iosp; -import ucar.nc2.util.Misc; -import javax.annotation.Nonnull; -import java.lang.invoke.MethodHandles; -import java.util.Arrays; -import static ucar.ma2.DataType.*; - -/** - * Implementation of EnhanceScaleMissingUnsigned for unsigned data, scale/offset packed data, and missing data. - * - * @author caron - * @author cwardgar - * @see EnhanceScaleMissingUnsigned - */ -class EnhanceScaleMissingUnsignedImpl implements EnhanceScaleMissingUnsigned { - private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private DataType origDataType, unsignedConversionType, scaledOffsetType; - - // defaults from NetcdfDataset modes - private boolean invalidDataIsMissing = NetcdfDataset.invalidDataIsMissing; - private boolean fillValueIsMissing = NetcdfDataset.fillValueIsMissing; - private boolean missingDataIsMissing = NetcdfDataset.missingDataIsMissing; - - private boolean useScaleOffset; - private double scale = 1.0, offset; - - private boolean hasValidRange, hasValidMin, hasValidMax; - private double validMin = -Double.MAX_VALUE, validMax = Double.MAX_VALUE; - - private boolean hasFillValue; - private double fillValue; // LOOK: making it double not really correct. What about CHAR? - - private boolean hasMissingValue; - private double[] missingValue; // LOOK: also wrong to make double, for the same reason. - - private DataType.Signedness signedness; - - - /** - * Constructor, when you dont want anything done. - */ - EnhanceScaleMissingUnsignedImpl() {} - - /** - * Constructor, default values. - * - * @param forVar the Variable to decorate. - */ - EnhanceScaleMissingUnsignedImpl(VariableDS forVar, Set enhancements) { - this(forVar, enhancements, NetcdfDataset.fillValueIsMissing, NetcdfDataset.invalidDataIsMissing, - NetcdfDataset.missingDataIsMissing); - } - - /** - * Constructor. - * If scale/offset attributes are found, remove them from the decorated variable. - * - * @param forVar the Variable to decorate. - * @param fillValueIsMissing use _FillValue for isMissing() - * @param invalidDataIsMissing use valid_range for isMissing() - * @param missingDataIsMissing use missing_value for isMissing() - */ - EnhanceScaleMissingUnsignedImpl(VariableDS forVar, Set enhancements, boolean fillValueIsMissing, - boolean invalidDataIsMissing, boolean missingDataIsMissing) { - this.fillValueIsMissing = fillValueIsMissing; - this.invalidDataIsMissing = invalidDataIsMissing; - this.missingDataIsMissing = missingDataIsMissing; - - this.origDataType = forVar.getDataType(); - this.unsignedConversionType = origDataType; - - // unsignedConversionType is initialized to origDataType, and origDataType may be a non-integral type that doesn't - // have an "unsigned flavor" (such as FLOAT and DOUBLE). Furthermore, unsignedConversionType may start out as - // integral, but then be widened to non-integral (i.e. LONG -> DOUBLE). For these reasons, we cannot rely upon - // unsignedConversionType to store the signedness of the variable. We need a separate field. - this.signedness = origDataType.getSignedness(); - - // In the event of conflict, "unsigned" wins. Potential conflicts include: - // 1. origDataType is unsigned, but variable has "_Unsigned == false" attribute. - // 2. origDataType is signed, but variable has "_Unsigned == true" attribute. - if (signedness == Signedness.SIGNED) { - String unsignedAtt = forVar.attributes().findAttributeString(CDM.UNSIGNED, null); - if (unsignedAtt != null && unsignedAtt.equalsIgnoreCase("true")) { - this.signedness = Signedness.UNSIGNED; - } - } - - if (signedness == Signedness.UNSIGNED) { - // We may need a larger data type to hold the results of the unsigned conversion. - this.unsignedConversionType = nextLarger(origDataType).withSignedness(Signedness.UNSIGNED); - logger.debug("assign unsignedConversionType = {}", unsignedConversionType); - } - - DataType scaleType = null, offsetType = null, validType = null; - logger.debug("{} for Variable = {}", getClass().getSimpleName(), forVar.getShortName()); - - Attribute scaleAtt = forVar.findAttribute(CDM.SCALE_FACTOR); - if (scaleAtt != null && !scaleAtt.isString()) { - scaleType = getAttributeDataType(scaleAtt); - scale = convertUnsigned(scaleAtt.getNumericValue(), scaleType).doubleValue(); - useScaleOffset = enhancements.contains(Enhance.ApplyScaleOffset); - logger.debug("scale = {} type = {}", scale, scaleType); - } - - Attribute offsetAtt = forVar.findAttribute(CDM.ADD_OFFSET); - if (offsetAtt != null && !offsetAtt.isString()) { - offsetType = getAttributeDataType(offsetAtt); - offset = convertUnsigned(offsetAtt.getNumericValue(), offsetType).doubleValue(); - useScaleOffset = enhancements.contains(Enhance.ApplyScaleOffset); - logger.debug("offset = {}", offset); - } - - ////// missing data : valid_range. assume here its in units of unpacked data. correct this below - Attribute validRangeAtt = forVar.findAttribute(CDM.VALID_RANGE); - if (validRangeAtt != null && !validRangeAtt.isString() && validRangeAtt.getLength() > 1) { - validType = getAttributeDataType(validRangeAtt); - validMin = convertUnsigned(validRangeAtt.getNumericValue(0), validType).doubleValue(); - validMax = convertUnsigned(validRangeAtt.getNumericValue(1), validType).doubleValue(); - hasValidRange = true; - logger.debug("valid_range = {} {}", validMin, validMax); - } - - Attribute validMinAtt = forVar.findAttribute(CDM.VALID_MIN); - Attribute validMaxAtt = forVar.findAttribute(CDM.VALID_MAX); - - // Only process the valid_min and valid_max attributes if valid_range isn't present. - if (!hasValidRange) { - if (validMinAtt != null && !validMinAtt.isString()) { - validType = getAttributeDataType(validMinAtt); - validMin = convertUnsigned(validMinAtt.getNumericValue(), validType).doubleValue(); - hasValidMin = true; - logger.debug("valid_min = {}", validMin); - } - - if (validMaxAtt != null && !validMaxAtt.isString()) { - validType = largestOf(validType, getAttributeDataType(validMaxAtt)); - validMax = convertUnsigned(validMaxAtt.getNumericValue(), validType).doubleValue(); - hasValidMax = true; - logger.debug("valid_min = {}", validMax); - } - - if (hasValidMin && hasValidMax) { - hasValidRange = true; - } - } - - /// _FillValue - Attribute fillValueAtt = forVar.findAttribute(CDM.FILL_VALUE); - if (fillValueAtt != null && !fillValueAtt.isString()) { - DataType fillType = getAttributeDataType(fillValueAtt); - fillValue = convertUnsigned(fillValueAtt.getNumericValue(), fillType).doubleValue(); - fillValue = applyScaleOffset(fillValue); // This will fail when _FillValue is CHAR. - hasFillValue = true; - } else { - // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value. - String fileTypeId = forVar.orgFileTypeId; - boolean isNetcdfIosp = DataFormatType.NETCDF.getDescription().equals(fileTypeId) - || DataFormatType.NETCDF4.getDescription().equals(fileTypeId); - - if (isNetcdfIosp) { - if (unsignedConversionType.isNumeric()) { - fillValue = applyScaleOffset(N3iosp.getFillValueDefault(unsignedConversionType)); - hasFillValue = true; - } - } - } - - /// missing_value - Attribute missingValueAtt = forVar.findAttribute(CDM.MISSING_VALUE); - if (missingValueAtt != null) { - if (missingValueAtt.isString()) { - String svalue = missingValueAtt.getStringValue(); - if (origDataType == DataType.CHAR) { - missingValue = new double[1]; - if (svalue.isEmpty()) { - missingValue[0] = 0; - } else { - missingValue[0] = svalue.charAt(0); - } - - hasMissingValue = true; - } else { // not a CHAR - try to fix problem where they use a numeric value as a String attribute - try { - missingValue = new double[1]; - missingValue[0] = Double.parseDouble(svalue); - hasMissingValue = true; - } catch (NumberFormatException ex) { - logger.debug("String missing_value not parseable as double = {}", missingValueAtt.getStringValue()); - } - } - } else { // not a string - DataType missType = getAttributeDataType(missingValueAtt); - - missingValue = new double[missingValueAtt.getLength()]; - for (int i = 0; i < missingValue.length; i++) { - missingValue[i] = convertUnsigned(missingValueAtt.getNumericValue(i), missType).doubleValue(); - missingValue[i] = applyScaleOffset(missingValue[i]); - } - logger.debug("missing_data: {}", Arrays.toString(missingValue)); - - for (double mv : missingValue) { - if (!Double.isNaN(mv)) { - hasMissingValue = true; // dont need to do anything if its already a NaN - break; - } - } - } - } - - /// assign convertedDataType if needed - if (useScaleOffset) { - scaledOffsetType = largestOf(unsignedConversionType, scaleType, offsetType).withSignedness(signedness); - logger.debug("assign scaledOffsetType = {}", scaledOffsetType); - - // validData may be packed or unpacked - if (hasValidData()) { - if (rank(validType) == rank(largestOf(scaleType, offsetType)) - && rank(validType) > rank(unsignedConversionType)) { - // If valid_range is the same type as the wider of scale_factor and add_offset, PLUS - // it is wider than the (packed) data, we know that the valid_range values were stored as unpacked. - // We already assumed that this was the case when we first read the attribute values, so there's - // nothing for us to do here. - } else { - // Otherwise, the valid_range values were stored as packed. So now we must unpack them. - if (hasValidRange || hasValidMin) { - validMin = applyScaleOffset(validMin); - } - if (hasValidRange || hasValidMax) { - validMax = applyScaleOffset(validMax); - } - } - // During the scaling process, it is possible that the valid minimum and maximum values have effectively been - // swapped (for example, when the scale value is negative). Go ahead and check to make sure the valid min is - // actually less than the valid max, and if not, fix it. See https://github.com/Unidata/netcdf-java/issues/572. - if (validMin > validMax) { - double tmp = validMin; - validMin = validMax; - validMax = tmp; - } - } - } - } - - // Get the data type of an attribute. Make it unsigned if the variable is unsigned. - private DataType getAttributeDataType(Attribute attribute) { - DataType dataType = attribute.getDataType(); - if (signedness == Signedness.UNSIGNED) { - // If variable is unsigned, make its integral attributes unsigned too. - dataType = dataType.withSignedness(signedness); - } - return dataType; - } - - /** - * Returns a distinct integer for each of the {@link DataType#isNumeric() numeric} data types that can be used to - * (roughly) order them by the range of the DataType. {@code BYTE < UBYTE < SHORT < USHORT < INT < UINT < - * LONG < ULONG < FLOAT < DOUBLE}. {@code -1} will be returned for all non-numeric data types. - * - * @param dataType a numeric data type. - * @return a distinct integer for each of the numeric data types that can be used to (roughly) order them by size. - */ - public static int rank(DataType dataType) { - if (dataType == null) { - return -1; - } - - switch (dataType) { - case BYTE: - return 0; - case UBYTE: - return 1; - case SHORT: - return 2; - case USHORT: - return 3; - case INT: - return 4; - case UINT: - return 5; - case LONG: - return 6; - case ULONG: - return 7; - case FLOAT: - return 8; - case DOUBLE: - return 9; - default: - return -1; - } - } - - /** - * Returns the data type that is the largest among the arguments. Relative sizes of data types are determined via - * {@link #rank(DataType)}. - * - * @param dataTypes an array of numeric data types. - * @return the data type that is the largest among the arguments. - */ - public static DataType largestOf(DataType... dataTypes) { - DataType widest = null; - for (DataType dataType : dataTypes) { - if (widest == null) { - widest = dataType; - } else if (rank(dataType) > rank(widest)) { - widest = dataType; - } - } - return widest; - } - - /** - * Returns the smallest numeric data type that: - *

      - *
    1. can hold a larger integer than {@code dataType} can
      2. - *
    2. if integral, has the same signedness as {@code dataType}
      3. - *
    - * The relative sizes of data types are determined in a manner consistent with {@link #rank(DataType)}. - *

    - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - * - *
    ArgumentResult
    BYTESHORT
    UBYTEUSHORT
    SHORTINT
    USHORTUINT
    INTLONG
    UINTULONG
    LONGDOUBLE
    ULONGDOUBLE
    Any other data typeJust return argument
    - *

    - * The returned type is intended to be just big enough to hold the result of performing an unsigned conversion of a - * value of the smaller type. For example, the {@code byte} value {@code -106} equals {@code 150} when interpreted - * as unsigned. That won't fit in a (signed) {@code byte}, but it will fit in a {@code short}. - * - * @param dataType an integral data type. - * @return the next larger type. - */ - public static DataType nextLarger(DataType dataType) { - switch (dataType) { - case BYTE: - return SHORT; - case UBYTE: - return USHORT; - case SHORT: - return INT; - case USHORT: - return UINT; - case INT: - return LONG; - case UINT: - return ULONG; - case LONG: - case ULONG: - return DOUBLE; - default: - return dataType; - } - } - - @Override - public double getScaleFactor() { - return scale; - } - - @Override - public double getOffset() { - return offset; - } - - @Override - public Signedness getSignedness() { - return signedness; - } - - @Override - public DataType getScaledOffsetType() { - return scaledOffsetType; - } - - @Nonnull - @Override - public DataType getUnsignedConversionType() { - return unsignedConversionType; - } - - @Override - public boolean hasValidData() { - return hasValidRange || hasValidMin || hasValidMax; - } - - @Override - public double getValidMin() { - return validMin; - } - - @Override - public double getValidMax() { - return validMax; - } - - @Override - public boolean isInvalidData(double val) { - // valid_min and valid_max may have been multiplied by scale_factor, which could be a float, not a double. - // That potential loss of precision means that we cannot do the nearlyEquals() comparison with - // Misc.defaultMaxRelativeDiffDouble. - boolean greaterThanOrEqualToValidMin = - Misc.nearlyEquals(val, validMin, Misc.defaultMaxRelativeDiffFloat) || val > validMin; - boolean lessThanOrEqualToValidMax = - Misc.nearlyEquals(val, validMax, Misc.defaultMaxRelativeDiffFloat) || val < validMax; - - return (hasValidRange && !(greaterThanOrEqualToValidMin && lessThanOrEqualToValidMax)) - || (hasValidMin && !greaterThanOrEqualToValidMin) || (hasValidMax && !lessThanOrEqualToValidMax); - } - - @Override - public boolean hasFillValue() { - return hasFillValue; - } - - @Override - public boolean isFillValue(double val) { - return hasFillValue && Misc.nearlyEquals(val, fillValue, Misc.defaultMaxRelativeDiffFloat); - } - - @Override - public double getFillValue() { - return fillValue; - } - - @Override - public boolean hasScaleOffset() { - return useScaleOffset; - } - - @Override - public boolean hasMissingValue() { - return hasMissingValue; - } - - @Override - public boolean isMissingValue(double val) { - if (!hasMissingValue) { - return false; - } - for (double aMissingValue : missingValue) { - if (Misc.nearlyEquals(val, aMissingValue, Misc.defaultMaxRelativeDiffFloat)) { - return true; - } - } - return false; - } - - @Override - public double[] getMissingValues() { - return missingValue; - } - - @Override - public void setFillValueIsMissing(boolean b) { - this.fillValueIsMissing = b; - } - - @Override - public void setInvalidDataIsMissing(boolean b) { - this.invalidDataIsMissing = b; - } - - @Override - public void setMissingDataIsMissing(boolean b) { - this.missingDataIsMissing = b; - } - - @Override - public boolean hasMissing() { - return (invalidDataIsMissing && hasValidData()) || (fillValueIsMissing && hasFillValue()) - || (missingDataIsMissing && hasMissingValue()); - } - - @Override - public boolean isMissing(double val) { - if (Double.isNaN(val)) { - return true; - } else { - return (missingDataIsMissing && isMissingValue(val)) || (fillValueIsMissing && isFillValue(val)) - || (invalidDataIsMissing && isInvalidData(val)); - } - } - - - @Override - public Number convertUnsigned(Number value) { - return convertUnsigned(value, signedness); - } - - private static Number convertUnsigned(Number value, DataType dataType) { - return convertUnsigned(value, dataType.getSignedness()); - } - - private static Number convertUnsigned(Number value, Signedness signedness) { - if (signedness == Signedness.UNSIGNED) { - // Handle integral types that should be treated as unsigned by widening them if necessary. - return DataType.widenNumberIfNegative(value); - } else { - return value; - } - } - - @Override - public Array convertUnsigned(Array in) { - return convert(in, true, false, false); - } - - @Override - public double applyScaleOffset(Number value) { - double convertedValue = value.doubleValue(); - return useScaleOffset ? scale * convertedValue + offset : convertedValue; - } - - @Override - public Array applyScaleOffset(Array in) { - return convert(in, false, true, false); - } - - @Override - public Number convertMissing(Number value) { - return isMissing(value.doubleValue()) ? Double.NaN : value; - } - - @Override - public Array convertMissing(Array in) { - return convert(in, false, false, true); - } - - @Override - public Array convert(Array in, boolean convertUnsigned, boolean applyScaleOffset, boolean convertMissing) { - if (!in.getDataType().isNumeric() || (!convertUnsigned && !applyScaleOffset && !convertMissing)) { - return in; // Nothing to do! - } - - if (getSignedness() == Signedness.SIGNED) { - convertUnsigned = false; - } - if (!hasScaleOffset()) { - applyScaleOffset = false; - } - - DataType outType = origDataType; - if (convertUnsigned) { - outType = getUnsignedConversionType(); - } - if (applyScaleOffset) { - outType = getScaledOffsetType(); - } - - if (outType != DataType.FLOAT && outType != DataType.DOUBLE) { - convertMissing = false; - } - - Array out = Array.factory(outType, in.getShape()); - IndexIterator iterIn = in.getIndexIterator(); - IndexIterator iterOut = out.getIndexIterator(); - - while (iterIn.hasNext()) { - Number value = (Number) iterIn.getObjectNext(); - - if (convertUnsigned) { - value = convertUnsigned(value); - } - if (applyScaleOffset) { - value = applyScaleOffset(value); - } - if (convertMissing) { - value = convertMissing(value); - } - - iterOut.setObjectNext(value); - } - - return out; - } -} diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index 362c91d22d..44bc84b1e0 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -233,22 +233,21 @@ public void enhance(Set enhancements) { if (this.enhanceMode.contains(Enhance.ConvertEnums) && dataType.isEnum()) { setDataType(DataType.STRING); // LOOK promote data type to STRING ???? - return; // We can return here, because the other conversions don't apply to STRING. } // Initialize EnhanceScaleMissingUnsignedImpl. We can't do this in the constructors because this object may not // contain all of the relevant attributes at that time. NcMLReader is an example of this: the VariableDS is // constructed first, and then Attributes are added to it later. - this.scaleMissingUnsignedProxy = new EnhanceScaleMissingUnsignedImpl(this, this.enhanceMode); - - if (this.enhanceMode.contains(Enhance.ConvertUnsigned)) { - // We may need a larger data type to hold the results of the unsigned conversion. - setDataType(scaleMissingUnsignedProxy.getUnsignedConversionType()); + if (this.enhanceMode.contains(Enhance.ConvertUnsigned) && !dataType.isString()) { + this.unsignedConversion = UnsignedConversion.createFromVar(this); + this.dataType = unsignedConversion.getOutType(); } - - if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR) - && scaleMissingUnsignedProxy.hasScaleOffset()) { - setDataType(scaleMissingUnsignedProxy.getScaledOffsetType()); + if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR)) { + this.scaleOffset = ScaleOffset.createFromVariable(this); + this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType; + } + if (this.enhanceMode.contains(Enhance.ConvertMissing)) { + this.convertMissing = ConvertMissing.createFromVariable(this); } } @@ -272,13 +271,13 @@ Array convert(Array data, Set enhancements) { if (this.isVariableLength) { return data; } - if (enhancements.contains(Enhance.ConvertUnsigned)) { + if (enhancements.contains(Enhance.ConvertUnsigned) && unsignedConversion != null) { data = unsignedConversion.convertUnsigned(data); } if (enhancements.contains(Enhance.ApplyScaleOffset) && scaleOffset != null) { data = scaleOffset.removeScaleOffset(data); } - if (enhancements.contains(Enhance.ConvertMissing)) { + if (enhancements.contains(Enhance.ConvertMissing) && convertMissing != null) { data = convertMissing.convertMissing(data); } return data; @@ -543,7 +542,7 @@ public Array getMissingDataArray(int[] shape) { } Array array = Array.factoryConstant(getDataType(), shape, storage); - if (convertMissing.hasFillValue()) { + if (convertMissing != null && convertMissing.hasFillValue()) { array.setObject(0, convertMissing.getFillValue()); } return array; @@ -555,20 +554,23 @@ public Array getMissingDataArray(int[] shape) { * @param f put info here */ public void showScaleMissingProxy(Formatter f) { - f.format("has missing = %s%n", convertMissing.hasMissing()); - if (convertMissing.hasMissing()) { - if (convertMissing.hasMissingValue()) { - f.format(" missing value(s) = "); - for (double d : convertMissing.getMissingValues()) - f.format(" %f", d); - f.format("%n"); + f.format("has missing = %s%n", convertMissing != null); + if (convertMissing != null) { + if (convertMissing.hasMissing()) { + if (convertMissing.hasMissingValue()) { + f.format(" missing value(s) = "); + for (double d : convertMissing.getMissingValues()) + f.format(" %f", d); + f.format("%n"); + } + if (convertMissing.hasFillValue()) + f.format(" fillValue = %f%n", convertMissing.getFillValue()); + if (convertMissing.hasValidData()) + f.format(" valid min/max = [%f,%f]%n", convertMissing.getValidMin(), convertMissing.getValidMax()); } - if (convertMissing.hasFillValue()) - f.format(" fillValue = %f%n", convertMissing.getFillValue()); - if (convertMissing.hasValidData()) - f.format(" valid min/max = [%f,%f]%n", convertMissing.getValidMin(), convertMissing.getValidMax()); + f.format("FillValue or default = %s%n", convertMissing.getFillValue()); } - f.format("FillValue or default = %s%n", convertMissing.getFillValue()); + f.format("%nhas scale/offset = %s%n", scaleOffset != null); if (scaleOffset != null) { @@ -637,77 +639,83 @@ public double getOffset() { @Override public boolean hasMissing() { - // TODO: null or all false values? - return convertMissing.hasMissing(); + return convertMissing != null ? convertMissing.hasMissing() : false; } @Override public boolean isMissing(double val) { - return convertMissing.isMissing(val); + if (Double.isNaN(val)) { + return true; + } + return convertMissing != null ? convertMissing.isMissing(val) : false; } @Override public boolean hasValidData() { - return convertMissing.hasMissingValue(); + return convertMissing != null ? convertMissing.hasMissingValue() : false; } @Override public double getValidMin() { - return convertMissing.getValidMin(); + return convertMissing != null ? convertMissing.getValidMin() : -Double.MAX_VALUE; } @Override public double getValidMax() { - return convertMissing.getValidMax(); + return convertMissing != null ? convertMissing.getValidMax() : Double.MAX_VALUE; } @Override public boolean isInvalidData(double val) { - return convertMissing.isInvalidData(val); + return convertMissing != null ? convertMissing.isInvalidData(val) : false; } @Override public boolean hasFillValue() { - return convertMissing.hasFillValue(); + return convertMissing != null ? convertMissing.hasFillValue() : false; } @Override public double getFillValue() { - return convertMissing.getFillValue(); + return convertMissing != null ? convertMissing.getFillValue() : Double.MAX_VALUE; } @Override public boolean isFillValue(double val) { - return convertMissing.isFillValue(val); + return convertMissing != null ? convertMissing.isFillValue(val) : false; } @Override public boolean hasMissingValue() { - return convertMissing.hasMissingValue(); + return convertMissing != null ? convertMissing.hasMissingValue() : false; } @Override public double[] getMissingValues() { - return convertMissing.getMissingValues(); + return convertMissing != null ? convertMissing.getMissingValues() : new double[] {0}; } @Override public boolean isMissingValue(double val) { - return convertMissing.isMissingValue(val); + return convertMissing != null ? convertMissing.isMissingValue(val) : false; } /** @deprecated Use NetcdfDataset.builder() */ @Deprecated @Override public void setFillValueIsMissing(boolean b) { - scaleMissingUnsignedProxy.setFillValueIsMissing(b); + if (convertMissing != null) { + convertMissing.setFillValueIsMissing(b); + } } /** @deprecated Use NetcdfDataset.builder() */ @Deprecated @Override public void setInvalidDataIsMissing(boolean b) { - scaleMissingUnsignedProxy.setInvalidDataIsMissing(b); + if (convertMissing != null) { + convertMissing.setInvalidDataIsMissing(b); + } } public boolean missingDataIsMissing() { @@ -726,7 +734,9 @@ public boolean invalidDataIsMissing() { @Deprecated @Override public void setMissingDataIsMissing(boolean b) { - scaleMissingUnsignedProxy.setMissingDataIsMissing(b); + if (convertMissing != null) { + convertMissing.setMissingDataIsMissing(b); + } } @Nullable @@ -767,12 +777,12 @@ public Array convertUnsigned(Array in) { @Override public Number convertMissing(Number value) { - return convertMissing.convertMissing(value); + return convertMissing != null ? convertMissing.convertMissing(value) : value; } @Override public Array convertMissing(Array in) { - return convertMissing.convertMissing(in); + return convertMissing != null ? convertMissing.convertMissing(in) : in; } /** @@ -781,14 +791,19 @@ public Array convertMissing(Array in) { @Override @Deprecated public Array convert(Array in, boolean convertUnsigned, boolean applyScaleOffset, boolean convertMissing) { - return scaleMissingUnsignedProxy.convert(in, convertUnsigned, applyScaleOffset, convertMissing); + if (this.unsignedConversion != null) { + in = unsignedConversion.convertUnsigned(in); + } + if (this.scaleOffset != null) { + in = scaleOffset.removeScaleOffset(in); + } + return convertMissing(in); } //////////////////////////////////////////////////////////////////////////////////////////// // TODO remove in version 6. private EnhancementsImpl enhanceProxy; private List coordSysNames; - private EnhanceScaleMissingUnsignedImpl scaleMissingUnsignedProxy = new EnhanceScaleMissingUnsignedImpl(); // TODO make immutable in version 6 private UnsignedConversion unsignedConversion; @@ -823,18 +838,18 @@ protected VariableDS(Builder builder, Group parentGroup) { if (this.enhanceMode.contains(Enhance.ConvertEnums) && dataType.isEnum()) { this.dataType = DataType.STRING; // LOOK promote enum data type to STRING ???? - } else { - if (this.enhanceMode.contains(Enhance.ConvertUnsigned) && !dataType.isEnum()) { - this.unsignedConversion = UnsignedConversion.createFromVar(this); - this.dataType = unsignedConversion.getOutType(); - } - if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR)) { - this.scaleOffset = ScaleOffset.createFromVariable(this); - this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType; - } - if (this.enhanceMode.contains(Enhance.ConvertMissing)) { - this.convertMissing = ConvertMissing.createFromVariable(this); - } + } + + if (this.enhanceMode.contains(Enhance.ConvertUnsigned) && !dataType.isString()) { + this.unsignedConversion = UnsignedConversion.createFromVar(this); + this.dataType = unsignedConversion.getOutType(); + } + if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR)) { + this.scaleOffset = ScaleOffset.createFromVariable(this); + this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType; + } + if (this.enhanceMode.contains(Enhance.ConvertMissing)) { + this.convertMissing = ConvertMissing.createFromVariable(this); } // We have to complete this after the NetcdfDataset is built. diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java index 1a52e75564..602becfc25 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java @@ -10,8 +10,6 @@ import ucar.nc2.iosp.netcdf3.N3iosp; import ucar.nc2.util.Misc; -import java.util.Arrays; - public class ConvertMissing { private boolean hasValidMin, hasValidMax; @@ -243,12 +241,32 @@ public boolean isMissing(double val) { } } + @Deprecated + public void setFillValueIsMissing(boolean b) { + this.fillValueIsMissing = b; + } + + @Deprecated + public void setInvalidDataIsMissing(boolean b) { + this.invalidDataIsMissing = b; + } + + @Deprecated + public void setMissingDataIsMissing(boolean b) { + this.missingDataIsMissing = b; + } + public Number convertMissing(Number value) { return isMissing(value.doubleValue()) ? Double.NaN : value; } public Array convertMissing(Array in) { - Array out = Array.factory(in.getDataType(), in.getShape()); + DataType type = in.getDataType(); + if (!type.isNumeric()) { + return in; + } + + Array out = Array.factory(type, in.getShape()); IndexIterator iterIn = in.getIndexIterator(); IndexIterator iterOut = out.getIndexIterator(); diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java index dcfefb94d4..acca827b7d 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java @@ -77,7 +77,7 @@ public static ScaleOffset createFromVariable(VariableDS var) { Attribute scaleAtt = var.findAttribute(CDM.SCALE_FACTOR); if (scaleAtt != null && !scaleAtt.isString()) { scaleType = FilterHelpers.getAttributeDataType(scaleAtt, signedness); - scale = var.convertUnsigned(scaleAtt.getNumericValue()).doubleValue(); + scale = 1 / var.convertUnsigned(scaleAtt.getNumericValue()).doubleValue(); } Attribute offsetAtt = var.findAttribute(CDM.ADD_OFFSET); @@ -92,8 +92,8 @@ public static ScaleOffset createFromVariable(VariableDS var) { Map scaleOffsetProps = new HashMap<>(); scaleOffsetProps.put(ScaleOffset.Keys.OFFSET_KEY, offset); scaleOffsetProps.put(ScaleOffset.Keys.SCALE_KEY, scale); - scaleOffsetProps.put(ScaleOffset.Keys.DTYPE_KEY, origDataType); - scaleOffsetProps.put(ScaleOffset.Keys.ASTYPE_KEY, scaledOffsetType); + scaleOffsetProps.put(ScaleOffset.Keys.DTYPE_KEY, scaledOffsetType); + scaleOffsetProps.put(ScaleOffset.Keys.ASTYPE_KEY, origDataType); return new ScaleOffset(scaleOffsetProps); } return null; @@ -154,26 +154,32 @@ public double getOffset() { } public DataType getScaledOffsetType() { - return this.astype; + return this.dtype; } @Override public byte[] encode(byte[] dataIn) { - if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return dataIn; } + if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { + return dataIn; + } Array out = applyScaleOffset(FilterHelpers.bytesToArray(dataIn, dtype, dtypeOrder)); return FilterHelpers.arrayToBytes(out, astype, astypeOrder); } @Override public byte[] decode(byte[] dataIn) { - if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return dataIn; } + if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { + return dataIn; + } Array out = removeScaleOffset(FilterHelpers.bytesToArray(dataIn, astype, astypeOrder)); return FilterHelpers.arrayToBytes(out, dtype, dtypeOrder); } // not used anywhere yet public Array applyScaleOffset(Array in) { - if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return in; } + if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { + return in; + } // use wider datatype if unsigned DataType outType = astype; if (astype.getSignedness() == Signedness.UNSIGNED) { @@ -197,7 +203,9 @@ public Array applyScaleOffset(Array in) { } public Array removeScaleOffset(Array in) { - if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return in; } + if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { + return in; + } // use wider datatype if unsigned DataType outType = dtype; if (dtype.getSignedness() == Signedness.UNSIGNED) { diff --git a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java index 8c1da9ea3a..8d71981736 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java @@ -3,7 +3,6 @@ import ucar.ma2.Array; import ucar.ma2.DataType; import ucar.ma2.IndexIterator; -import ucar.nc2.Variable; import ucar.nc2.constants.CDM; import ucar.nc2.dataset.VariableDS; @@ -47,7 +46,6 @@ public DataType getOutType() { } public Number convertUnsigned(Number value) { - Number newVal = DataType.widenNumberIfNegative(value); return DataType.widenNumberIfNegative(value); } diff --git a/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java index e54256d4a2..b141629128 100644 --- a/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java +++ b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java @@ -8,7 +8,6 @@ import ucar.ma2.DataType; import ucar.ma2.InvalidRangeException; import ucar.nc2.Attribute; -import ucar.nc2.Dimension; import ucar.nc2.Variable; import ucar.nc2.constants.CDM; import ucar.nc2.dataset.NetcdfDataset; @@ -21,128 +20,125 @@ public class TestEnhancements { - private static NetcdfDataset ncd; - - private static short[] signedShorts = new short[] {123, 124, 125, 126, 127, -128, -127, -126, -125, -124}; - - private static final float VALID_MIN = 100; - private static final float VALID_MAX = 200; - private static final float FILL_VALUE = 150; - private static float[] missingData = new float[]{90, 100, Float.NaN, 120, 130, 140, 150, 190, 200, 210}; - - private static final Short SIGNED_SCALED_MAX = -126; - private static final Short SIGNED_SCALED_FILL_VALUE = -128; - - @ClassRule - public static final TemporaryFolder tempFolder = new TemporaryFolder(); - - @BeforeClass - public static void setUp() throws IOException , InvalidRangeException { - final int data_len = 10; - String filePath = tempFolder.newFile().getAbsolutePath(); - NetcdfFormatWriter.Builder builder = NetcdfFormatWriter.createNewNetcdf3(filePath); - Dimension dim = builder.addDimension("dim", data_len); - - Array signedData = Array.factory(DataType.SHORT, new int[]{data_len}, signedShorts); - // signed shorts - builder.addVariable("signedVar", DataType.SHORT, "dim"); - // unsigned shorts - builder.addVariable("unsignedVar", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.UNSIGNED, "true")); - - // scaled and offset data - builder.addVariable("scaleOffsetVar", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.SCALE_FACTOR, .1)) - .addAttribute(new Attribute(CDM.ADD_OFFSET, 10)); - - Array missingDataArray = Array.factory(DataType.FLOAT, new int[]{data_len}, missingData); - // Data with min - builder.addVariable("validMin", DataType.FLOAT, "dim").addAttribute(new Attribute(CDM.VALID_MIN, VALID_MIN)); - // Data with min and max - builder.addVariable("validMinMax", DataType.FLOAT, "dim") - .addAttribute(new Attribute(CDM.VALID_MIN, VALID_MIN)) - .addAttribute(new Attribute(CDM.VALID_MAX, VALID_MAX)); - // Data with range and fill value - Array range = Array.factory(DataType.FLOAT, new int[]{2}, new float[]{VALID_MIN, VALID_MAX}); - builder.addVariable("validRange", DataType.FLOAT, "dim") - .addAttribute(Attribute.builder(CDM.VALID_RANGE).setValues(range).build()) - .addAttribute(Attribute.builder(CDM.FILL_VALUE).setNumericValue(FILL_VALUE, true).build()); - - // unsigned, scaled/offset, and missing value - Array enhanceAllArray = Array.factory(DataType.SHORT, new int[]{data_len}, signedShorts); - builder.addVariable("enhanceAll", DataType.SHORT, "dim") - .addAttribute(new Attribute(CDM.UNSIGNED, "true")) - .addAttribute(new Attribute(CDM.SCALE_FACTOR, .1)) - .addAttribute(new Attribute(CDM.ADD_OFFSET, 10)) - .addAttribute(new Attribute(CDM.VALID_MAX, SIGNED_SCALED_MAX)) - .addAttribute(new Attribute(CDM.FILL_VALUE, SIGNED_SCALED_FILL_VALUE)); - - // write data - NetcdfFormatWriter writer = builder.build(); - writer.write(writer.findVariable("signedVar"), new int[1], signedData); - writer.write(writer.findVariable("unsignedVar"), new int[1], signedData); - writer.write(writer.findVariable("scaleOffsetVar"), new int[1], signedData); - writer.write(writer.findVariable("validMin"), new int[1], missingDataArray); - writer.write(writer.findVariable("validMinMax"), new int[1], missingDataArray); - writer.write(writer.findVariable("validRange"), new int[1], missingDataArray); - writer.write(writer.findVariable("enhanceAll"), new int[1], enhanceAllArray); - writer.close(); - ncd = NetcdfDatasets.openDataset(filePath); - } - - @Test - public void testUnsignedConversion() throws IOException { - final int[] unsignedValues = new int[]{123, 124, 125, 126, 127, -128, -127, -126, -125, -126}; - // signed var - Variable v = ncd.findVariable("signedVar"); - Array data = v.read(); - assertThat(data.isUnsigned()).isFalse(); - assertThat(data.getDataType()).isEqualTo(DataType.SHORT); - assertThat((short[])data.copyTo1DJavaArray()).isEqualTo(signedShorts); - - // var with unsigned data type - v = ncd.findVariable("unsignedVar"); - data = v.read(); - assertThat(data.isUnsigned()).isTrue(); - assertThat(data.getDataType()).isEqualTo(DataType.UINT); - assertThat((int[])data.copyTo1DJavaArray()).isEqualTo(unsignedValues); - } - - @Test - public void testScaleOffset() throws IOException { - final double[] expected = new double[]{1240, 1250, 1260, 1270, 1280, -1270, -1260, -1250, -1240, -1230}; - // signed var - Variable v = ncd.findVariable("scaleOffsetVar"); - Array data = v.read(); - assertThat(data.isUnsigned()).isFalse(); - assertThat(data.getDataType()).isEqualTo(DataType.SHORT); - assertThat((double[])data.copyTo1DJavaArray()).isEqualTo(expected); - } - - @Test - public void testConvertMissing() throws IOException { - // var with valid min - float[] expected = new float[]{Float.NaN, 100, Float.NaN, 120, 130, 140, 150, 190, 200, 210}; - Variable v = ncd.findVariable("validMin"); - Array data = v.read(); - assertThat((float[])data.copyTo1DJavaArray()).isEqualTo(expected); - - // var with valid min and max - expected = new float[]{Float.NaN, 100, Float.NaN, 120, 130, 140, 150, 190, 200, Float.NaN}; - v = ncd.findVariable("validMinMax"); - data = v.read(); - assertThat((float[])data.copyTo1DJavaArray()).isEqualTo(expected); - - // var with valid range and fill value - expected = new float[]{Float.NaN, 100, Float.NaN, 120, 130, 140, Float.NaN, 190, 200, Float.NaN}; - v = ncd.findVariable("validRange"); - data = v.read(); - assertThat((float[])data.copyTo1DJavaArray()).isEqualTo(expected); - } - - @Test - public void testCombinedEnhancements() throws IOException { - int[] expected = new int[]{1240, 1250, 1260, 1270, 1280, 0, 1278, 1260, 0, 0}; - Variable v = ncd.findVariable("enhanceAll"); - Array data = v.read(); - assertThat((int[])data.copyTo1DJavaArray()).isEqualTo(expected); - } + private static NetcdfDataset ncd; + + private static short[] signedShorts = new short[] {-5, -4, -3, -2, -1, 0, 1, 2, 3, 4}; + + private static final float VALID_MIN = 100; + private static final float VALID_MAX = 200; + private static final float FILL_VALUE = 150; + private static float[] missingData = new float[] {90, 100, Float.NaN, 120, 130, 140, 150, 190, 200, 210}; + + private static final Short SIGNED_SCALED_MAX = -2; + private static final Short SIGNED_SCALED_FILL_VALUE = -4; + + @ClassRule + public static final TemporaryFolder tempFolder = new TemporaryFolder(); + + @BeforeClass + public static void setUp() throws IOException, InvalidRangeException { + final int data_len = 10; + String filePath = tempFolder.newFile().getAbsolutePath(); + NetcdfFormatWriter.Builder builder = NetcdfFormatWriter.createNewNetcdf3(filePath); + builder.addDimension("dim", data_len); + + Array signedData = Array.factory(DataType.SHORT, new int[] {data_len}, signedShorts); + // signed shorts + builder.addVariable("signedVar", DataType.SHORT, "dim"); + // unsigned shorts + builder.addVariable("unsignedVar", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.UNSIGNED, "true")); + + // scaled and offset data + builder.addVariable("scaleOffsetVar", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.SCALE_FACTOR, 10)) + .addAttribute(new Attribute(CDM.ADD_OFFSET, 10)); + + Array missingDataArray = Array.factory(DataType.FLOAT, new int[] {data_len}, missingData); + // Data with min + builder.addVariable("validMin", DataType.FLOAT, "dim").addAttribute(new Attribute(CDM.VALID_MIN, VALID_MIN)); + // Data with min and max + builder.addVariable("validMinMax", DataType.FLOAT, "dim").addAttribute(new Attribute(CDM.VALID_MIN, VALID_MIN)) + .addAttribute(new Attribute(CDM.VALID_MAX, VALID_MAX)); + // Data with range and fill value + Array range = Array.factory(DataType.FLOAT, new int[] {2}, new float[] {VALID_MIN, VALID_MAX}); + builder.addVariable("validRange", DataType.FLOAT, "dim") + .addAttribute(Attribute.builder(CDM.VALID_RANGE).setValues(range).build()) + .addAttribute(Attribute.builder(CDM.FILL_VALUE).setNumericValue(FILL_VALUE, true).build()); + + // unsigned, scaled/offset, and missing value + Array enhanceAllArray = Array.factory(DataType.SHORT, new int[] {data_len}, signedShorts); + builder.addVariable("enhanceAll", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.UNSIGNED, "true")) + .addAttribute(new Attribute(CDM.SCALE_FACTOR, 10)).addAttribute(new Attribute(CDM.ADD_OFFSET, 10)) + .addAttribute(new Attribute(CDM.VALID_MAX, SIGNED_SCALED_MAX)) + .addAttribute(new Attribute(CDM.FILL_VALUE, SIGNED_SCALED_FILL_VALUE)); + + // write data + NetcdfFormatWriter writer = builder.build(); + writer.write(writer.findVariable("signedVar"), new int[1], signedData); + writer.write(writer.findVariable("unsignedVar"), new int[1], signedData); + writer.write(writer.findVariable("scaleOffsetVar"), new int[1], signedData); + writer.write(writer.findVariable("validMin"), new int[1], missingDataArray); + writer.write(writer.findVariable("validMinMax"), new int[1], missingDataArray); + writer.write(writer.findVariable("validRange"), new int[1], missingDataArray); + writer.write(writer.findVariable("enhanceAll"), new int[1], enhanceAllArray); + writer.close(); + ncd = NetcdfDatasets.openDataset(filePath); + } + + @Test + public void testUnsignedConversion() throws IOException { + final int[] unsignedValues = new int[] {65531, 65532, 65533, 65534, 65535, 0, 1, 2, 3, 4}; + // signed var + Variable v = ncd.findVariable("signedVar"); + Array data = v.read(); + assertThat(data.isUnsigned()).isFalse(); + assertThat(data.getDataType()).isEqualTo(DataType.SHORT); + assertThat((short[]) data.copyTo1DJavaArray()).isEqualTo(signedShorts); + + // var with unsigned data type + v = ncd.findVariable("unsignedVar"); + data = v.read(); + assertThat(data.isUnsigned()).isTrue(); + assertThat(data.getDataType()).isEqualTo(DataType.UINT); + assertThat((int[]) data.copyTo1DJavaArray()).isEqualTo(unsignedValues); + } + + @Test + public void testScaleOffset() throws IOException { + final int[] expected = new int[] {-40, -30, -20, -10, 0, 10, 20, 30, 40, 50}; + // signed var + Variable v = ncd.findVariable("scaleOffsetVar"); + Array data = v.read(); + assertThat(data.isUnsigned()).isFalse(); + assertThat(data.getDataType()).isEqualTo(DataType.INT); + assertThat((int[]) data.copyTo1DJavaArray()).isEqualTo(expected); + } + + @Test + public void testConvertMissing() throws IOException { + // var with valid min + float[] expected = new float[] {Float.NaN, 100, Float.NaN, 120, 130, 140, 150, 190, 200, 210}; + Variable v = ncd.findVariable("validMin"); + Array data = v.read(); + assertThat((float[]) data.copyTo1DJavaArray()).isEqualTo(expected); + + // var with valid min and max + expected = new float[] {Float.NaN, 100, Float.NaN, 120, 130, 140, 150, 190, 200, Float.NaN}; + v = ncd.findVariable("validMinMax"); + data = v.read(); + assertThat((float[]) data.copyTo1DJavaArray()).isEqualTo(expected); + + // var with valid range and fill value + expected = new float[] {Float.NaN, 100, Float.NaN, 120, 130, 140, Float.NaN, 190, 200, Float.NaN}; + v = ncd.findVariable("validRange"); + data = v.read(); + assertThat((float[]) data.copyTo1DJavaArray()).isEqualTo(expected); + } + + @Test + public void testCombinedEnhancements() throws IOException { + int[] expected = new int[] {655320, 0, 655340, 655350, 0, 10, 20, 30, 40, 50}; + Variable v = ncd.findVariable("enhanceAll"); + Array data = v.read(); + assertThat((int[]) data.copyTo1DJavaArray()).isEqualTo(expected); + } } diff --git a/cdm/misc/src/test/java/ucar/nc2/TestSequence.java b/cdm/misc/src/test/java/ucar/nc2/TestSequence.java index 41093ce757..2e1aa5ecd3 100644 --- a/cdm/misc/src/test/java/ucar/nc2/TestSequence.java +++ b/cdm/misc/src/test/java/ucar/nc2/TestSequence.java @@ -6,30 +6,19 @@ package ucar.nc2; import com.google.common.collect.Sets; -import java.io.File; import java.io.IOException; -import java.lang.invoke.MethodHandles; import java.util.Arrays; import java.util.List; import org.junit.Assert; import org.junit.Test; import org.junit.experimental.categories.Category; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import ucar.ma2.Array; -import ucar.ma2.ArrayFloat; -import ucar.ma2.ArraySequence; -import ucar.ma2.ArrayStructure; -import ucar.ma2.MAMath; import ucar.ma2.StructureData; import ucar.ma2.StructureDataIterator; -import ucar.nc2.dataset.NetcdfDatasets; import ucar.unidata.util.test.TestDir; import ucar.unidata.util.test.category.NeedsCdmUnitTest; /** Test Sequences constructed when reading NLDN datasets. */ public class TestSequence { - private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @Test @Category(NeedsCdmUnitTest.class) From 3f412b3ea6ecb711c3469bda1aa103b28afc159a Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Fri, 28 Apr 2023 15:39:50 -0700 Subject: [PATCH 10/24] apply fill value without using enhance modes --- .../java/ucar/nc2/dataset/VariableDS.java | 56 +++++++++++++++++-- .../java/ucar/nc2/filter/ConvertMissing.java | 25 +-------- .../test/java/ucar/nc2/ncml/TestEnhance.java | 4 -- 3 files changed, 55 insertions(+), 30 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index 44bc84b1e0..30a6f1f4e7 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -9,12 +9,14 @@ import ucar.ma2.*; import ucar.nc2.*; import ucar.nc2.constants.CDM; +import ucar.nc2.constants.DataFormatType; import ucar.nc2.dataset.NetcdfDataset.Enhance; import ucar.nc2.filter.ConvertMissing; import ucar.nc2.filter.FilterHelpers; import ucar.nc2.filter.ScaleOffset; import ucar.nc2.filter.UnsignedConversion; import ucar.nc2.internal.dataset.CoordinatesHelper; +import ucar.nc2.iosp.netcdf3.N3iosp; import ucar.nc2.util.CancelTask; import javax.annotation.Nullable; import java.io.IOException; @@ -170,6 +172,8 @@ protected VariableDS(VariableDS vds, boolean isCopy) { this.unsignedConversion = vds.unsignedConversion; this.scaleOffset = vds.scaleOffset; this.convertMissing = vds.convertMissing; + this.fillValue = vds.getFillValue(); + this.hasFillValue = vds.hasFillValue(); // Add this so that old VariableDS units agrees with new VariableDS units. String units = vds.getUnitsString(); @@ -242,6 +246,26 @@ public void enhance(Set enhancements) { this.unsignedConversion = UnsignedConversion.createFromVar(this); this.dataType = unsignedConversion.getOutType(); } + // need fill value info before convertMissing + Attribute fillValueAtt = findAttribute(CDM.FILL_VALUE); + if (fillValueAtt != null && !fillValueAtt.isString()) { + fillValue = convertUnsigned(fillValueAtt.getNumericValue()).doubleValue(); + fillValue = applyScaleOffset(fillValue); // This will fail when _FillValue is CHAR. + hasFillValue = true; + } else { + // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value. + String fileTypeId = getFileTypeId(); + boolean isNetcdfIosp = DataFormatType.NETCDF.getDescription().equals(fileTypeId) + || DataFormatType.NETCDF4.getDescription().equals(fileTypeId); + + if (isNetcdfIosp) { + DataType unsignedConversionType = getUnsignedConversionType(); + if (unsignedConversionType.isNumeric()) { + fillValue = applyScaleOffset(N3iosp.getFillValueDefault(unsignedConversionType)); + hasFillValue = true; + } + } + } if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR)) { this.scaleOffset = ScaleOffset.createFromVariable(this); this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType; @@ -542,8 +566,8 @@ public Array getMissingDataArray(int[] shape) { } Array array = Array.factoryConstant(getDataType(), shape, storage); - if (convertMissing != null && convertMissing.hasFillValue()) { - array.setObject(0, convertMissing.getFillValue()); + if (hasFillValue) { + array.setObject(0, fillValue); } return array; } @@ -672,12 +696,12 @@ public boolean isInvalidData(double val) { @Override public boolean hasFillValue() { - return convertMissing != null ? convertMissing.hasFillValue() : false; + return hasFillValue; } @Override public double getFillValue() { - return convertMissing != null ? convertMissing.getFillValue() : Double.MAX_VALUE; + return fillValue; } @Override @@ -816,6 +840,9 @@ public Array convert(Array in, boolean convertUnsigned, boolean applyScaleOffset protected String orgName; // in case Variable was renamed, and we need to keep track of the original name String orgFileTypeId; // the original fileTypeId. + private boolean hasFillValue = false; + private double fillValue = Double.MAX_VALUE; + protected VariableDS(Builder builder, Group parentGroup) { super(builder, parentGroup); @@ -848,6 +875,27 @@ protected VariableDS(Builder builder, Group parentGroup) { this.scaleOffset = ScaleOffset.createFromVariable(this); this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType; } + + // need fill value info before convertMissing + Attribute fillValueAtt = findAttribute(CDM.FILL_VALUE); + if (fillValueAtt != null && !fillValueAtt.isString()) { + fillValue = convertUnsigned(fillValueAtt.getNumericValue()).doubleValue(); + fillValue = applyScaleOffset(fillValue); // This will fail when _FillValue is CHAR. + hasFillValue = true; + } else { + // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value. + String fileTypeId = getFileTypeId(); + boolean isNetcdfIosp = DataFormatType.NETCDF.getDescription().equals(fileTypeId) + || DataFormatType.NETCDF4.getDescription().equals(fileTypeId); + + if (isNetcdfIosp) { + DataType unsignedConversionType = getUnsignedConversionType(); + if (unsignedConversionType.isNumeric()) { + fillValue = applyScaleOffset(N3iosp.getFillValueDefault(unsignedConversionType)); + hasFillValue = true; + } + } + } if (this.enhanceMode.contains(Enhance.ConvertMissing)) { this.convertMissing = ConvertMissing.createFromVariable(this); } diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java index 602becfc25..9ebeba8f2d 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java @@ -87,28 +87,9 @@ public static ConvertMissing createFromVariable(VariableDS var) { } } - /// _FillValue - double fillValue = Double.MAX_VALUE; - boolean hasFillValue = false; - Attribute fillValueAtt = var.findAttribute(CDM.FILL_VALUE); - if (fillValueAtt != null && !fillValueAtt.isString()) { - fillValue = var.convertUnsigned(fillValueAtt.getNumericValue()).doubleValue(); - fillValue = var.applyScaleOffset(fillValue); // This will fail when _FillValue is CHAR. - hasFillValue = true; - } else { - // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value. - String fileTypeId = var.getFileTypeId(); - boolean isNetcdfIosp = DataFormatType.NETCDF.getDescription().equals(fileTypeId) - || DataFormatType.NETCDF4.getDescription().equals(fileTypeId); - - if (isNetcdfIosp) { - DataType unsignedConversionType = var.getUnsignedConversionType(); - if (unsignedConversionType.isNumeric()) { - fillValue = var.applyScaleOffset(N3iosp.getFillValueDefault(unsignedConversionType)); - hasFillValue = true; - } - } - } + /// fill_value + boolean hasFillValue = var.hasFillValue(); + double fillValue = var.getFillValue(); /// missing_value double[] missingValue = null; diff --git a/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhance.java b/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhance.java index bffe5e574a..4d60a7987d 100644 --- a/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhance.java +++ b/cdm/core/src/test/java/ucar/nc2/ncml/TestEnhance.java @@ -6,10 +6,7 @@ import static com.google.common.truth.Truth.assertThat; import java.io.IOException; -import java.lang.invoke.MethodHandles; import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import ucar.ma2.DataType; import ucar.nc2.NetcdfFile; import ucar.nc2.Variable; @@ -18,7 +15,6 @@ /** Test NcmlNew enhancement */ public class TestEnhance { - private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static String dataDir = TestDir.cdmLocalTestDataDir + "ncml/enhance/"; @Test From c2c9b7408292a8f0cbf7405b003c03ca80fc047c Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Tue, 2 May 2023 09:52:06 -0700 Subject: [PATCH 11/24] onlyy convertMissing for floats and doubles --- .../main/java/ucar/nc2/dataset/VariableDS.java | 17 +++++++---------- .../main/java/ucar/nc2/filter/ScaleOffset.java | 13 +++++++++++-- .../ucar/nc2/filter/UnsignedConversion.java | 15 +++++++++------ 3 files changed, 27 insertions(+), 18 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index 30a6f1f4e7..0d8542b65d 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -301,7 +301,8 @@ Array convert(Array data, Set enhancements) { if (enhancements.contains(Enhance.ApplyScaleOffset) && scaleOffset != null) { data = scaleOffset.removeScaleOffset(data); } - if (enhancements.contains(Enhance.ConvertMissing) && convertMissing != null) { + if (enhancements.contains(Enhance.ConvertMissing) && convertMissing != null + && (dataType == DataType.FLOAT || dataType == DataType.DOUBLE)) { data = convertMissing.convertMissing(data); } return data; @@ -806,7 +807,9 @@ public Number convertMissing(Number value) { @Override public Array convertMissing(Array in) { - return convertMissing != null ? convertMissing.convertMissing(in) : in; + return (convertMissing != null && (dataType == DataType.FLOAT || dataType == DataType.DOUBLE)) + ? convertMissing.convertMissing(in) + : in; } /** @@ -815,13 +818,7 @@ public Array convertMissing(Array in) { @Override @Deprecated public Array convert(Array in, boolean convertUnsigned, boolean applyScaleOffset, boolean convertMissing) { - if (this.unsignedConversion != null) { - in = unsignedConversion.convertUnsigned(in); - } - if (this.scaleOffset != null) { - in = scaleOffset.removeScaleOffset(in); - } - return convertMissing(in); + return convertMissing(applyScaleOffset(convertUnsigned(in))); } //////////////////////////////////////////////////////////////////////////////////////////// @@ -867,7 +864,7 @@ protected VariableDS(Builder builder, Group parentGroup) { this.dataType = DataType.STRING; // LOOK promote enum data type to STRING ???? } - if (this.enhanceMode.contains(Enhance.ConvertUnsigned) && !dataType.isString()) { + if (this.enhanceMode.contains(Enhance.ConvertUnsigned) && dataType.isIntegral()) { this.unsignedConversion = UnsignedConversion.createFromVar(this); this.dataType = unsignedConversion.getOutType(); } diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java index acca827b7d..1c6a0b7a65 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java @@ -77,13 +77,22 @@ public static ScaleOffset createFromVariable(VariableDS var) { Attribute scaleAtt = var.findAttribute(CDM.SCALE_FACTOR); if (scaleAtt != null && !scaleAtt.isString()) { scaleType = FilterHelpers.getAttributeDataType(scaleAtt, signedness); - scale = 1 / var.convertUnsigned(scaleAtt.getNumericValue()).doubleValue(); + Number scaleVal = scaleAtt.getNumericValue(); + if (scaleType.isUnsigned()) { + scaleVal = var.convertUnsigned(scaleVal); + } + scale = 1 / scaleVal.doubleValue(); } Attribute offsetAtt = var.findAttribute(CDM.ADD_OFFSET); if (offsetAtt != null && !offsetAtt.isString()) { offsetType = FilterHelpers.getAttributeDataType(offsetAtt, signedness); - offset = var.convertUnsigned(offsetAtt.getNumericValue()).doubleValue(); + Number offsetVal = offsetAtt.getNumericValue(); + if (offsetType.isUnsigned()) { + ; + offsetVal = var.convertUnsigned(offsetVal); + } + offset = offsetVal.doubleValue(); } if (scale != DEFAULT_SCALE || offset != DEFAULT_OFFSET) { DataType scaledOffsetType = diff --git a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java index 8d71981736..8dd58a2069 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java @@ -49,17 +49,20 @@ public Number convertUnsigned(Number value) { return DataType.widenNumberIfNegative(value); } - public Array convertUnsigned(Array in) { Array out = Array.factory(outType, in.getShape()); IndexIterator iterIn = in.getIndexIterator(); IndexIterator iterOut = out.getIndexIterator(); - // iterate and convert elements - while (iterIn.hasNext()) { - Number value = (Number) iterIn.getObjectNext(); - value = convertUnsigned(value); - iterOut.setObjectNext(value); + try { + // iterate and convert elements + while (iterIn.hasNext()) { + Number value = (Number) iterIn.getObjectNext(); + value = convertUnsigned(value); + iterOut.setObjectNext(value); + } + } catch (ClassCastException ex) { + return in; } return out; From 4ed4c834e743706c181c2f19a7b6c1f4a1f5ba1f Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Mon, 8 May 2023 14:59:03 -0700 Subject: [PATCH 12/24] fix test --- cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java | 1 - .../src/test/java/ucar/nc2/filter/TestEnhancements.java | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java index 1c6a0b7a65..1ae2fe949b 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java @@ -89,7 +89,6 @@ public static ScaleOffset createFromVariable(VariableDS var) { offsetType = FilterHelpers.getAttributeDataType(offsetAtt, signedness); Number offsetVal = offsetAtt.getNumericValue(); if (offsetType.isUnsigned()) { - ; offsetVal = var.convertUnsigned(offsetVal); } offset = offsetVal.doubleValue(); diff --git a/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java index b141629128..adf1a0fe17 100644 --- a/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java +++ b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java @@ -67,7 +67,7 @@ public static void setUp() throws IOException, InvalidRangeException { // unsigned, scaled/offset, and missing value Array enhanceAllArray = Array.factory(DataType.SHORT, new int[] {data_len}, signedShorts); builder.addVariable("enhanceAll", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.UNSIGNED, "true")) - .addAttribute(new Attribute(CDM.SCALE_FACTOR, 10)).addAttribute(new Attribute(CDM.ADD_OFFSET, 10)) + .addAttribute(new Attribute(CDM.SCALE_FACTOR, 10.0)).addAttribute(new Attribute(CDM.ADD_OFFSET, 10)) .addAttribute(new Attribute(CDM.VALID_MAX, SIGNED_SCALED_MAX)) .addAttribute(new Attribute(CDM.FILL_VALUE, SIGNED_SCALED_FILL_VALUE)); @@ -136,9 +136,9 @@ public void testConvertMissing() throws IOException { @Test public void testCombinedEnhancements() throws IOException { - int[] expected = new int[] {655320, 0, 655340, 655350, 0, 10, 20, 30, 40, 50}; + double[] expected = new double[] {655320, Double.NaN, 655340, 655350, Double.NaN, 10, 20, 30, 40, 50}; Variable v = ncd.findVariable("enhanceAll"); Array data = v.read(); - assertThat((int[]) data.copyTo1DJavaArray()).isEqualTo(expected); + assertThat((double[]) data.copyTo1DJavaArray()).isEqualTo(expected); } } From 4f033f0a73e7e8318bafd9ae161087dacb71cb5e Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Tue, 9 May 2023 14:59:01 -0700 Subject: [PATCH 13/24] fix application of convertUnsigned --- .../java/ucar/nc2/dataset/VariableDS.java | 81 +++++++------------ .../java/ucar/nc2/filter/ConvertMissing.java | 15 ++-- .../java/ucar/nc2/filter/ScaleOffset.java | 12 +-- .../ucar/nc2/filter/UnsignedConversion.java | 32 +++++--- .../TestScaleOffsetMissingUnsigned.java | 9 ++- .../ucar/nc2/dataset/TestStandardVar.java | 8 +- 6 files changed, 71 insertions(+), 86 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index 0d8542b65d..2fb8a04f42 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -235,44 +235,7 @@ public void enhance(Set enhancements) { setDataType(orgDataType); } - if (this.enhanceMode.contains(Enhance.ConvertEnums) && dataType.isEnum()) { - setDataType(DataType.STRING); // LOOK promote data type to STRING ???? - } - - // Initialize EnhanceScaleMissingUnsignedImpl. We can't do this in the constructors because this object may not - // contain all of the relevant attributes at that time. NcMLReader is an example of this: the VariableDS is - // constructed first, and then Attributes are added to it later. - if (this.enhanceMode.contains(Enhance.ConvertUnsigned) && !dataType.isString()) { - this.unsignedConversion = UnsignedConversion.createFromVar(this); - this.dataType = unsignedConversion.getOutType(); - } - // need fill value info before convertMissing - Attribute fillValueAtt = findAttribute(CDM.FILL_VALUE); - if (fillValueAtt != null && !fillValueAtt.isString()) { - fillValue = convertUnsigned(fillValueAtt.getNumericValue()).doubleValue(); - fillValue = applyScaleOffset(fillValue); // This will fail when _FillValue is CHAR. - hasFillValue = true; - } else { - // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value. - String fileTypeId = getFileTypeId(); - boolean isNetcdfIosp = DataFormatType.NETCDF.getDescription().equals(fileTypeId) - || DataFormatType.NETCDF4.getDescription().equals(fileTypeId); - - if (isNetcdfIosp) { - DataType unsignedConversionType = getUnsignedConversionType(); - if (unsignedConversionType.isNumeric()) { - fillValue = applyScaleOffset(N3iosp.getFillValueDefault(unsignedConversionType)); - hasFillValue = true; - } - } - } - if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR)) { - this.scaleOffset = ScaleOffset.createFromVariable(this); - this.dataType = scaleOffset != null ? scaleOffset.getScaledOffsetType() : this.dataType; - } - if (this.enhanceMode.contains(Enhance.ConvertMissing)) { - this.convertMissing = ConvertMissing.createFromVariable(this); - } + createEnhancements(); } boolean needConvert() { @@ -777,7 +740,7 @@ public DataType getUnsignedConversionType() { @Override public DataType.Signedness getSignedness() { - return unsignedConversion != null ? DataType.Signedness.SIGNED : dataType.getSignedness(); + return unsignedConversion != null ? unsignedConversion.getSignedness() : dataType.getSignedness(); } @Override @@ -795,6 +758,10 @@ public Number convertUnsigned(Number value) { return unsignedConversion != null ? unsignedConversion.convertUnsigned(value) : value; } + public Number convertUnsigned(Number value, DataType dataType) { + return unsignedConversion != null ? unsignedConversion.convertUnsigned(value, dataType) : value; + } + @Override public Array convertUnsigned(Array in) { return unsignedConversion != null ? unsignedConversion.convertUnsigned(in) : in; @@ -860,13 +827,21 @@ protected VariableDS(Builder builder, Group parentGroup) { this.orgFileTypeId = builder.orgFileTypeId; this.enhanceProxy = new EnhancementsImpl(this, builder.units, builder.getDescription()); + createEnhancements(); + + // We have to complete this after the NetcdfDataset is built. + this.coordSysNames = builder.coordSysNames; + } + + private void createEnhancements() { + // Enhancement Objects are created if it is contained in Enhance if (this.enhanceMode.contains(Enhance.ConvertEnums) && dataType.isEnum()) { - this.dataType = DataType.STRING; // LOOK promote enum data type to STRING ???? + this.dataType = DataType.STRING; } - if (this.enhanceMode.contains(Enhance.ConvertUnsigned) && dataType.isIntegral()) { + if (this.enhanceMode.contains(Enhance.ConvertUnsigned)) { this.unsignedConversion = UnsignedConversion.createFromVar(this); - this.dataType = unsignedConversion.getOutType(); + this.dataType = unsignedConversion != null ? unsignedConversion.getOutType() : dataType; } if (this.enhanceMode.contains(Enhance.ApplyScaleOffset) && (dataType.isNumeric() || dataType == DataType.CHAR)) { this.scaleOffset = ScaleOffset.createFromVariable(this); @@ -876,19 +851,22 @@ protected VariableDS(Builder builder, Group parentGroup) { // need fill value info before convertMissing Attribute fillValueAtt = findAttribute(CDM.FILL_VALUE); if (fillValueAtt != null && !fillValueAtt.isString()) { - fillValue = convertUnsigned(fillValueAtt.getNumericValue()).doubleValue(); - fillValue = applyScaleOffset(fillValue); // This will fail when _FillValue is CHAR. + DataType fillType = FilterHelpers.getAttributeDataType(fillValueAtt, getSignedness()); + fillValue = applyScaleOffset(convertUnsigned(fillValueAtt.getNumericValue(), fillType).doubleValue()); // This + // will + // fail + // when + // _FillValue + // is CHAR. hasFillValue = true; } else { // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value. - String fileTypeId = getFileTypeId(); - boolean isNetcdfIosp = DataFormatType.NETCDF.getDescription().equals(fileTypeId) - || DataFormatType.NETCDF4.getDescription().equals(fileTypeId); + boolean isNetcdfIosp = DataFormatType.NETCDF.getDescription().equals(orgFileTypeId) + || DataFormatType.NETCDF4.getDescription().equals(orgFileTypeId); if (isNetcdfIosp) { - DataType unsignedConversionType = getUnsignedConversionType(); - if (unsignedConversionType.isNumeric()) { - fillValue = applyScaleOffset(N3iosp.getFillValueDefault(unsignedConversionType)); + if (dataType.isNumeric()) { + fillValue = applyScaleOffset(N3iosp.getFillValueDefault(dataType)); hasFillValue = true; } } @@ -896,9 +874,6 @@ protected VariableDS(Builder builder, Group parentGroup) { if (this.enhanceMode.contains(Enhance.ConvertMissing)) { this.convertMissing = ConvertMissing.createFromVariable(this); } - - // We have to complete this after the NetcdfDataset is built. - this.coordSysNames = builder.coordSysNames; } public Builder toBuilder() { diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java index 9ebeba8f2d..850a5ac784 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java @@ -36,8 +36,8 @@ public static ConvertMissing createFromVariable(VariableDS var) { DataType validType = null; if (validRangeAtt != null && !validRangeAtt.isString() && validRangeAtt.getLength() > 1) { validType = FilterHelpers.getAttributeDataType(validRangeAtt, signedness); - validMin = var.convertUnsigned(validRangeAtt.getNumericValue(0)).doubleValue(); - validMax = var.convertUnsigned(validRangeAtt.getNumericValue(1)).doubleValue(); + validMin = var.convertUnsigned(validRangeAtt.getNumericValue(0), validType).doubleValue(); + validMax = var.convertUnsigned(validRangeAtt.getNumericValue(1), validType).doubleValue(); hasValidMin = true; hasValidMax = true; } @@ -49,13 +49,13 @@ public static ConvertMissing createFromVariable(VariableDS var) { if (!hasValidMin) { if (validMinAtt != null && !validMinAtt.isString()) { validType = FilterHelpers.getAttributeDataType(validMinAtt, signedness); - validMin = var.convertUnsigned(validMinAtt.getNumericValue()).doubleValue(); + validMin = var.convertUnsigned(validMinAtt.getNumericValue(), validType).doubleValue(); hasValidMin = true; } if (validMaxAtt != null && !validMaxAtt.isString()) { validType = FilterHelpers.largestOf(validType, FilterHelpers.getAttributeDataType(validMaxAtt, signedness)); - validMax = var.convertUnsigned(validMaxAtt.getNumericValue()).doubleValue(); + validMax = var.convertUnsigned(validMaxAtt.getNumericValue(), validType).doubleValue(); hasValidMax = true; } } @@ -105,21 +105,20 @@ public static ConvertMissing createFromVariable(VariableDS var) { } else { missingValue[0] = svalue.charAt(0); } - - hasMissingValue = true; } else { // not a CHAR - try to fix problem where they use a numeric value as a String attribute try { missingValue = new double[1]; missingValue[0] = Double.parseDouble(svalue); - hasMissingValue = true; } catch (NumberFormatException ex) { // TODO add logger } + hasMissingValue = true; } } else { // not a string missingValue = new double[missingValueAtt.getLength()]; + DataType missingType = FilterHelpers.getAttributeDataType(missingValueAtt, signedness); for (int i = 0; i < missingValue.length; i++) { - missingValue[i] = var.convertUnsigned(missingValueAtt.getNumericValue(i)).doubleValue(); + missingValue[i] = var.convertUnsigned(missingValueAtt.getNumericValue(i), missingType).doubleValue(); missingValue[i] = var.applyScaleOffset(missingValue[i]); } diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java index 1ae2fe949b..15e9e16b93 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java @@ -77,21 +77,13 @@ public static ScaleOffset createFromVariable(VariableDS var) { Attribute scaleAtt = var.findAttribute(CDM.SCALE_FACTOR); if (scaleAtt != null && !scaleAtt.isString()) { scaleType = FilterHelpers.getAttributeDataType(scaleAtt, signedness); - Number scaleVal = scaleAtt.getNumericValue(); - if (scaleType.isUnsigned()) { - scaleVal = var.convertUnsigned(scaleVal); - } - scale = 1 / scaleVal.doubleValue(); + scale = 1 / (var.convertUnsigned(scaleAtt.getNumericValue(), scaleType).doubleValue()); } Attribute offsetAtt = var.findAttribute(CDM.ADD_OFFSET); if (offsetAtt != null && !offsetAtt.isString()) { offsetType = FilterHelpers.getAttributeDataType(offsetAtt, signedness); - Number offsetVal = offsetAtt.getNumericValue(); - if (offsetType.isUnsigned()) { - offsetVal = var.convertUnsigned(offsetVal); - } - offset = offsetVal.doubleValue(); + offset = var.convertUnsigned(offsetAtt.getNumericValue(), offsetType).doubleValue(); } if (scale != DEFAULT_SCALE || offset != DEFAULT_OFFSET) { DataType scaledOffsetType = diff --git a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java index 8dd58a2069..13cfc262bb 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java @@ -9,15 +9,17 @@ public class UnsignedConversion { private DataType outType; + private DataType.Signedness signedness; public static UnsignedConversion createFromVar(VariableDS var) { DataType origDataType = var.getDataType(); + // won't need to convert non-integral types + if (origDataType.isIntegral()) { + return new UnsignedConversion(origDataType, DataType.Signedness.SIGNED); + } DataType unsignedConversionType = origDataType; - - // unsignedConversionType is initialized to origDataType, and origDataType may be a non-integral type that doesn't - // have an "unsigned flavor" (such as FLOAT and DOUBLE). Furthermore, unsignedConversionType may start out as - // integral, but then be widened to non-integral (i.e. LONG -> DOUBLE). For these reasons, we cannot rely upon - // unsignedConversionType to store the signedness of the variable. We need a separate field. + // unsignedConversionType may start out as integral, but then be widened to non-integral (i.e. LONG -> DOUBLE) + // so we cannot rely upon unsignedConversionType to store the signedness of the variable. We need a separate field. DataType.Signedness signedness = origDataType.getSignedness(); // In the event of conflict, "unsigned" wins. Potential conflicts include: @@ -34,22 +36,30 @@ public static UnsignedConversion createFromVar(VariableDS var) { // We may need a larger data type to hold the results of the unsigned conversion. unsignedConversionType = FilterHelpers.nextLarger(origDataType).withSignedness(DataType.Signedness.UNSIGNED); } - return new UnsignedConversion(unsignedConversionType); + return new UnsignedConversion(unsignedConversionType, signedness); } - public UnsignedConversion(DataType outType) { + public UnsignedConversion(DataType outType, DataType.Signedness signedness) { this.outType = outType; + this.signedness = signedness; } public DataType getOutType() { return this.outType; } + public DataType.Signedness getSignedness() { + return this.signedness; + } + public Number convertUnsigned(Number value) { - return DataType.widenNumberIfNegative(value); + return this.signedness == DataType.Signedness.UNSIGNED ? DataType.widenNumberIfNegative(value) : value; } public Array convertUnsigned(Array in) { + if (signedness == DataType.Signedness.SIGNED) { + return in; + } Array out = Array.factory(outType, in.getShape()); IndexIterator iterIn = in.getIndexIterator(); IndexIterator iterOut = out.getIndexIterator(); @@ -58,7 +68,7 @@ public Array convertUnsigned(Array in) { // iterate and convert elements while (iterIn.hasNext()) { Number value = (Number) iterIn.getObjectNext(); - value = convertUnsigned(value); + value = DataType.widenNumberIfNegative(value); iterOut.setObjectNext(value); } } catch (ClassCastException ex) { @@ -67,4 +77,8 @@ public Array convertUnsigned(Array in) { return out; } + + public static Number convertUnsigned(Number value, DataType dataType) { + return dataType.isUnsigned() ? DataType.widenNumberIfNegative(value) : value; + } } diff --git a/cdm/core/src/test/java/ucar/nc2/dataset/TestScaleOffsetMissingUnsigned.java b/cdm/core/src/test/java/ucar/nc2/dataset/TestScaleOffsetMissingUnsigned.java index f3e61e6a0e..cb2faddb43 100644 --- a/cdm/core/src/test/java/ucar/nc2/dataset/TestScaleOffsetMissingUnsigned.java +++ b/cdm/core/src/test/java/ucar/nc2/dataset/TestScaleOffsetMissingUnsigned.java @@ -280,8 +280,8 @@ public void testScaleOffsetMissingUnsigned() throws URISyntaxException, IOExcept Assert.assertEquals(DataType.UINT, var.getDataType()); // These vals are the same as ones from "missingUnsigned", but with a scale_factor of 100 and offset of 1 applied. - int[] expecteds = new int[] {14901, 15001, 25001, 25101, 25501, 8001}; - int[] actuals = (int[]) var.read().getStorage(); + long[] expecteds = new long[] {14901, 15001, 25001, 25101, 25501, 8001}; + long[] actuals = (long[]) var.read().getStorage(); Assert.assertArrayEquals(expecteds, actuals); } } @@ -346,10 +346,14 @@ public void testUnpackedValidRange() throws IOException, URISyntaxException { * data. Otherwise it is in the units of the external (packed) data. */ // As a result, scale_factor will not be applied to it. + // with the Builder design, we don't read the enhancement properties unless the corresponding enhancement is + // turned on + var.addEnhancement(Enhance.ConvertMissing); Assert2.assertNearlyEquals(9.9f, (float) var.getValidMin()); Assert2.assertNearlyEquals(10.1f, (float) var.getValidMax()); Assert.assertEquals(DataType.FLOAT, var.getDataType()); // scale_factor is float. + var.removeEnhancement(Enhance.ConvertMissing); float[] expecteds = new float[] {9.8f, 9.9f, 10.0f, 10.1f, 10.2f}; float[] actuals = (float[]) var.read().getStorage(); @@ -363,6 +367,7 @@ public void testUnsignedOffsetAttribute() throws IOException, URISyntaxException try (NetcdfDataset ncd = NetcdfDatasets.openDataset(testResource.getAbsolutePath(), true, null)) { VariableDS var = (VariableDS) ncd.findVariable("unsignedOffsetAttribute"); + var.addEnhancement(Enhance.ApplyScaleOffset); Assert.assertEquals(156, var.getOffset(), 0); Assert.assertEquals(DataType.BYTE, var.getDataType()); // No change to data type. diff --git a/cdm/core/src/test/java/ucar/nc2/dataset/TestStandardVar.java b/cdm/core/src/test/java/ucar/nc2/dataset/TestStandardVar.java index 65ab5ba824..5a08b82162 100644 --- a/cdm/core/src/test/java/ucar/nc2/dataset/TestStandardVar.java +++ b/cdm/core/src/test/java/ucar/nc2/dataset/TestStandardVar.java @@ -218,8 +218,8 @@ private void readByte() throws Exception { assert (vs.hasMissing()); assert (vs.hasFillValue()); - assert (vs.isMissing((double) ((byte) 255))); - assert (vs.isFillValue((double) ((byte) 255))); + assert (vs.isMissing((byte) 255)); + assert (vs.isFillValue((byte) 255)); Array A = vs.read(); assert (A.getElementType() == byte.class) : A.getElementType(); @@ -256,8 +256,8 @@ private void readShortMissing() throws Exception { assert (vs.hasMissing()); assert (vs.hasMissingValue()); - assert (vs.isMissing((double) ((short) -9999))); - assert (vs.isMissingValue((double) ((short) -9999))); + assert (vs.isMissing((short) -9999)); + assert (vs.isMissingValue((short) -9999)); Array A = vs.read(); Index ima = A.getIndex(); From 29ede7318b8f01ad11d68645eefa59a30fcfef63 Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Tue, 9 May 2023 15:51:18 -0700 Subject: [PATCH 14/24] fix misplaces hasMissing flag --- cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java index 850a5ac784..41d37f1d89 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java @@ -112,8 +112,8 @@ public static ConvertMissing createFromVariable(VariableDS var) { } catch (NumberFormatException ex) { // TODO add logger } - hasMissingValue = true; } + hasMissingValue = true; } else { // not a string missingValue = new double[missingValueAtt.getLength()]; DataType missingType = FilterHelpers.getAttributeDataType(missingValueAtt, signedness); From 41129766acdc8a4ff579409b7190d026bda85004 Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Tue, 9 May 2023 17:03:15 -0700 Subject: [PATCH 15/24] fix typo and formatting --- cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java | 7 +------ .../src/main/java/ucar/nc2/filter/UnsignedConversion.java | 2 +- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index 2fb8a04f42..e40bbf0938 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -852,12 +852,7 @@ private void createEnhancements() { Attribute fillValueAtt = findAttribute(CDM.FILL_VALUE); if (fillValueAtt != null && !fillValueAtt.isString()) { DataType fillType = FilterHelpers.getAttributeDataType(fillValueAtt, getSignedness()); - fillValue = applyScaleOffset(convertUnsigned(fillValueAtt.getNumericValue(), fillType).doubleValue()); // This - // will - // fail - // when - // _FillValue - // is CHAR. + fillValue = applyScaleOffset(convertUnsigned(fillValueAtt.getNumericValue(), fillType).doubleValue()); hasFillValue = true; } else { // No _FillValue attribute found. Instead, if file is NetCDF and variable is numeric, use the default fill value. diff --git a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java index 13cfc262bb..ea2860a412 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/UnsignedConversion.java @@ -14,7 +14,7 @@ public class UnsignedConversion { public static UnsignedConversion createFromVar(VariableDS var) { DataType origDataType = var.getDataType(); // won't need to convert non-integral types - if (origDataType.isIntegral()) { + if (!origDataType.isIntegral()) { return new UnsignedConversion(origDataType, DataType.Signedness.SIGNED); } DataType unsignedConversionType = origDataType; From 6f595abadc9c241df2116a587de89702c613e7d0 Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Wed, 10 May 2023 09:21:43 -0700 Subject: [PATCH 16/24] preserve orgDataType if no orgVar --- cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java | 3 +++ cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index e40bbf0938..b92b875bc8 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -814,6 +814,9 @@ protected VariableDS(Builder builder, Group parentGroup) { this.orgVar = builder.orgVar; this.orgDataType = builder.orgDataType; this.orgName = builder.orgName; + if (this.orgDataType == null) { + this.orgDataType = dataType; + } // Make sure that units has been trimmed. // Replace with correct case diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java index 41d37f1d89..cd2c71ac7b 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java @@ -63,7 +63,7 @@ public static ConvertMissing createFromVariable(VariableDS var) { // check if validData values are stored packed or unpacked if (hasValidMin || hasValidMax) { if (FilterHelpers.rank(validType) == FilterHelpers.rank(var.getScaledOffsetType()) - && FilterHelpers.rank(validType) > FilterHelpers.rank(var.getUnsignedConversionType())) { + && FilterHelpers.rank(validType) > FilterHelpers.rank(var.getOriginalDataType())) { // If valid_range is the same type as the wider of scale_factor and add_offset, PLUS // it is wider than the (packed) data, we know that the valid_range values were stored as unpacked. // We already assumed that this was the case when we first read the attribute values, so there's From 4552be7ae220bab142ad49dde44a8c44f785e0e5 Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Fri, 30 Jun 2023 16:31:33 -0500 Subject: [PATCH 17/24] propagate enhanceMode to coordinate vars --- cdm-test/build.gradle | 2 +- .../ucar/nc2/ft/point/TestPointDatasets.java | 84 +++++++++---------- .../java/ucar/nc2/dataset/VariableDS.java | 1 + 3 files changed, 44 insertions(+), 43 deletions(-) diff --git a/cdm-test/build.gradle b/cdm-test/build.gradle index c1f3702565..1432636e70 100644 --- a/cdm-test/build.gradle +++ b/cdm-test/build.gradle @@ -40,7 +40,7 @@ tasks.register("testIndexCreation", Test) { } test { - dependsOn 'testIndexCreation' +// dependsOn 'testIndexCreation' // In addition to preventing TestGribIndexCreation from running during cdm-test:test, // this statement also excludes the results of TestGribIndexCreation from appearing in the cdm-test report: diff --git a/cdm-test/src/test/java/ucar/nc2/ft/point/TestPointDatasets.java b/cdm-test/src/test/java/ucar/nc2/ft/point/TestPointDatasets.java index 3e8e58a657..6e5690e5b7 100644 --- a/cdm-test/src/test/java/ucar/nc2/ft/point/TestPointDatasets.java +++ b/cdm-test/src/test/java/ucar/nc2/ft/point/TestPointDatasets.java @@ -108,47 +108,47 @@ private static List getCFDatasets() { private static List getPlugDatasets() { List result = new ArrayList<>(); - // cosmic - result - .add(new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/cosmic/wetPrf_C005.2007.294.16.22.G17_0001.0002_nc", - FeatureType.TRAJECTORY, 383}); - // ndbc - result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/ndbc/41001h1976.nc", FeatureType.STATION, 1405}); - result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/suomi/suoHWV_2006.105.00.00.0060_nc", - FeatureType.STATION, 124}); - result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/suomi/gsuPWV_2006.105.00.00.1440_nc", - FeatureType.STATION, 4848}); - // fsl wind profilers - result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_RASS_01hr_20091027_1500.nc", - FeatureType.STATION_PROFILE, 198}); - result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_RASS_06min_20091028_2318.nc", - FeatureType.STATION_PROFILE, 198}); - result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_wind_01hr_20091024_1200.nc", - FeatureType.STATION_PROFILE, 1728}); - result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_wind_06min_20091030_2330.nc", - FeatureType.STATION_PROFILE, 2088}); - // netcdf buoy / synoptic / metars ( robb's perl decoder output) - result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_METAR_latest.nc", FeatureType.POINT, 7}); - result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_Buoy_20090921_0000.nc", - FeatureType.POINT, 32452}); - result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_Synoptic_20090921_0000.nc", - FeatureType.POINT, 1516}); - // RAF-Nimbus - result.add( - new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/aircraft/135_ordrd.nc", FeatureType.TRAJECTORY, 7741}); - result.add( - new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/aircraft/raftrack.nc", FeatureType.TRAJECTORY, 8157}); - // Madis - result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/acars/acars_20091109_0800.nc", - FeatureType.TRAJECTORY, 5063}); +// // cosmic +// result +// .add(new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/cosmic/wetPrf_C005.2007.294.16.22.G17_0001.0002_nc", +// FeatureType.TRAJECTORY, 383}); +// // ndbc +// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/ndbc/41001h1976.nc", FeatureType.STATION, 1405}); +// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/suomi/suoHWV_2006.105.00.00.0060_nc", +// FeatureType.STATION, 124}); +// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/suomi/gsuPWV_2006.105.00.00.1440_nc", +// FeatureType.STATION, 4848}); +// // fsl wind profilers +// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_RASS_01hr_20091027_1500.nc", +// FeatureType.STATION_PROFILE, 198}); +// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_RASS_06min_20091028_2318.nc", +// FeatureType.STATION_PROFILE, 198}); +// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_wind_01hr_20091024_1200.nc", +// FeatureType.STATION_PROFILE, 1728}); +// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_wind_06min_20091030_2330.nc", +// FeatureType.STATION_PROFILE, 2088}); +// // netcdf buoy / synoptic / metars ( robb's perl decoder output) +// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_METAR_latest.nc", FeatureType.POINT, 7}); +// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_Buoy_20090921_0000.nc", +// FeatureType.POINT, 32452}); +// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_Synoptic_20090921_0000.nc", +// FeatureType.POINT, 1516}); +// // RAF-Nimbus +// result.add( +// new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/aircraft/135_ordrd.nc", FeatureType.TRAJECTORY, 7741}); +// result.add( +// new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/aircraft/raftrack.nc", FeatureType.TRAJECTORY, 8157}); +// // Madis +// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/acars/acars_20091109_0800.nc", +// FeatureType.TRAJECTORY, 5063}); result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/19981110_1200", FeatureType.POINT, 2499}); - result.add( - new Object[] {TestDir.cdmUnitTestDir + "ft/station/madis2/hydro/20050729_1200", FeatureType.STATION, 1374}); - result.add( - new Object[] {TestDir.cdmUnitTestDir + "ft/sounding/netcdf/20070612_1200", FeatureType.STATION_PROFILE, 1788}); - // unidata point obs - result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/200501q3h-gr.nc", FeatureType.STATION, 5023}); - result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/20080814_LMA.ncml", FeatureType.POINT, 277477}); +// result.add( +// new Object[] {TestDir.cdmUnitTestDir + "ft/station/madis2/hydro/20050729_1200", FeatureType.STATION, 1374}); +// result.add( +// new Object[] {TestDir.cdmUnitTestDir + "ft/sounding/netcdf/20070612_1200", FeatureType.STATION_PROFILE, 1788}); +// // unidata point obs +// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/200501q3h-gr.nc", FeatureType.STATION, 5023}); +// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/20080814_LMA.ncml", FeatureType.POINT, 277477}); // FslRaob // assert 63 == checkPointDataset(TestDir.testdataDir + "sounding/netcdf/raob_soundings20216.cdf", @@ -174,9 +174,9 @@ private static List getMiscDatasets() { public static List getTestParameters() { List result = new ArrayList<>(); - result.addAll(getCFDatasets()); +// result.addAll(getCFDatasets()); result.addAll(getPlugDatasets()); - result.addAll(getMiscDatasets()); +// result.addAll(getMiscDatasets()); return result; } diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index b92b875bc8..b1e245948f 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -167,6 +167,7 @@ protected VariableDS(VariableDS vds, boolean isCopy) { this.orgVar = vds; this.orgDataType = vds.orgDataType; this.orgName = vds.orgName; + this.enhanceMode = vds.enhanceMode; this.enhanceProxy = new EnhancementsImpl(this); // decouple coordinate systems this.unsignedConversion = vds.unsignedConversion; From 9dc7aab5464845b76e1421642cce28e30bb8f0ef Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Mon, 3 Jul 2023 10:18:53 -0500 Subject: [PATCH 18/24] incorporate PR feedback --- cdm-test/build.gradle | 2 +- .../ucar/nc2/ft/point/TestPointDatasets.java | 84 +++++++++---------- .../dataset/EnhanceScaleMissingUnsigned.java | 2 +- .../java/ucar/nc2/dataset/VariableDS.java | 14 ++-- .../java/ucar/nc2/filter/ScaleOffset.java | 3 +- .../ucar/nc2/filter/TestEnhancements.java | 23 +++-- .../java/ucar/nc2/filter/TestFilters.java | 12 +-- 7 files changed, 74 insertions(+), 66 deletions(-) diff --git a/cdm-test/build.gradle b/cdm-test/build.gradle index 1432636e70..c1f3702565 100644 --- a/cdm-test/build.gradle +++ b/cdm-test/build.gradle @@ -40,7 +40,7 @@ tasks.register("testIndexCreation", Test) { } test { -// dependsOn 'testIndexCreation' + dependsOn 'testIndexCreation' // In addition to preventing TestGribIndexCreation from running during cdm-test:test, // this statement also excludes the results of TestGribIndexCreation from appearing in the cdm-test report: diff --git a/cdm-test/src/test/java/ucar/nc2/ft/point/TestPointDatasets.java b/cdm-test/src/test/java/ucar/nc2/ft/point/TestPointDatasets.java index 6e5690e5b7..3e8e58a657 100644 --- a/cdm-test/src/test/java/ucar/nc2/ft/point/TestPointDatasets.java +++ b/cdm-test/src/test/java/ucar/nc2/ft/point/TestPointDatasets.java @@ -108,47 +108,47 @@ private static List getCFDatasets() { private static List getPlugDatasets() { List result = new ArrayList<>(); -// // cosmic -// result -// .add(new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/cosmic/wetPrf_C005.2007.294.16.22.G17_0001.0002_nc", -// FeatureType.TRAJECTORY, 383}); -// // ndbc -// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/ndbc/41001h1976.nc", FeatureType.STATION, 1405}); -// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/suomi/suoHWV_2006.105.00.00.0060_nc", -// FeatureType.STATION, 124}); -// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/suomi/gsuPWV_2006.105.00.00.1440_nc", -// FeatureType.STATION, 4848}); -// // fsl wind profilers -// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_RASS_01hr_20091027_1500.nc", -// FeatureType.STATION_PROFILE, 198}); -// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_RASS_06min_20091028_2318.nc", -// FeatureType.STATION_PROFILE, 198}); -// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_wind_01hr_20091024_1200.nc", -// FeatureType.STATION_PROFILE, 1728}); -// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_wind_06min_20091030_2330.nc", -// FeatureType.STATION_PROFILE, 2088}); -// // netcdf buoy / synoptic / metars ( robb's perl decoder output) -// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_METAR_latest.nc", FeatureType.POINT, 7}); -// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_Buoy_20090921_0000.nc", -// FeatureType.POINT, 32452}); -// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_Synoptic_20090921_0000.nc", -// FeatureType.POINT, 1516}); -// // RAF-Nimbus -// result.add( -// new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/aircraft/135_ordrd.nc", FeatureType.TRAJECTORY, 7741}); -// result.add( -// new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/aircraft/raftrack.nc", FeatureType.TRAJECTORY, 8157}); -// // Madis -// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/acars/acars_20091109_0800.nc", -// FeatureType.TRAJECTORY, 5063}); + // cosmic + result + .add(new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/cosmic/wetPrf_C005.2007.294.16.22.G17_0001.0002_nc", + FeatureType.TRAJECTORY, 383}); + // ndbc + result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/ndbc/41001h1976.nc", FeatureType.STATION, 1405}); + result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/suomi/suoHWV_2006.105.00.00.0060_nc", + FeatureType.STATION, 124}); + result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/suomi/gsuPWV_2006.105.00.00.1440_nc", + FeatureType.STATION, 4848}); + // fsl wind profilers + result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_RASS_01hr_20091027_1500.nc", + FeatureType.STATION_PROFILE, 198}); + result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_RASS_06min_20091028_2318.nc", + FeatureType.STATION_PROFILE, 198}); + result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_wind_01hr_20091024_1200.nc", + FeatureType.STATION_PROFILE, 1728}); + result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_wind_06min_20091030_2330.nc", + FeatureType.STATION_PROFILE, 2088}); + // netcdf buoy / synoptic / metars ( robb's perl decoder output) + result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_METAR_latest.nc", FeatureType.POINT, 7}); + result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_Buoy_20090921_0000.nc", + FeatureType.POINT, 32452}); + result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_Synoptic_20090921_0000.nc", + FeatureType.POINT, 1516}); + // RAF-Nimbus + result.add( + new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/aircraft/135_ordrd.nc", FeatureType.TRAJECTORY, 7741}); + result.add( + new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/aircraft/raftrack.nc", FeatureType.TRAJECTORY, 8157}); + // Madis + result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/acars/acars_20091109_0800.nc", + FeatureType.TRAJECTORY, 5063}); result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/19981110_1200", FeatureType.POINT, 2499}); -// result.add( -// new Object[] {TestDir.cdmUnitTestDir + "ft/station/madis2/hydro/20050729_1200", FeatureType.STATION, 1374}); -// result.add( -// new Object[] {TestDir.cdmUnitTestDir + "ft/sounding/netcdf/20070612_1200", FeatureType.STATION_PROFILE, 1788}); -// // unidata point obs -// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/200501q3h-gr.nc", FeatureType.STATION, 5023}); -// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/20080814_LMA.ncml", FeatureType.POINT, 277477}); + result.add( + new Object[] {TestDir.cdmUnitTestDir + "ft/station/madis2/hydro/20050729_1200", FeatureType.STATION, 1374}); + result.add( + new Object[] {TestDir.cdmUnitTestDir + "ft/sounding/netcdf/20070612_1200", FeatureType.STATION_PROFILE, 1788}); + // unidata point obs + result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/200501q3h-gr.nc", FeatureType.STATION, 5023}); + result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/20080814_LMA.ncml", FeatureType.POINT, 277477}); // FslRaob // assert 63 == checkPointDataset(TestDir.testdataDir + "sounding/netcdf/raob_soundings20216.cdf", @@ -174,9 +174,9 @@ private static List getMiscDatasets() { public static List getTestParameters() { List result = new ArrayList<>(); -// result.addAll(getCFDatasets()); + result.addAll(getCFDatasets()); result.addAll(getPlugDatasets()); -// result.addAll(getMiscDatasets()); + result.addAll(getMiscDatasets()); return result; } diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsigned.java b/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsigned.java index 2a255435d2..7175820f6d 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsigned.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsigned.java @@ -38,7 +38,7 @@ * *

      *
    1. If scale_factor and/or add_offset variable attributes are present, then this is a "packed" Variable.
      2. - *
    2. The data type of the variable will be set to the {@link EnhanceScaleMissingUnsignedImpl#largestOf largest of}: + *
    3. The data type of the variable will be set to the {@link ucar.nc2.filter.FilterHelpers#largestOf largest of}: *
        *
      • the original data type
        • *
      • the unsigned conversion type, if applicable
        • diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index b1e245948f..2ee21bc47d 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -628,7 +628,7 @@ public double getOffset() { @Override public boolean hasMissing() { - return convertMissing != null ? convertMissing.hasMissing() : false; + return convertMissing != null && convertMissing.hasMissing(); } @Override @@ -636,12 +636,12 @@ public boolean isMissing(double val) { if (Double.isNaN(val)) { return true; } - return convertMissing != null ? convertMissing.isMissing(val) : false; + return convertMissing != null && convertMissing.isMissing(val); } @Override public boolean hasValidData() { - return convertMissing != null ? convertMissing.hasMissingValue() : false; + return convertMissing != null && convertMissing.hasMissingValue(); } @Override @@ -656,7 +656,7 @@ public double getValidMax() { @Override public boolean isInvalidData(double val) { - return convertMissing != null ? convertMissing.isInvalidData(val) : false; + return convertMissing != null && convertMissing.isInvalidData(val); } @Override @@ -671,12 +671,12 @@ public double getFillValue() { @Override public boolean isFillValue(double val) { - return convertMissing != null ? convertMissing.isFillValue(val) : false; + return convertMissing != null && convertMissing.isFillValue(val); } @Override public boolean hasMissingValue() { - return convertMissing != null ? convertMissing.hasMissingValue() : false; + return convertMissing != null && convertMissing.hasMissingValue(); } @Override @@ -686,7 +686,7 @@ public double[] getMissingValues() { @Override public boolean isMissingValue(double val) { - return convertMissing != null ? convertMissing.isMissingValue(val) : false; + return convertMissing != null && convertMissing.isMissingValue(val); } /** @deprecated Use NetcdfDataset.builder() */ diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java index 15e9e16b93..4362c29e2f 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java @@ -175,7 +175,6 @@ public byte[] decode(byte[] dataIn) { return FilterHelpers.arrayToBytes(out, dtype, dtypeOrder); } - // not used anywhere yet public Array applyScaleOffset(Array in) { if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) { return in; @@ -251,7 +250,7 @@ private static ByteOrder parseByteOrder(String dtype, ByteOrder defaultOrder) { public double applyScaleOffset(Number value) { double convertedValue = value.doubleValue(); - return Math.round((convertedValue - offset) * scale); + return (convertedValue - offset) * scale; } public double removeScaleOffset(Number value) { diff --git a/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java index adf1a0fe17..89a4b0cdab 100644 --- a/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java +++ b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java @@ -37,12 +37,12 @@ public class TestEnhancements { @BeforeClass public static void setUp() throws IOException, InvalidRangeException { - final int data_len = 10; + final int dataLen = 10; String filePath = tempFolder.newFile().getAbsolutePath(); NetcdfFormatWriter.Builder builder = NetcdfFormatWriter.createNewNetcdf3(filePath); - builder.addDimension("dim", data_len); + builder.addDimension("dim", dataLen); - Array signedData = Array.factory(DataType.SHORT, new int[] {data_len}, signedShorts); + Array signedData = Array.factory(DataType.SHORT, new int[] {dataLen}, signedShorts); // signed shorts builder.addVariable("signedVar", DataType.SHORT, "dim"); // unsigned shorts @@ -51,8 +51,10 @@ public static void setUp() throws IOException, InvalidRangeException { // scaled and offset data builder.addVariable("scaleOffsetVar", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.SCALE_FACTOR, 10)) .addAttribute(new Attribute(CDM.ADD_OFFSET, 10)); + // scaled data no offset + builder.addVariable("scaleNoOffsetVar", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.SCALE_FACTOR, 10)); - Array missingDataArray = Array.factory(DataType.FLOAT, new int[] {data_len}, missingData); + Array missingDataArray = Array.factory(DataType.FLOAT, new int[] {dataLen}, missingData); // Data with min builder.addVariable("validMin", DataType.FLOAT, "dim").addAttribute(new Attribute(CDM.VALID_MIN, VALID_MIN)); // Data with min and max @@ -65,7 +67,7 @@ public static void setUp() throws IOException, InvalidRangeException { .addAttribute(Attribute.builder(CDM.FILL_VALUE).setNumericValue(FILL_VALUE, true).build()); // unsigned, scaled/offset, and missing value - Array enhanceAllArray = Array.factory(DataType.SHORT, new int[] {data_len}, signedShorts); + Array enhanceAllArray = Array.factory(DataType.SHORT, new int[] {dataLen}, signedShorts); builder.addVariable("enhanceAll", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.UNSIGNED, "true")) .addAttribute(new Attribute(CDM.SCALE_FACTOR, 10.0)).addAttribute(new Attribute(CDM.ADD_OFFSET, 10)) .addAttribute(new Attribute(CDM.VALID_MAX, SIGNED_SCALED_MAX)) @@ -76,6 +78,7 @@ public static void setUp() throws IOException, InvalidRangeException { writer.write(writer.findVariable("signedVar"), new int[1], signedData); writer.write(writer.findVariable("unsignedVar"), new int[1], signedData); writer.write(writer.findVariable("scaleOffsetVar"), new int[1], signedData); + writer.write(writer.findVariable("scaleNoOffsetVar"), new int[1], signedData); writer.write(writer.findVariable("validMin"), new int[1], missingDataArray); writer.write(writer.findVariable("validMinMax"), new int[1], missingDataArray); writer.write(writer.findVariable("validRange"), new int[1], missingDataArray); @@ -104,13 +107,19 @@ public void testUnsignedConversion() throws IOException { @Test public void testScaleOffset() throws IOException { - final int[] expected = new int[] {-40, -30, -20, -10, 0, 10, 20, 30, 40, 50}; - // signed var + int[] expected = new int[] {-40, -30, -20, -10, 0, 10, 20, 30, 40, 50}; Variable v = ncd.findVariable("scaleOffsetVar"); Array data = v.read(); assertThat(data.isUnsigned()).isFalse(); assertThat(data.getDataType()).isEqualTo(DataType.INT); assertThat((int[]) data.copyTo1DJavaArray()).isEqualTo(expected); + + expected = new int[] {-50, -40, -30, -20, -10, 0, 10, 20, 30, 40}; + v = ncd.findVariable("scaleNoOffsetVar"); + data = v.read(); + assertThat(data.isUnsigned()).isFalse(); + assertThat(data.getDataType()).isEqualTo(DataType.INT); + assertThat((int[]) data.copyTo1DJavaArray()).isEqualTo(expected); } @Test diff --git a/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java b/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java index b98298efdf..67893d9893 100644 --- a/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java +++ b/cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java @@ -98,13 +98,13 @@ public void testScaleOffset() throws IOException { assertThat(decoded).isEqualTo(input); // test empty props - Map props2 = new HashMap<>(); - props2.put("id", "fixedscaleoffset"); - props2.put("dtype", " emptyProps = new HashMap<>(); + emptyProps.put("id", "fixedscaleoffset"); + emptyProps.put("dtype", " Date: Mon, 3 Jul 2023 10:33:41 -0500 Subject: [PATCH 19/24] add rounding back in --- cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java index 4362c29e2f..e748526ba2 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java @@ -250,7 +250,7 @@ private static ByteOrder parseByteOrder(String dtype, ByteOrder defaultOrder) { public double applyScaleOffset(Number value) { double convertedValue = value.doubleValue(); - return (convertedValue - offset) * scale; + return Math.round((convertedValue - offset) * scale); } public double removeScaleOffset(Number value) { From abbfbf5951f306da4fbc539b1eee71778fcbd8e5 Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Thu, 6 Jul 2023 11:04:10 -0400 Subject: [PATCH 20/24] Update cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java Co-authored-by: Tara Drwenski --- cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index 2ee21bc47d..c0260a0fbe 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -681,7 +681,7 @@ public boolean hasMissingValue() { @Override public double[] getMissingValues() { - return convertMissing != null ? convertMissing.getMissingValues() : new double[] {0}; + return convertMissing != null ? convertMissing.getMissingValues() : new double[0]; } @Override From 1e44cb3316acbbc9ab86e0a738b542cd60c29656 Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Thu, 6 Jul 2023 11:04:37 -0400 Subject: [PATCH 21/24] Update cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java Co-authored-by: Tara Drwenski --- cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index c0260a0fbe..314fa2b4e8 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -760,7 +760,7 @@ public Number convertUnsigned(Number value) { } public Number convertUnsigned(Number value, DataType dataType) { - return unsignedConversion != null ? unsignedConversion.convertUnsigned(value, dataType) : value; + return unsignedConversion != null ? UnsignedConversion.convertUnsigned(value, dataType) : value; } @Override From 8627be39035f1e696711fcb502dbebb91824ed20 Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Thu, 6 Jul 2023 11:05:05 -0400 Subject: [PATCH 22/24] Update cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java Co-authored-by: Tara Drwenski --- cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java index cd2c71ac7b..73835864c7 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ConvertMissing.java @@ -5,9 +5,7 @@ import ucar.ma2.IndexIterator; import ucar.nc2.Attribute; import ucar.nc2.constants.CDM; -import ucar.nc2.constants.DataFormatType; import ucar.nc2.dataset.VariableDS; -import ucar.nc2.iosp.netcdf3.N3iosp; import ucar.nc2.util.Misc; public class ConvertMissing { From 5a56969ea244cde93ad5f02133cf01e7101812eb Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Thu, 6 Jul 2023 13:35:08 -0400 Subject: [PATCH 23/24] PR feedback and ignore DAP4 remote tests --- .../src/main/java/ucar/nc2/dataset/NetcdfDataset.java | 2 +- cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java | 6 +++--- cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java | 8 +++++++- .../src/test/java/ucar/nc2/filter/TestEnhancements.java | 6 ++++-- dap4/src/test/java/dap4/test/TestRemote.java | 2 ++ 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/NetcdfDataset.java b/cdm/core/src/main/java/ucar/nc2/dataset/NetcdfDataset.java index 4b78cab001..18209d83fc 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/NetcdfDataset.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/NetcdfDataset.java @@ -104,7 +104,7 @@ public enum Enhance { /** Apply scale and offset to values, promoting the data type if needed. */ ApplyScaleOffset, /** - * Replace {@link EnhanceScaleMissingUnsigned#isMissing missing} data with NaNs, for efficiency. Note that if the + * Replace {@link ucar.nc2.filter.ConvertMissing#isMissing missing} data with NaNs, for efficiency. Note that if the * enhanced data type is not {@code FLOAT} or {@code DOUBLE}, this has no effect. */ ConvertMissing, diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index 2ee21bc47d..b7b6c87d9b 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -265,8 +265,7 @@ Array convert(Array data, Set enhancements) { if (enhancements.contains(Enhance.ApplyScaleOffset) && scaleOffset != null) { data = scaleOffset.removeScaleOffset(data); } - if (enhancements.contains(Enhance.ConvertMissing) && convertMissing != null - && (dataType == DataType.FLOAT || dataType == DataType.DOUBLE)) { + if (enhancements.contains(Enhance.ConvertMissing) && convertMissing != null) { data = convertMissing.convertMissing(data); } return data; @@ -870,7 +869,8 @@ private void createEnhancements() { } } } - if (this.enhanceMode.contains(Enhance.ConvertMissing)) { + if (this.enhanceMode.contains(Enhance.ConvertMissing) + && (dataType == DataType.FLOAT || dataType == DataType.DOUBLE)) { this.convertMissing = ConvertMissing.createFromVariable(this); } } diff --git a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java index e748526ba2..90202ecb7b 100644 --- a/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java +++ b/cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java @@ -250,11 +250,17 @@ private static ByteOrder parseByteOrder(String dtype, ByteOrder defaultOrder) { public double applyScaleOffset(Number value) { double convertedValue = value.doubleValue(); - return Math.round((convertedValue - offset) * scale); + if (astype.isIntegral()) { + return Math.round((convertedValue - offset) * scale); + } + return (convertedValue - offset) * scale; } public double removeScaleOffset(Number value) { double convertedValue = value.doubleValue(); + if (dtype.isIntegral()) { + return Math.round(convertedValue / scale + offset); + } return convertedValue / scale + offset; } diff --git a/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java index 89a4b0cdab..e1b2558a7e 100644 --- a/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java +++ b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java @@ -31,6 +31,7 @@ public class TestEnhancements { private static final Short SIGNED_SCALED_MAX = -2; private static final Short SIGNED_SCALED_FILL_VALUE = -4; + private static final Short SIGNED_SCALED_MISSING_VALUE = -3; @ClassRule public static final TemporaryFolder tempFolder = new TemporaryFolder(); @@ -71,7 +72,8 @@ public static void setUp() throws IOException, InvalidRangeException { builder.addVariable("enhanceAll", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.UNSIGNED, "true")) .addAttribute(new Attribute(CDM.SCALE_FACTOR, 10.0)).addAttribute(new Attribute(CDM.ADD_OFFSET, 10)) .addAttribute(new Attribute(CDM.VALID_MAX, SIGNED_SCALED_MAX)) - .addAttribute(new Attribute(CDM.FILL_VALUE, SIGNED_SCALED_FILL_VALUE)); + .addAttribute(new Attribute(CDM.FILL_VALUE, SIGNED_SCALED_FILL_VALUE)) + .addAttribute(new Attribute(CDM.MISSING_VALUE, SIGNED_SCALED_MISSING_VALUE)); // write data NetcdfFormatWriter writer = builder.build(); @@ -145,7 +147,7 @@ public void testConvertMissing() throws IOException { @Test public void testCombinedEnhancements() throws IOException { - double[] expected = new double[] {655320, Double.NaN, 655340, 655350, Double.NaN, 10, 20, 30, 40, 50}; + double[] expected = new double[] {655320, Double.NaN, Double.NaN, 655350, Double.NaN, 10, 20, 30, 40, 50}; Variable v = ncd.findVariable("enhanceAll"); Array data = v.read(); assertThat((double[]) data.copyTo1DJavaArray()).isEqualTo(expected); diff --git a/dap4/src/test/java/dap4/test/TestRemote.java b/dap4/src/test/java/dap4/test/TestRemote.java index 201c61b5b9..10ecf417f8 100644 --- a/dap4/src/test/java/dap4/test/TestRemote.java +++ b/dap4/src/test/java/dap4/test/TestRemote.java @@ -8,6 +8,7 @@ import dap4.core.util.DapConstants; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -31,6 +32,7 @@ * tests by adding fields to the TestCase object. */ +@Ignore("Disable while RemoteTest server is not working") @RunWith(Parameterized.class) public class TestRemote extends DapTestCommon implements Dap4ManifestIF { From 940cc63b2caa17e94a62b91a2210514af072dbd0 Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Thu, 6 Jul 2023 13:46:23 -0400 Subject: [PATCH 24/24] revert refactor --- .../src/main/java/ucar/nc2/dataset/VariableDS.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java index 691ba72e95..2ee21bc47d 100644 --- a/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java +++ b/cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java @@ -265,7 +265,8 @@ Array convert(Array data, Set enhancements) { if (enhancements.contains(Enhance.ApplyScaleOffset) && scaleOffset != null) { data = scaleOffset.removeScaleOffset(data); } - if (enhancements.contains(Enhance.ConvertMissing) && convertMissing != null) { + if (enhancements.contains(Enhance.ConvertMissing) && convertMissing != null + && (dataType == DataType.FLOAT || dataType == DataType.DOUBLE)) { data = convertMissing.convertMissing(data); } return data; @@ -680,7 +681,7 @@ public boolean hasMissingValue() { @Override public double[] getMissingValues() { - return convertMissing != null ? convertMissing.getMissingValues() : new double[0]; + return convertMissing != null ? convertMissing.getMissingValues() : new double[] {0}; } @Override @@ -759,7 +760,7 @@ public Number convertUnsigned(Number value) { } public Number convertUnsigned(Number value, DataType dataType) { - return unsignedConversion != null ? UnsignedConversion.convertUnsigned(value, dataType) : value; + return unsignedConversion != null ? unsignedConversion.convertUnsigned(value, dataType) : value; } @Override @@ -869,8 +870,7 @@ private void createEnhancements() { } } } - if (this.enhanceMode.contains(Enhance.ConvertMissing) - && (dataType == DataType.FLOAT || dataType == DataType.DOUBLE)) { + if (this.enhanceMode.contains(Enhance.ConvertMissing)) { this.convertMissing = ConvertMissing.createFromVariable(this); } }