From a86c29b695aec26d2027e0b972273dad17219e51 Mon Sep 17 00:00:00 2001 From: Hailey Johnson Date: Mon, 11 Mar 2024 17:41:43 -0700 Subject: [PATCH 1/5] revert switch to fuzzy equals --- cdm/core/src/main/java/ucar/nc2/util/Misc.java | 8 ++++---- .../java/ucar/nc2/filter/TestEnhancements.java | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/util/Misc.java b/cdm/core/src/main/java/ucar/nc2/util/Misc.java index 0f34f5c1c4..888c1dfe7e 100644 --- a/cdm/core/src/main/java/ucar/nc2/util/Misc.java +++ b/cdm/core/src/main/java/ucar/nc2/util/Misc.java @@ -97,8 +97,8 @@ public static boolean nearlyEquals(float a, float b) { } /** AbsoluteDifference is less than maxAbsDiff. */ - public static boolean nearlyEquals(float a, float b, float maxAbsDiff) { - return DoubleMath.fuzzyEquals(a, b, maxAbsDiff); + public static boolean nearlyEquals(float a, float b, float maxRelDiff) { + return relativeDifference(a, b) < maxRelDiff; } /** AbsoluteDifference is less than {@link #defaultMaxRelativeDiffDouble}. */ @@ -107,8 +107,8 @@ public static boolean nearlyEquals(double a, double b) { } /** AbsoluteDifference is less than maxAbsDiff. */ - public static boolean nearlyEquals(double a, double b, double maxAbsDiff) { - return DoubleMath.fuzzyEquals(a, b, maxAbsDiff); + public static boolean nearlyEquals(double a, double b, double maxRelDiff) { + return relativeDifference(a, b) < maxRelDiff; } /** AbsoluteDifference is less than maxAbsDiff. */ diff --git a/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java index e1b2558a7e..316340753e 100644 --- a/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java +++ b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java @@ -17,6 +17,7 @@ import static com.google.common.truth.Truth.assertThat; import java.io.IOException; +import java.util.Arrays; public class TestEnhancements { @@ -75,6 +76,14 @@ public static void setUp() throws IOException, InvalidRangeException { .addAttribute(new Attribute(CDM.FILL_VALUE, SIGNED_SCALED_FILL_VALUE)) .addAttribute(new Attribute(CDM.MISSING_VALUE, SIGNED_SCALED_MISSING_VALUE)); + // short data with small scale and offsets + Array smallVals = Array.factory(DataType.FLOAT, new int[] {dataLen}, missingData); + + builder.addVariable("smallVals", DataType.FLOAT, "dim") + .addAttribute(new Attribute(CDM.SCALE_FACTOR, 1e-12)).addAttribute(new Attribute(CDM.ADD_OFFSET, 1e-12)) + .addAttribute(new Attribute(CDM.FILL_VALUE, 110)); + + // write data NetcdfFormatWriter writer = builder.build(); writer.write(writer.findVariable("signedVar"), new int[1], signedData); @@ -85,6 +94,7 @@ public static void setUp() throws IOException, InvalidRangeException { writer.write(writer.findVariable("validMinMax"), new int[1], missingDataArray); writer.write(writer.findVariable("validRange"), new int[1], missingDataArray); writer.write(writer.findVariable("enhanceAll"), new int[1], enhanceAllArray); + writer.write(writer.findVariable("smallVals"), new int[1], smallVals); writer.close(); ncd = NetcdfDatasets.openDataset(filePath); } @@ -152,4 +162,11 @@ public void testCombinedEnhancements() throws IOException { Array data = v.read(); assertThat((double[]) data.copyTo1DJavaArray()).isEqualTo(expected); } + @Test + public void testConvertMissingWithSmallScaleAndOffset() throws IOException { + Variable v = ncd.findVariable("smallVals"); + Array data = v.read(); + assertThat(Double.isNaN(data.getDouble(2))).isTrue(); + assertThat(Double.isNaN(data.getDouble(1))).isFalse(); + } } From 748fc83a283636ae24d7cbf8f2ddd9978f8b30ca Mon Sep 17 00:00:00 2001 From: Hailey Johnson Date: Tue, 12 Mar 2024 08:38:13 -0700 Subject: [PATCH 2/5] spotless --- .../java/ucar/nc2/filter/TestEnhancements.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java index 316340753e..e60a4d29bf 100644 --- a/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java +++ b/cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java @@ -79,9 +79,8 @@ public static void setUp() throws IOException, InvalidRangeException { // short data with small scale and offsets Array smallVals = Array.factory(DataType.FLOAT, new int[] {dataLen}, missingData); - builder.addVariable("smallVals", DataType.FLOAT, "dim") - .addAttribute(new Attribute(CDM.SCALE_FACTOR, 1e-12)).addAttribute(new Attribute(CDM.ADD_OFFSET, 1e-12)) - .addAttribute(new Attribute(CDM.FILL_VALUE, 110)); + builder.addVariable("smallVals", DataType.FLOAT, "dim").addAttribute(new Attribute(CDM.SCALE_FACTOR, 1e-12)) + .addAttribute(new Attribute(CDM.ADD_OFFSET, 1e-12)).addAttribute(new Attribute(CDM.FILL_VALUE, 110)); // write data @@ -162,11 +161,12 @@ public void testCombinedEnhancements() throws IOException { Array data = v.read(); assertThat((double[]) data.copyTo1DJavaArray()).isEqualTo(expected); } + @Test public void testConvertMissingWithSmallScaleAndOffset() throws IOException { - Variable v = ncd.findVariable("smallVals"); - Array data = v.read(); - assertThat(Double.isNaN(data.getDouble(2))).isTrue(); - assertThat(Double.isNaN(data.getDouble(1))).isFalse(); + Variable v = ncd.findVariable("smallVals"); + Array data = v.read(); + assertThat(Double.isNaN(data.getDouble(2))).isTrue(); + assertThat(Double.isNaN(data.getDouble(1))).isFalse(); } } From 986debbf3d7d42c5d50760a7586ed8fcf39748f8 Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Tue, 12 Mar 2024 09:06:45 -0700 Subject: [PATCH 3/5] Update cdm/core/src/main/java/ucar/nc2/util/Misc.java Co-authored-by: Tara Drwenski --- cdm/core/src/main/java/ucar/nc2/util/Misc.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdm/core/src/main/java/ucar/nc2/util/Misc.java b/cdm/core/src/main/java/ucar/nc2/util/Misc.java index 888c1dfe7e..15f7be4d77 100644 --- a/cdm/core/src/main/java/ucar/nc2/util/Misc.java +++ b/cdm/core/src/main/java/ucar/nc2/util/Misc.java @@ -96,7 +96,7 @@ public static boolean nearlyEquals(float a, float b) { return nearlyEquals(a, b, defaultMaxRelativeDiffFloat); } - /** AbsoluteDifference is less than maxAbsDiff. */ + /** RelativeDifference is less than maxRelDiff. */ public static boolean nearlyEquals(float a, float b, float maxRelDiff) { return relativeDifference(a, b) < maxRelDiff; } From 3bb2909ef9c21539719c1c981ffb65b255a5d9b6 Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Tue, 12 Mar 2024 09:06:50 -0700 Subject: [PATCH 4/5] Update cdm/core/src/main/java/ucar/nc2/util/Misc.java Co-authored-by: Tara Drwenski --- cdm/core/src/main/java/ucar/nc2/util/Misc.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdm/core/src/main/java/ucar/nc2/util/Misc.java b/cdm/core/src/main/java/ucar/nc2/util/Misc.java index 15f7be4d77..86fd2619b8 100644 --- a/cdm/core/src/main/java/ucar/nc2/util/Misc.java +++ b/cdm/core/src/main/java/ucar/nc2/util/Misc.java @@ -106,7 +106,7 @@ public static boolean nearlyEquals(double a, double b) { return nearlyEquals(a, b, defaultMaxRelativeDiffDouble); } - /** AbsoluteDifference is less than maxAbsDiff. */ + /** RelativeDifference is less than maxRelDiff. */ public static boolean nearlyEquals(double a, double b, double maxRelDiff) { return relativeDifference(a, b) < maxRelDiff; } From d6eca0717034f05a231cdd44000d560f6310286a Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 12 Mar 2024 13:37:38 -0600 Subject: [PATCH 5/5] Ensure location is not null so aggregation cache is used --- .../nc2/ft2/coverage/CoverageDatasetFactory.java | 15 ++++++++++++++- .../internal/ncml/s3/S3AggScanFeatureType.java | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/ft2/coverage/CoverageDatasetFactory.java b/cdm/core/src/main/java/ucar/nc2/ft2/coverage/CoverageDatasetFactory.java index 32835fcb59..83e31b374f 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft2/coverage/CoverageDatasetFactory.java +++ b/cdm/core/src/main/java/ucar/nc2/ft2/coverage/CoverageDatasetFactory.java @@ -126,8 +126,21 @@ public static Optional openGrib(String endpoint) { } + /** + * @deprecated Use openNcmlString(String, String) + */ + @Deprecated public static Optional openNcmlString(String ncml) throws IOException { - NetcdfDataset ncd = NetcdfDatasets.openNcmlDataset(new StringReader(ncml), null, null); + return openNcmlString(ncml, null); + } + + /** + * @param ncml the NcML as a String + * @param location the URL location string of the NcML document, + * or may be just a unique name for caching purposes (if null, aggregation cache will not be used). + */ + public static Optional openNcmlString(String ncml, String location) throws IOException { + NetcdfDataset ncd = NetcdfDatasets.openNcmlDataset(new StringReader(ncml), location, null); DtCoverageDataset gds = new DtCoverageDataset(ncd); if (!gds.getGrids().isEmpty()) { diff --git a/cdm/s3/src/test/java/ucar/nc2/internal/ncml/s3/S3AggScanFeatureType.java b/cdm/s3/src/test/java/ucar/nc2/internal/ncml/s3/S3AggScanFeatureType.java index 4301623f21..fc4bbf2705 100644 --- a/cdm/s3/src/test/java/ucar/nc2/internal/ncml/s3/S3AggScanFeatureType.java +++ b/cdm/s3/src/test/java/ucar/nc2/internal/ncml/s3/S3AggScanFeatureType.java @@ -96,7 +96,7 @@ public void testAggNewOpenAsCoverage() throws IOException, InvalidRangeException ncml = ncmlStream.collect(Collectors.joining()); } assertThat(ncml).isNotNull(); - Optional fdc = CoverageDatasetFactory.openNcmlString(ncml); + Optional fdc = CoverageDatasetFactory.openNcmlString(ncml, null); assertThat(fdc.isPresent()).isTrue(); List cc = fdc.get().getCoverageCollections(); checkCoverages(cc);