From bd6e7da8fe37648c771dfdd1cc0cab8293528474 Mon Sep 17 00:00:00 2001 From: Sean Arms <67096+lesserwhirls@users.noreply.github.com> Date: Thu, 13 Oct 2022 08:11:09 -0600 Subject: [PATCH 01/70] Add runtime support for AWS Security Token Service Also, use the AWS BOM to manage AWS related dependencies. --- cdm/s3/build.gradle | 2 ++ netcdf-java-platform/build.gradle | 9 ++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cdm/s3/build.gradle b/cdm/s3/build.gradle index b9c945e3d7..8bc59d5887 100644 --- a/cdm/s3/build.gradle +++ b/cdm/s3/build.gradle @@ -26,6 +26,8 @@ dependencies { implementation 'software.amazon.awssdk:apache-client' implementation 'com.google.code.findbugs:jsr305' + runtimeOnly 'software.amazon.awssdk:sts' + testImplementation project(':cdm:cdm-radial') testImplementation project(':cdm-test-utils') diff --git a/netcdf-java-platform/build.gradle b/netcdf-java-platform/build.gradle index 17c061bcc0..2bb2e4d600 100644 --- a/netcdf-java-platform/build.gradle +++ b/netcdf-java-platform/build.gradle @@ -9,6 +9,8 @@ javaPlatform { } dependencies { + def awsVersion = '2.17.290' + api enforcedPlatform("software.amazon.awssdk:bom:${awsVersion}") api enforcedPlatform('com.fasterxml.jackson:jackson-bom:2.13.2.20220324') constraints { // Note: The depVersion variable is defined in gradle/any/shared-mvn-coords.gradle and is used for dependencies @@ -43,9 +45,10 @@ dependencies { // cdm-vis5d (vis5d IOSP) api 'edu.wisc.ssec:visad:2.0-20130124' - // cdm-s3 (S3RandomAccessFile) - api 'software.amazon.awssdk:s3:2.17.156' - api 'software.amazon.awssdk:apache-client:2.17.156' + // toolsUI (everything else is happy with the aws BOM) + api "software.amazon.awssdk:s3:2.17.290:${awsVersion}" + api "software.amazon.awssdk:apache-client:${awsVersion}" + runtime "software.amazon.awssdk:sts:${awsVersion}" // apache httpclient api 'org.apache.httpcomponents:httpclient:4.5.13' From 7573ce633506fee95a0c4ac59365311b96269aed Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Thu, 13 Oct 2022 10:05:55 -0700 Subject: [PATCH 02/70] upgrade protobuf and jackson libraries --- gradle/any/shared-mvn-coords.gradle | 2 +- legacy/build.gradle | 2 +- netcdf-java-platform/build.gradle | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/gradle/any/shared-mvn-coords.gradle b/gradle/any/shared-mvn-coords.gradle index dbeef006b1..653185fed0 100644 --- a/gradle/any/shared-mvn-coords.gradle +++ b/gradle/any/shared-mvn-coords.gradle @@ -22,5 +22,5 @@ ext { // coords. depVersion = [:] depVersion.slf4j = '1.7.28' - depVersion.protobuf = '3.19.3' + depVersion.protobuf = '3.21.7' } diff --git a/legacy/build.gradle b/legacy/build.gradle index 1443e3ecf7..0334436b07 100644 --- a/legacy/build.gradle +++ b/legacy/build.gradle @@ -23,7 +23,7 @@ dependencies { compile 'com.amazonaws:aws-java-sdk-s3' // For CrawlableDatasetAmazonS3. constraints { - implementation('com.fasterxml.jackson.core:jackson-databind:2.13.2.1') { + implementation('com.fasterxml.jackson.core:jackson-databind:2.14.0-rc1') { because 'Replacement v2.6.7.3, which is recomended by aws-java-sdk v1.x for those who do' 'not need java 6 compatibility - see https://github.com/aws/aws-sdk-java#cve-2017-15095--cve-2018-7489' } diff --git a/netcdf-java-platform/build.gradle b/netcdf-java-platform/build.gradle index 2bb2e4d600..ebf22b3a68 100644 --- a/netcdf-java-platform/build.gradle +++ b/netcdf-java-platform/build.gradle @@ -11,7 +11,7 @@ javaPlatform { dependencies { def awsVersion = '2.17.290' api enforcedPlatform("software.amazon.awssdk:bom:${awsVersion}") - api enforcedPlatform('com.fasterxml.jackson:jackson-bom:2.13.2.20220324') + api enforcedPlatform('com.fasterxml.jackson:jackson-bom:2.14.0-rc1') constraints { // Note: The depVersion variable is defined in gradle/any/shared-mvn-coords.gradle and is used for dependencies // that are used in different configurations of a build that need access to the full maven coordinates. From ddb50be51b92e5b09eaa0b32d42ce27027c62aa0 Mon Sep 17 00:00:00 2001 From: Sean Arms <67096+lesserwhirls@users.noreply.github.com> Date: Fri, 14 Oct 2022 07:52:16 -0600 Subject: [PATCH 03/70] Exclude netty nio client from aws sts dependency --- cdm/s3/build.gradle | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cdm/s3/build.gradle b/cdm/s3/build.gradle index 8bc59d5887..d5f1c988b6 100644 --- a/cdm/s3/build.gradle +++ b/cdm/s3/build.gradle @@ -26,7 +26,10 @@ dependencies { implementation 'software.amazon.awssdk:apache-client' implementation 'com.google.code.findbugs:jsr305' - runtimeOnly 'software.amazon.awssdk:sts' + runtimeOnly('software.amazon.awssdk:sts') { + // see above comment about awssdk and netty-nio-client + exclude group: 'software.amazon.awssdk', module: 'netty-nio-client' + } testImplementation project(':cdm:cdm-radial') testImplementation project(':cdm-test-utils') From 19f9476ed8e605e04ab6013a90ba59dbbb2d17d3 Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Fri, 14 Oct 2022 11:17:11 -0700 Subject: [PATCH 04/70] clean up zarr build and check for null location --- cdm/zarr/build.gradle | 13 ++----------- .../main/java/thredds/inventory/zarr/MFileZip.java | 2 +- netcdf-java-platform/build.gradle | 5 ++++- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/cdm/zarr/build.gradle b/cdm/zarr/build.gradle index a534adfab8..9ede9ddcc0 100644 --- a/cdm/zarr/build.gradle +++ b/cdm/zarr/build.gradle @@ -3,26 +3,18 @@ ext.title = 'CDM Zarr support library' apply from: "$rootDir/gradle/any/dependencies.gradle" apply from: "$rootDir/gradle/any/java-library.gradle" -apply from: "$rootDir/gradle/any/protobuf.gradle" dependencies { api enforcedPlatform(project(':netcdf-java-platform')) testImplementation enforcedPlatform(project(':netcdf-java-testing-platform')) api project(':cdm:cdm-core') - api project(':cdm:cdm-s3') - implementation 'org.jdom:jdom2' - implementation 'com.google.code.findbugs:jsr305' - implementation 'com.beust:jcommander' - implementation 'com.google.protobuf:protobuf-java' - implementation 'com.google.re2j:re2j' implementation 'org.slf4j:slf4j-api' - implementation 'com.google.guava:guava' implementation 'com.fasterxml.jackson.core:jackson-core' implementation 'com.fasterxml.jackson.core:jackson-databind' - implementation('software.amazon.awssdk:s3') { + testImplementation('software.amazon.awssdk:s3') { // exclude netty nio client due to open CVEs. See // https://github.com/aws/aws-sdk-java-v2/issues/1632 // we don't use the nio http client in our S3 related code, @@ -32,8 +24,7 @@ dependencies { exclude group: 'software.amazon.awssdk', module: 'netty-nio-client' } - implementation 'software.amazon.awssdk:apache-client' - + testImplementation project(':cdm:cdm-s3') testImplementation project(':cdm-test-utils') testImplementation 'com.google.truth:truth' diff --git a/cdm/zarr/src/main/java/thredds/inventory/zarr/MFileZip.java b/cdm/zarr/src/main/java/thredds/inventory/zarr/MFileZip.java index 8dda9c3aa6..7a5835d41e 100644 --- a/cdm/zarr/src/main/java/thredds/inventory/zarr/MFileZip.java +++ b/cdm/zarr/src/main/java/thredds/inventory/zarr/MFileZip.java @@ -197,7 +197,7 @@ public String getProtocol() { @Override public boolean canProvide(String location) { - return location.contains(ext); + return location != null && location.contains(ext); } @Nullable diff --git a/netcdf-java-platform/build.gradle b/netcdf-java-platform/build.gradle index ebf22b3a68..9771dcdbb7 100644 --- a/netcdf-java-platform/build.gradle +++ b/netcdf-java-platform/build.gradle @@ -11,7 +11,6 @@ javaPlatform { dependencies { def awsVersion = '2.17.290' api enforcedPlatform("software.amazon.awssdk:bom:${awsVersion}") - api enforcedPlatform('com.fasterxml.jackson:jackson-bom:2.14.0-rc1') constraints { // Note: The depVersion variable is defined in gradle/any/shared-mvn-coords.gradle and is used for dependencies // that are used in different configurations of a build that need access to the full maven coordinates. @@ -45,6 +44,10 @@ dependencies { // cdm-vis5d (vis5d IOSP) api 'edu.wisc.ssec:visad:2.0-20130124' + // cdm-zarr + api 'com.fasterxml.jackson.core:jackson-core:2.14.0-rc1' + api 'com.fasterxml.jackson.core:jackson-databind:2.14.0-rc1' + // toolsUI (everything else is happy with the aws BOM) api "software.amazon.awssdk:s3:2.17.290:${awsVersion}" api "software.amazon.awssdk:apache-client:${awsVersion}" From 625f6693c5c3f46c6d73ba3dad059a9107057d8d Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Fri, 14 Oct 2022 11:29:44 -0700 Subject: [PATCH 05/70] remove unused constraint on jackson --- legacy/build.gradle | 6 ------ 1 file changed, 6 deletions(-) diff --git a/legacy/build.gradle b/legacy/build.gradle index 0334436b07..cc8255bf3f 100644 --- a/legacy/build.gradle +++ b/legacy/build.gradle @@ -22,12 +22,6 @@ dependencies { compile 'org.slf4j:slf4j-api' compile 'com.amazonaws:aws-java-sdk-s3' // For CrawlableDatasetAmazonS3. - constraints { - implementation('com.fasterxml.jackson.core:jackson-databind:2.14.0-rc1') { - because 'Replacement v2.6.7.3, which is recomended by aws-java-sdk v1.x for those who do' - 'not need java 6 compatibility - see https://github.com/aws/aws-sdk-java#cve-2017-15095--cve-2018-7489' - } - } testImplementation project(':cdm-test-utils') // These are all for Spock. From aac292ed399041a9f4455a483b2b63b81a082061 Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Fri, 14 Oct 2022 13:00:46 -0700 Subject: [PATCH 06/70] add jackson BOM back to platform --- netcdf-java-platform/build.gradle | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/netcdf-java-platform/build.gradle b/netcdf-java-platform/build.gradle index 9771dcdbb7..db66d8fef9 100644 --- a/netcdf-java-platform/build.gradle +++ b/netcdf-java-platform/build.gradle @@ -10,7 +10,9 @@ javaPlatform { dependencies { def awsVersion = '2.17.290' + def jacksonVersion = '2.14.0-rc1' api enforcedPlatform("software.amazon.awssdk:bom:${awsVersion}") + api enforcedPlatform("com.fasterxml.jackson:jackson-bom:${jacksonVersion}") constraints { // Note: The depVersion variable is defined in gradle/any/shared-mvn-coords.gradle and is used for dependencies // that are used in different configurations of a build that need access to the full maven coordinates. @@ -44,9 +46,9 @@ dependencies { // cdm-vis5d (vis5d IOSP) api 'edu.wisc.ssec:visad:2.0-20130124' - // cdm-zarr - api 'com.fasterxml.jackson.core:jackson-core:2.14.0-rc1' - api 'com.fasterxml.jackson.core:jackson-databind:2.14.0-rc1' + // netcdfAll (everything else is happy with the Jackson BOM) + api "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" + api "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" // toolsUI (everything else is happy with the aws BOM) api "software.amazon.awssdk:s3:2.17.290:${awsVersion}" From 2ab20236ecece7e7f861115b6d0dd46d624b809a Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 29 Sep 2022 11:12:18 -0600 Subject: [PATCH 07/70] Add test that internal attribute is not written --- cdm/core/src/test/java/ucar/nc2/write/TestWrite.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cdm/core/src/test/java/ucar/nc2/write/TestWrite.java b/cdm/core/src/test/java/ucar/nc2/write/TestWrite.java index c37830d115..9c368ee949 100644 --- a/cdm/core/src/test/java/ucar/nc2/write/TestWrite.java +++ b/cdm/core/src/test/java/ucar/nc2/write/TestWrite.java @@ -4,6 +4,8 @@ */ package ucar.nc2.write; +import static com.google.common.truth.Truth.assertThat; + import org.junit.*; import org.junit.rules.TemporaryFolder; import org.junit.runners.MethodSorters; @@ -19,6 +21,7 @@ import ucar.nc2.NetcdfFile; import ucar.nc2.NetcdfFiles; import ucar.nc2.Variable; +import ucar.nc2.constants.CDM; /** Test NetcdfFormatWriter */ @FixMethodOrder(MethodSorters.NAME_ASCENDING) @@ -72,6 +75,7 @@ public static void setupClass() throws IOException { writerb.addAttribute(new Attribute("versionI", 1)); writerb.addAttribute(new Attribute("versionS", (short) 2)); writerb.addAttribute(new Attribute("versionB", (byte) 3)); + writerb.addAttribute(new Attribute(CDM.NCPROPERTIES, "test internal attribute is removed when writing")); // test some errors try { @@ -208,6 +212,13 @@ public static void setupClass() throws IOException { } } + @Test + public void shouldRemoveInternalAttribute() throws IOException { + try (NetcdfFile netcdfFile = NetcdfFiles.open(writerLocation)) { + assertThat(netcdfFile.findGlobalAttributeIgnoreCase(CDM.NCPROPERTIES)).isNull(); + } + } + @Test public void testReadBack() throws IOException { try (NetcdfFile ncfile = NetcdfFiles.open(writerLocation)) { From fe027176ba68c18e011e1845ac149aaaf3400941 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 28 Sep 2022 16:56:28 -0600 Subject: [PATCH 08/70] Do not write attributes that are only used internally in netcdf3 --- .../ucar/nc2/internal/iosp/netcdf3/N3headerWriter.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/internal/iosp/netcdf3/N3headerWriter.java b/cdm/core/src/main/java/ucar/nc2/internal/iosp/netcdf3/N3headerWriter.java index 1fe9275522..2cb14b793e 100644 --- a/cdm/core/src/main/java/ucar/nc2/internal/iosp/netcdf3/N3headerWriter.java +++ b/cdm/core/src/main/java/ucar/nc2/internal/iosp/netcdf3/N3headerWriter.java @@ -10,6 +10,8 @@ import java.nio.charset.StandardCharsets; import java.util.Formatter; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import ucar.ma2.Array; import ucar.ma2.DataType; import ucar.ma2.IndexIterator; @@ -215,7 +217,10 @@ int sizeHeader(boolean largeFile) { private void writeAtts(Iterable atts, Formatter fout) throws IOException { - int n = Iterables.size(atts); + final List attributesToWrite = StreamSupport.stream(atts.spliterator(), false) + .filter(att -> !Attribute.isspecial(att)).collect(Collectors.toList()); + + int n = Iterables.size(attributesToWrite); if (n == 0) { raf.writeInt(0); raf.writeInt(0); @@ -225,7 +230,7 @@ private void writeAtts(Iterable atts, Formatter fout) throws IOExcept } int count = 0; - for (Attribute att : atts) { + for (Attribute att : attributesToWrite) { if (fout != null) fout.format("***att %d pos= %d%n", count, raf.getFilePointer()); From 61a39496f0660de86f3c72fe05ae123d7c94ac7c Mon Sep 17 00:00:00 2001 From: "R. Schmunk" <8718035+rschmunk@users.noreply.github.com> Date: Mon, 17 Oct 2022 23:05:23 -0400 Subject: [PATCH 09/70] ZarrIosp: Watch for floating point fill values encoded as Strings If a Zarr variable's fill value attribute had a String value ("NaN", "Infinity", or "-Infinity") or was reported as null, a ClassCastException would occur when IospHelper tried to cast the String to a Number. --- .../java/ucar/nc2/iosp/zarr/ZarrIosp.java | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java b/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java index 2649c89d1d..5b4ce59a31 100644 --- a/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java +++ b/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java @@ -19,13 +19,14 @@ import ucar.unidata.io.zarr.RandomAccessDirectory; import java.io.IOException; +import java.lang.invoke.MethodHandles; /** * IOSP for reading/writing Zarr/NCZarr formats */ public class ZarrIosp extends AbstractIOServiceProvider { - static final Logger logger = LoggerFactory.getLogger(ZarrIosp.class); + static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup ( ).lookupClass ( )); private static final String fileTypeId = "Zarr"; private static final String fileTypeDescription = "Zarr v2 formatted dataset"; @@ -68,9 +69,56 @@ public Array readData(Variable v2, Section section) { ZarrHeader.VInfo vinfo = (ZarrHeader.VInfo) v2.getSPobject(); DataType dataType = v2.getDataType(); + logger.debug("DataType is '{}'", dataType); + + // Watch for floating point fill values encoded as Strings + final Object fillValueObj = vinfo.getFillValue(); + + Object fillValue = fillValueObj; + + if (fillValueObj instanceof String) { + final String fillValueStr = (String)fillValueObj; + + logger.debug("Fill value is String with value '{}'", fillValueStr); + + if ("".equals(fillValueStr)) { + fillValue = null; + } else { + switch (dataType) { + case FLOAT: + if ("NaN".equals(fillValueStr)) { + fillValue = Float.NaN; + } else if ("Infinity".equals(fillValueStr)) { + fillValue = Float.POSITIVE_INFINITY; + } else if ("-Infinity".equals(fillValueStr)) { + fillValue = Float.NEGATIVE_INFINITY; + } else { + logger.debug("String value '{}' not handled for float fill value", fillValueStr); + } + break; + + case DOUBLE: + if ("NaN".equals(fillValueStr)) { + fillValue = Float.NaN; + } else if ("Infinity".equals(fillValueStr)) { + fillValue = Float.POSITIVE_INFINITY; + } else if ("-Infinity".equals(fillValueStr)) { + fillValue = Float.NEGATIVE_INFINITY; + } else { + logger.debug("String value '{}' not handled for float fill value", fillValueStr); + } + break; + + default: + logger.debug("String value '{}' not handled for {} fill value", fillValueStr, dataType); + break; + } + } + } + // create layout object Layout layout = new ZarrLayoutBB(v2, section, this.raf); - Object data = IospHelper.readDataFill((LayoutBB) layout, dataType, vinfo.getFillValue()); + Object data = IospHelper.readDataFill((LayoutBB) layout, dataType, fillValue); Array array = Array.factory(dataType, section.getShape(), data); if (vinfo.getOrder() == ZArray.Order.F) { @@ -85,3 +133,4 @@ public Array readData(Variable v2, Section section) { return array; } } + From 983355b8675dacba6dc323ac227e78670aae2577 Mon Sep 17 00:00:00 2001 From: "R. Schmunk" <8718035+rschmunk@users.noreply.github.com> Date: Mon, 17 Oct 2022 23:10:45 -0400 Subject: [PATCH 10/70] Update ZarrIosp handling of Double fill values Too much copy pasta when creating Double case. --- cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java b/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java index 5b4ce59a31..c76bb4c5d1 100644 --- a/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java +++ b/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java @@ -99,11 +99,11 @@ public Array readData(Variable v2, Section section) { case DOUBLE: if ("NaN".equals(fillValueStr)) { - fillValue = Float.NaN; + fillValue = Double.NaN; } else if ("Infinity".equals(fillValueStr)) { - fillValue = Float.POSITIVE_INFINITY; + fillValue = Double.POSITIVE_INFINITY; } else if ("-Infinity".equals(fillValueStr)) { - fillValue = Float.NEGATIVE_INFINITY; + fillValue = Double.NEGATIVE_INFINITY; } else { logger.debug("String value '{}' not handled for float fill value", fillValueStr); } From 7731f8c3360a885c2d9d2bb13dec08fb30e343f6 Mon Sep 17 00:00:00 2001 From: "R. Schmunk" <8718035+rschmunk@users.noreply.github.com> Date: Tue, 18 Oct 2022 00:01:17 -0400 Subject: [PATCH 11/70] ZarrIosp: spotless --- cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java b/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java index c76bb4c5d1..3e11ed1e34 100644 --- a/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java +++ b/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java @@ -26,7 +26,7 @@ */ public class ZarrIosp extends AbstractIOServiceProvider { - static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup ( ).lookupClass ( )); + static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); private static final String fileTypeId = "Zarr"; private static final String fileTypeDescription = "Zarr v2 formatted dataset"; @@ -77,7 +77,7 @@ public Array readData(Variable v2, Section section) { Object fillValue = fillValueObj; if (fillValueObj instanceof String) { - final String fillValueStr = (String)fillValueObj; + final String fillValueStr = (String) fillValueObj; logger.debug("Fill value is String with value '{}'", fillValueStr); @@ -133,4 +133,3 @@ public Array readData(Variable v2, Section section) { return array; } } - From 546e725de5c676ea8a9fdb86fde8c038c3c56441 Mon Sep 17 00:00:00 2001 From: "R. Schmunk" <8718035+rschmunk@users.noreply.github.com> Date: Tue, 18 Oct 2022 00:38:04 -0400 Subject: [PATCH 12/70] ZarrIosp One more float vs double copypasta. --- cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java b/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java index 3e11ed1e34..e522ef02c2 100644 --- a/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java +++ b/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java @@ -105,7 +105,7 @@ public Array readData(Variable v2, Section section) { } else if ("-Infinity".equals(fillValueStr)) { fillValue = Double.NEGATIVE_INFINITY; } else { - logger.debug("String value '{}' not handled for float fill value", fillValueStr); + logger.debug("String value '{}' not handled for double fill value", fillValueStr); } break; From 799f4ab239f4ea70da0c2a230c28f5594e4fe54c Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Fri, 21 Oct 2022 13:37:13 -0700 Subject: [PATCH 13/70] add test and test data; remove unused data --- .../main/java/ucar/nc2/iosp/zarr/ZArray.java | 5 +- .../java/ucar/nc2/iosp/zarr/ZarrIosp.java | 47 ++++++++++-------- .../.zgroup | 0 .../data/fill_values.zarr/double_inf/.zarray | 22 ++++++++ .../data/fill_values.zarr/double_nan/.zarray | 22 ++++++++ .../data/fill_values.zarr/double_ninf/.zarray | 22 ++++++++ .../data/fill_values.zarr/float_inf/.zarray | 22 ++++++++ .../data/fill_values.zarr/float_nan/.zarray | 22 ++++++++ .../data/fill_values.zarr/float_ninf/.zarray | 22 ++++++++ .../src/test/data/group_with_attrs/.zattrs | 3 -- .../group_with_attrs/F_order_array/.zarray | 16 ------ .../group_with_attrs/F_order_array/.zattrs | 10 ---- .../data/group_with_attrs/F_order_array/0.0 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/0.1 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/0.2 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/0.3 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/1.0 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/1.1 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/1.2 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/1.3 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/2.0 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/2.1 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/2.2 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/2.3 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/3.0 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/3.1 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/3.2 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/3.3 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/4.0 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/4.1 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/4.2 | Bin 80 -> 0 bytes .../data/group_with_attrs/F_order_array/4.3 | Bin 80 -> 0 bytes .../test/data/group_with_attrs/nested/.zarray | 17 ------- .../src/test/data/group_with_attrs/nested/0/0 | Bin 200 -> 0 bytes .../src/test/data/group_with_attrs/nested/0/1 | Bin 200 -> 0 bytes .../src/test/data/group_with_attrs/nested/1/0 | Bin 200 -> 0 bytes .../src/test/data/group_with_attrs/nested/1/1 | Bin 200 -> 0 bytes .../group_with_attrs/partial_fill1/.zarray | 16 ------ .../data/group_with_attrs/partial_fill1/0.0 | Bin 400 -> 0 bytes .../group_with_attrs/partial_fill2/.zarray | 16 ------ .../data/group_with_attrs/partial_fill2/1.1 | Bin 400 -> 0 bytes .../group_with_attrs/uninitialized/.zarray | 16 ------ .../src/test/data/group_with_dims/.zgroup | 3 -- .../test/data/group_with_dims/var1D/.zarray | 14 ------ .../src/test/data/group_with_dims/var1D/0 | Bin 20 -> 0 bytes .../src/test/data/group_with_dims/var1D/1 | Bin 20 -> 0 bytes .../src/test/data/group_with_dims/var1D/2 | Bin 20 -> 0 bytes .../src/test/data/group_with_dims/var1D/3 | Bin 20 -> 0 bytes .../test/data/group_with_dims/var2D/.zarray | 16 ------ .../src/test/data/group_with_dims/var2D/0.0 | Bin 100 -> 0 bytes .../src/test/data/group_with_dims/var2D/0.1 | Bin 100 -> 0 bytes .../src/test/data/group_with_dims/var2D/0.2 | Bin 100 -> 0 bytes .../src/test/data/group_with_dims/var2D/0.3 | Bin 100 -> 0 bytes .../src/test/data/group_with_dims/var2D/1.0 | Bin 100 -> 0 bytes .../src/test/data/group_with_dims/var2D/1.1 | Bin 100 -> 0 bytes .../src/test/data/group_with_dims/var2D/1.2 | Bin 100 -> 0 bytes .../src/test/data/group_with_dims/var2D/1.3 | Bin 100 -> 0 bytes .../src/test/data/group_with_dims/var2D/2.0 | Bin 100 -> 0 bytes .../src/test/data/group_with_dims/var2D/2.1 | Bin 100 -> 0 bytes .../src/test/data/group_with_dims/var2D/2.2 | Bin 100 -> 0 bytes .../src/test/data/group_with_dims/var2D/2.3 | Bin 100 -> 0 bytes .../src/test/data/group_with_dims/var2D/3.0 | Bin 100 -> 0 bytes .../src/test/data/group_with_dims/var2D/3.1 | Bin 100 -> 0 bytes .../src/test/data/group_with_dims/var2D/3.2 | Bin 100 -> 0 bytes .../src/test/data/group_with_dims/var2D/3.3 | Bin 100 -> 0 bytes .../test/data/group_with_dims/var3D/.zarray | 18 ------- .../src/test/data/group_with_dims/var3D/0.0.0 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/0.0.1 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/0.0.2 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/0.0.3 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/0.1.0 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/0.1.1 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/0.1.2 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/0.1.3 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/0.2.0 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/0.2.1 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/0.2.2 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/0.2.3 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/0.3.0 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/0.3.1 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/0.3.2 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/0.3.3 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/1.0.0 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/1.0.1 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/1.0.2 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/1.0.3 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/1.1.0 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/1.1.1 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/1.1.2 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/1.1.3 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/1.2.0 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/1.2.1 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/1.2.2 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/1.2.3 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/1.3.0 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/1.3.1 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/1.3.2 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/1.3.3 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/2.0.0 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/2.0.1 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/2.0.2 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/2.0.3 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/2.1.0 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/2.1.1 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/2.1.2 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/2.1.3 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/2.2.0 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/2.2.1 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/2.2.2 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/2.2.3 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/2.3.0 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/2.3.1 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/2.3.2 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/2.3.3 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/3.0.0 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/3.0.1 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/3.0.2 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/3.0.3 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/3.1.0 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/3.1.1 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/3.1.2 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/3.1.3 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/3.2.0 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/3.2.1 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/3.2.2 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/3.2.3 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/3.3.0 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/3.3.1 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/3.3.2 | Bin 500 -> 0 bytes .../src/test/data/group_with_dims/var3D/3.3.3 | Bin 500 -> 0 bytes .../test/data/group_with_dims/var4D/.zarray | 20 -------- .../test/data/group_with_dims/var4D/0.0.0.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.0.0.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.0.0.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.0.0.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.0.1.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.0.1.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.0.1.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.0.1.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.0.2.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.0.2.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.0.2.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.0.2.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.0.3.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.0.3.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.0.3.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.0.3.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.1.0.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.1.0.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.1.0.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.1.0.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.1.1.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.1.1.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.1.1.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.1.1.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.1.2.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.1.2.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.1.2.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.1.2.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.1.3.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.1.3.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.1.3.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.1.3.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.2.0.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.2.0.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.2.0.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.2.0.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.2.1.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.2.1.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.2.1.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.2.1.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.2.2.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.2.2.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.2.2.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.2.2.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.2.3.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.2.3.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.2.3.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.2.3.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.3.0.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.3.0.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.3.0.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.3.0.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.3.1.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.3.1.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.3.1.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.3.1.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.3.2.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.3.2.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.3.2.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.3.2.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.3.3.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.3.3.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.3.3.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/0.3.3.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.0.0.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.0.0.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.0.0.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.0.0.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.0.1.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.0.1.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.0.1.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.0.1.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.0.2.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.0.2.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.0.2.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.0.2.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.0.3.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.0.3.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.0.3.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.0.3.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.1.0.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.1.0.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.1.0.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.1.0.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.1.1.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.1.1.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.1.1.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.1.1.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.1.2.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.1.2.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.1.2.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.1.2.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.1.3.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.1.3.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.1.3.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.1.3.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.2.0.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.2.0.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.2.0.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.2.0.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.2.1.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.2.1.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.2.1.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.2.1.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.2.2.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.2.2.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.2.2.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.2.2.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.2.3.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.2.3.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.2.3.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.2.3.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.3.0.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.3.0.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.3.0.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.3.0.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.3.1.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.3.1.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.3.1.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.3.1.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.3.2.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.3.2.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.3.2.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.3.2.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.3.3.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.3.3.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.3.3.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/1.3.3.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.0.0.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.0.0.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.0.0.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.0.0.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.0.1.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.0.1.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.0.1.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.0.1.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.0.2.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.0.2.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.0.2.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.0.2.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.0.3.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.0.3.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.0.3.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.0.3.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.1.0.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.1.0.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.1.0.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.1.0.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.1.1.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.1.1.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.1.1.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.1.1.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.1.2.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.1.2.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.1.2.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.1.2.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.1.3.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.1.3.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.1.3.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.1.3.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.2.0.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.2.0.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.2.0.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.2.0.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.2.1.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.2.1.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.2.1.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.2.1.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.2.2.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.2.2.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.2.2.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.2.2.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.2.3.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.2.3.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.2.3.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.2.3.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.3.0.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.3.0.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.3.0.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.3.0.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.3.1.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.3.1.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.3.1.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.3.1.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.3.2.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.3.2.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.3.2.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.3.2.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.3.3.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.3.3.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.3.3.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/2.3.3.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.0.0.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.0.0.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.0.0.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.0.0.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.0.1.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.0.1.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.0.1.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.0.1.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.0.2.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.0.2.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.0.2.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.0.2.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.0.3.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.0.3.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.0.3.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.0.3.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.1.0.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.1.0.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.1.0.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.1.0.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.1.1.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.1.1.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.1.1.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.1.1.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.1.2.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.1.2.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.1.2.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.1.2.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.1.3.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.1.3.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.1.3.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.1.3.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.2.0.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.2.0.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.2.0.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.2.0.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.2.1.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.2.1.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.2.1.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.2.1.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.2.2.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.2.2.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.2.2.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.2.2.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.2.3.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.2.3.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.2.3.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.2.3.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.3.0.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.3.0.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.3.0.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.3.0.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.3.1.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.3.1.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.3.1.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.3.1.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.3.2.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.3.2.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.3.2.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.3.2.3 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.3.3.0 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.3.3.1 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.3.3.2 | Bin 2500 -> 0 bytes .../test/data/group_with_dims/var4D/3.3.3.3 | Bin 2500 -> 0 bytes .../java/ucar/nc2/iosp/zarr/TestZarrIosp.java | 23 +++++++++ 388 files changed, 186 insertions(+), 186 deletions(-) rename cdm/zarr/src/test/data/{group_with_attrs => fill_values.zarr}/.zgroup (100%) create mode 100644 cdm/zarr/src/test/data/fill_values.zarr/double_inf/.zarray create mode 100644 cdm/zarr/src/test/data/fill_values.zarr/double_nan/.zarray create mode 100644 cdm/zarr/src/test/data/fill_values.zarr/double_ninf/.zarray create mode 100644 cdm/zarr/src/test/data/fill_values.zarr/float_inf/.zarray create mode 100644 cdm/zarr/src/test/data/fill_values.zarr/float_nan/.zarray create mode 100644 cdm/zarr/src/test/data/fill_values.zarr/float_ninf/.zarray delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/.zattrs delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/.zarray delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/.zattrs delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/4.0 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/4.1 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/4.2 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/F_order_array/4.3 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/nested/.zarray delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/nested/0/0 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/nested/0/1 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/nested/1/0 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/nested/1/1 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/partial_fill1/.zarray delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/partial_fill1/0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/partial_fill2/.zarray delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/partial_fill2/1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_attrs/uninitialized/.zarray delete mode 100644 cdm/zarr/src/test/data/group_with_dims/.zgroup delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var1D/.zarray delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var1D/0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var1D/1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var1D/2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var1D/3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var2D/.zarray delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var2D/0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var2D/0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var2D/0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var2D/0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var2D/1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var2D/1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var2D/1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var2D/1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var2D/2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var2D/2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var2D/2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var2D/2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var2D/3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var2D/3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var2D/3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var2D/3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/.zarray delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/0.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/0.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/0.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/0.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/0.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/0.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/0.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/0.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/0.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/0.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/0.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/0.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/0.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/0.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/0.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/0.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/1.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/1.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/1.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/1.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/1.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/1.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/1.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/1.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/1.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/1.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/1.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/1.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/1.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/1.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/1.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/1.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/2.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/2.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/2.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/2.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/2.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/2.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/2.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/2.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/2.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/2.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/2.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/2.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/2.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/2.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/2.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/2.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/3.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/3.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/3.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/3.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/3.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/3.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/3.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/3.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/3.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/3.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/3.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/3.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/3.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/3.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/3.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var3D/3.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/.zarray delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.0.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.0.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.0.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.0.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.0.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.0.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.0.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.0.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.0.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.0.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.0.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.0.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.0.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.0.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.0.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.0.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.1.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.1.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.1.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.1.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.1.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.1.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.1.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.1.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.1.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.1.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.1.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.1.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.1.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.1.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.1.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.1.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.2.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.2.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.2.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.2.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.2.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.2.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.2.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.2.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.2.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.2.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.2.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.2.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.2.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.2.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.2.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.2.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.3.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.3.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.3.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.3.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.3.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.3.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.3.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.3.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.3.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.3.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.3.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.3.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.3.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.3.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.3.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/0.3.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.0.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.0.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.0.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.0.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.0.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.0.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.0.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.0.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.0.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.0.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.0.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.0.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.0.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.0.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.0.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.0.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.1.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.1.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.1.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.1.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.1.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.1.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.1.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.1.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.1.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.1.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.1.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.1.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.1.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.1.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.1.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.1.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.2.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.2.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.2.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.2.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.2.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.2.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.2.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.2.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.2.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.2.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.2.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.2.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.2.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.2.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.2.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.2.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.3.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.3.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.3.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.3.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.3.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.3.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.3.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.3.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.3.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.3.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.3.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.3.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.3.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.3.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.3.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/1.3.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.0.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.0.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.0.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.0.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.0.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.0.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.0.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.0.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.0.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.0.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.0.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.0.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.0.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.0.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.0.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.0.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.1.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.1.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.1.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.1.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.1.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.1.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.1.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.1.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.1.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.1.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.1.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.1.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.1.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.1.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.1.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.1.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.2.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.2.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.2.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.2.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.2.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.2.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.2.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.2.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.2.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.2.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.2.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.2.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.2.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.2.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.2.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.2.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.3.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.3.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.3.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.3.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.3.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.3.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.3.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.3.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.3.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.3.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.3.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.3.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.3.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.3.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.3.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/2.3.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.0.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.0.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.0.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.0.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.0.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.0.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.0.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.0.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.0.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.0.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.0.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.0.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.0.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.0.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.0.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.0.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.1.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.1.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.1.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.1.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.1.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.1.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.1.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.1.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.1.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.1.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.1.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.1.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.1.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.1.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.1.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.1.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.2.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.2.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.2.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.2.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.2.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.2.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.2.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.2.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.2.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.2.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.2.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.2.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.2.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.2.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.2.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.2.3.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.3.0.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.3.0.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.3.0.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.3.0.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.3.1.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.3.1.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.3.1.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.3.1.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.3.2.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.3.2.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.3.2.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.3.2.3 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.3.3.0 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.3.3.1 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.3.3.2 delete mode 100644 cdm/zarr/src/test/data/group_with_dims/var4D/3.3.3.3 diff --git a/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZArray.java b/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZArray.java index 8481313f58..a34c465a89 100644 --- a/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZArray.java +++ b/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZArray.java @@ -178,9 +178,12 @@ public ZArray deserialize(JsonParser p, DeserializationContext ctxt) throws IOEx int[] chunks = StreamSupport.stream(((ArrayNode) root.path(ZarrKeys.CHUNKS)).spliterator(), false) .mapToInt(JsonNode::asInt).toArray(); String dtype = ((JsonNode) root.path(ZarrKeys.DTYPE)).asText(); + JsonNode fillValueNode = (JsonNode) root.path(ZarrKeys.FILL_VALUE); final Object fill; - if (fillValueNode.isLong()) { + if (fillValueNode.isInt()) { + fill = fillValueNode.asInt(); + } else if (fillValueNode.isLong()) { fill = fillValueNode.longValue(); } else if (fillValueNode.isFloat()) { fill = fillValueNode.floatValue(); diff --git a/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java b/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java index e522ef02c2..15bf286267 100644 --- a/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java +++ b/cdm/zarr/src/main/java/ucar/nc2/iosp/zarr/ZarrIosp.java @@ -68,9 +68,29 @@ public Array readData(Variable v2, Section section) { // find variable in RAF ZarrHeader.VInfo vinfo = (ZarrHeader.VInfo) v2.getSPobject(); DataType dataType = v2.getDataType(); - logger.debug("DataType is '{}'", dataType); + Object fillValue = getFillValue(vinfo, dataType); + + // create layout object + Layout layout = new ZarrLayoutBB(v2, section, this.raf); + Object data = IospHelper.readDataFill((LayoutBB) layout, dataType, fillValue); + + Array array = Array.factory(dataType, section.getShape(), data); + if (vinfo.getOrder() == ZArray.Order.F) { + int n = v2.getDimensions().size(); + int[] dims = new int[n]; + for (int i = 0; i < n; i++) { + dims[i] = n - i - 1; + } + array = array.permute(dims); + } + + return array; + } + + private Object getFillValue(ZarrHeader.VInfo vinfo, DataType dataType) { + // Watch for floating point fill values encoded as Strings final Object fillValueObj = vinfo.getFillValue(); @@ -78,11 +98,13 @@ public Array readData(Variable v2, Section section) { if (fillValueObj instanceof String) { final String fillValueStr = (String) fillValueObj; - logger.debug("Fill value is String with value '{}'", fillValueStr); + if (dataType.isString()) { + return fillValueStr; + } - if ("".equals(fillValueStr)) { - fillValue = null; + if (fillValueStr.isEmpty()) { + return null; } else { switch (dataType) { case FLOAT: @@ -115,21 +137,6 @@ public Array readData(Variable v2, Section section) { } } } - - // create layout object - Layout layout = new ZarrLayoutBB(v2, section, this.raf); - Object data = IospHelper.readDataFill((LayoutBB) layout, dataType, fillValue); - - Array array = Array.factory(dataType, section.getShape(), data); - if (vinfo.getOrder() == ZArray.Order.F) { - int n = v2.getDimensions().size(); - int[] dims = new int[n]; - for (int i = 0; i < n; i++) { - dims[i] = n - i - 1; - } - array = array.permute(dims); - } - - return array; + return fillValue; } } diff --git a/cdm/zarr/src/test/data/group_with_attrs/.zgroup b/cdm/zarr/src/test/data/fill_values.zarr/.zgroup similarity index 100% rename from cdm/zarr/src/test/data/group_with_attrs/.zgroup rename to cdm/zarr/src/test/data/fill_values.zarr/.zgroup diff --git a/cdm/zarr/src/test/data/fill_values.zarr/double_inf/.zarray b/cdm/zarr/src/test/data/fill_values.zarr/double_inf/.zarray new file mode 100644 index 0000000000..e1fdfed835 --- /dev/null +++ b/cdm/zarr/src/test/data/fill_values.zarr/double_inf/.zarray @@ -0,0 +1,22 @@ +{ + "chunks": [ + 2, + 1 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": ">f8", + "fill_value": "Infinity", + "filters": null, + "order": "C", + "shape": [ + 4, + 2 + ], + "zarr_format": 2 +} \ No newline at end of file diff --git a/cdm/zarr/src/test/data/fill_values.zarr/double_nan/.zarray b/cdm/zarr/src/test/data/fill_values.zarr/double_nan/.zarray new file mode 100644 index 0000000000..8fb66f0d1a --- /dev/null +++ b/cdm/zarr/src/test/data/fill_values.zarr/double_nan/.zarray @@ -0,0 +1,22 @@ +{ + "chunks": [ + 2, + 1 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": ">f8", + "fill_value": "NaN", + "filters": null, + "order": "C", + "shape": [ + 4, + 2 + ], + "zarr_format": 2 +} \ No newline at end of file diff --git a/cdm/zarr/src/test/data/fill_values.zarr/double_ninf/.zarray b/cdm/zarr/src/test/data/fill_values.zarr/double_ninf/.zarray new file mode 100644 index 0000000000..46e46641d5 --- /dev/null +++ b/cdm/zarr/src/test/data/fill_values.zarr/double_ninf/.zarray @@ -0,0 +1,22 @@ +{ + "chunks": [ + 2, + 1 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": ">f8", + "fill_value": "-Infinity", + "filters": null, + "order": "C", + "shape": [ + 4, + 2 + ], + "zarr_format": 2 +} \ No newline at end of file diff --git a/cdm/zarr/src/test/data/fill_values.zarr/float_inf/.zarray b/cdm/zarr/src/test/data/fill_values.zarr/float_inf/.zarray new file mode 100644 index 0000000000..12a8e7625a --- /dev/null +++ b/cdm/zarr/src/test/data/fill_values.zarr/float_inf/.zarray @@ -0,0 +1,22 @@ +{ + "chunks": [ + 2, + 1 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": ">f4", + "fill_value": "Infinity", + "filters": null, + "order": "C", + "shape": [ + 4, + 2 + ], + "zarr_format": 2 +} \ No newline at end of file diff --git a/cdm/zarr/src/test/data/fill_values.zarr/float_nan/.zarray b/cdm/zarr/src/test/data/fill_values.zarr/float_nan/.zarray new file mode 100644 index 0000000000..50cfd7c085 --- /dev/null +++ b/cdm/zarr/src/test/data/fill_values.zarr/float_nan/.zarray @@ -0,0 +1,22 @@ +{ + "chunks": [ + 2, + 1 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": ">f4", + "fill_value": "NaN", + "filters": null, + "order": "C", + "shape": [ + 4, + 2 + ], + "zarr_format": 2 +} \ No newline at end of file diff --git a/cdm/zarr/src/test/data/fill_values.zarr/float_ninf/.zarray b/cdm/zarr/src/test/data/fill_values.zarr/float_ninf/.zarray new file mode 100644 index 0000000000..63bdb1759a --- /dev/null +++ b/cdm/zarr/src/test/data/fill_values.zarr/float_ninf/.zarray @@ -0,0 +1,22 @@ +{ + "chunks": [ + 2, + 1 + ], + "compressor": { + "blocksize": 0, + "clevel": 5, + "cname": "lz4", + "id": "blosc", + "shuffle": 1 + }, + "dtype": ">f4", + "fill_value": "-Infinity", + "filters": null, + "order": "C", + "shape": [ + 4, + 2 + ], + "zarr_format": 2 +} \ No newline at end of file diff --git a/cdm/zarr/src/test/data/group_with_attrs/.zattrs b/cdm/zarr/src/test/data/group_with_attrs/.zattrs deleted file mode 100644 index cf836337bf..0000000000 --- a/cdm/zarr/src/test/data/group_with_attrs/.zattrs +++ /dev/null @@ -1,3 +0,0 @@ -{ - "group_attr": "foo" -} \ No newline at end of file diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/.zarray b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/.zarray deleted file mode 100644 index 4572daa603..0000000000 --- a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/.zarray +++ /dev/null @@ -1,16 +0,0 @@ -{ - "chunks": [ - 4, - 5 - ], - "compressor": null, - "dtype": "O00dwFDF6Tf diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/0.1 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/0.1 deleted file mode 100644 index 2e5d506d33d2c41448cd49524cb557d6f4da55dc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 fcmZQ&U|?WH!fZe`GG+(zkue95kBm8id}Is&6vzOL diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/0.2 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/0.2 deleted file mode 100644 index 4fc0c650b5532e8db0e68c4d178a899cc543fdcd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 fcmd;LU|`@v!rVYMGUfsDkufijkBs?%d}Is&B`^T- diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/0.3 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/0.3 deleted file mode 100644 index 47880e61ccedaff7521ee760dfa2d0c1c0e0ffb4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 fcmd;QU|`@!!U8}xG8P2#k+BevkBo(Zd}Is&HJAZa diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/1.0 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/1.0 deleted file mode 100644 index 0a190306b1a6cba0e874e673f1ce4d313714d815..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 ccmZQzKn09IE;42U@{ut!kdKU6fP7>O00dwFDF6Tf diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/1.1 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/1.1 deleted file mode 100644 index 2e5d506d33d2c41448cd49524cb557d6f4da55dc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 fcmZQ&U|?WH!fZe`GG+(zkue95kBm8id}Is&6vzOL diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/1.2 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/1.2 deleted file mode 100644 index 4fc0c650b5532e8db0e68c4d178a899cc543fdcd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 fcmd;LU|`@v!rVYMGUfsDkufijkBs?%d}Is&B`^T- diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/1.3 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/1.3 deleted file mode 100644 index 47880e61ccedaff7521ee760dfa2d0c1c0e0ffb4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 fcmd;QU|`@!!U8}xG8P2#k+BevkBo(Zd}Is&HJAZa diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/2.0 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/2.0 deleted file mode 100644 index 0a190306b1a6cba0e874e673f1ce4d313714d815..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 ccmZQzKn09IE;42U@{ut!kdKU6fP7>O00dwFDF6Tf diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/2.1 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/2.1 deleted file mode 100644 index 2e5d506d33d2c41448cd49524cb557d6f4da55dc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 fcmZQ&U|?WH!fZe`GG+(zkue95kBm8id}Is&6vzOL diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/2.2 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/2.2 deleted file mode 100644 index 4fc0c650b5532e8db0e68c4d178a899cc543fdcd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 fcmd;LU|`@v!rVYMGUfsDkufijkBs?%d}Is&B`^T- diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/2.3 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/2.3 deleted file mode 100644 index 47880e61ccedaff7521ee760dfa2d0c1c0e0ffb4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 fcmd;QU|`@!!U8}xG8P2#k+BevkBo(Zd}Is&HJAZa diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/3.0 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/3.0 deleted file mode 100644 index 0a190306b1a6cba0e874e673f1ce4d313714d815..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 ccmZQzKn09IE;42U@{ut!kdKU6fP7>O00dwFDF6Tf diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/3.1 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/3.1 deleted file mode 100644 index 2e5d506d33d2c41448cd49524cb557d6f4da55dc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 fcmZQ&U|?WH!fZe`GG+(zkue95kBm8id}Is&6vzOL diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/3.2 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/3.2 deleted file mode 100644 index 4fc0c650b5532e8db0e68c4d178a899cc543fdcd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 fcmd;LU|`@v!rVYMGUfsDkufijkBs?%d}Is&B`^T- diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/3.3 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/3.3 deleted file mode 100644 index 47880e61ccedaff7521ee760dfa2d0c1c0e0ffb4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 fcmd;QU|`@!!U8}xG8P2#k+BevkBo(Zd}Is&HJAZa diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/4.0 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/4.0 deleted file mode 100644 index 0a190306b1a6cba0e874e673f1ce4d313714d815..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 ccmZQzKn09IE;42U@{ut!kdKU6fP7>O00dwFDF6Tf diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/4.1 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/4.1 deleted file mode 100644 index 2e5d506d33d2c41448cd49524cb557d6f4da55dc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 fcmZQ&U|?WH!fZe`GG+(zkue95kBm8id}Is&6vzOL diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/4.2 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/4.2 deleted file mode 100644 index 4fc0c650b5532e8db0e68c4d178a899cc543fdcd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 fcmd;LU|`@v!rVYMGUfsDkufijkBs?%d}Is&B`^T- diff --git a/cdm/zarr/src/test/data/group_with_attrs/F_order_array/4.3 b/cdm/zarr/src/test/data/group_with_attrs/F_order_array/4.3 deleted file mode 100644 index 47880e61ccedaff7521ee760dfa2d0c1c0e0ffb4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 80 fcmd;QU|`@!!U8}xG8P2#k+BevkBo(Zd}Is&HJAZa diff --git a/cdm/zarr/src/test/data/group_with_attrs/nested/.zarray b/cdm/zarr/src/test/data/group_with_attrs/nested/.zarray deleted file mode 100644 index 337cc66c01..0000000000 --- a/cdm/zarr/src/test/data/group_with_attrs/nested/.zarray +++ /dev/null @@ -1,17 +0,0 @@ -{ - "chunks": [ - 10, - 10 - ], - "compressor": null, - "dtype": " stores; @BeforeClass @@ -273,4 +277,23 @@ private void _testNested(String location) throws IOException { assertThat(data.get1DJavaArray(DataType.SHORT)).isEqualTo(expected); } + @Test + public void testFillValues() throws IOException { + NetcdfFile ncfile = NetcdfFiles.open(FILL_VALUES_DATA); + + Array float_nan = ncfile.findVariable("float_nan").read(); + assertThat(float_nan.getFloat(0)).isEqualTo(Float.NaN); + Array float_inf = ncfile.findVariable("float_inf").read(); + assertThat(float_inf.getFloat(0)).isEqualTo(Float.POSITIVE_INFINITY); + Array float_ninf = ncfile.findVariable("float_ninf").read(); + assertThat(float_ninf.getFloat(0)).isEqualTo(Float.NEGATIVE_INFINITY); + + Array double_nan = ncfile.findVariable("float_nan").read(); + assertThat(double_nan.getDouble(0)).isEqualTo(Double.NaN); + Array double_inf = ncfile.findVariable("float_inf").read(); + assertThat(double_inf.getDouble(0)).isEqualTo(Double.POSITIVE_INFINITY); + Array double_ninf = ncfile.findVariable("float_ninf").read(); + assertThat(double_ninf.getDouble(0)).isEqualTo(Double.NEGATIVE_INFINITY); + } + } From 71133ae2b736bc3eb24d2d431e1ed481401d24aa Mon Sep 17 00:00:00 2001 From: Megan Lerman Date: Mon, 24 Oct 2022 17:27:10 -0700 Subject: [PATCH 14/70] Add test for opening a creating a new CalendarDateUnit using a string with extra whitespace in the time unit (test currently fails) --- .../src/test/java/ucar/nc2/time/TestCalendars.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java b/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java index 707e42b88a..8906b8eb48 100644 --- a/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java +++ b/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java @@ -148,6 +148,16 @@ public void testZeroYear() { assert unit2.getCalendar() == Calendar.proleptic_gregorian; } + @Test + public void TestExtraWhitespaceInUnit(){ + //test time unit with extra space between date and time + for (Calendar cal : Calendar.values()) { + CalendarDateUnit unit = CalendarDateUnit.withCalendar(cal, "calendar years since 2022-01-01 12:34:00"); + + //need to add assert + } + } + } From 61810117a660d048f3f97eb98414f0be6e6313b5 Mon Sep 17 00:00:00 2001 From: Megan Lerman Date: Tue, 25 Oct 2022 12:12:37 -0700 Subject: [PATCH 15/70] Refactor test to test with all possible whitespace characters compatible with udunits (test currently throws an exception) --- .../test/java/ucar/nc2/time/TestCalendars.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java b/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java index 8906b8eb48..da6d8bb810 100644 --- a/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java +++ b/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java @@ -149,15 +149,19 @@ public void testZeroYear() { } @Test - public void TestExtraWhitespaceInUnit(){ - //test time unit with extra space between date and time + public void testWhiteSpaceCompatibilityWithUdunits(){ + String date = "2022-1-1"; + String time = "12:34:00Z"; + String[] udunitsWhitespace = new String[]{" ", " ", "\t", "\n", "\r", "\f"}; for (Calendar cal : Calendar.values()) { - CalendarDateUnit unit = CalendarDateUnit.withCalendar(cal, "calendar years since 2022-01-01 12:34:00"); + CalendarDateUnit cleanUnit = CalendarDateUnit.withCalendar(cal, String.format("secs since %sT%s", date, time)); - //need to add assert + for(String whitespace : udunitsWhitespace){ + CalendarDateUnit testUnit = CalendarDateUnit.withCalendar(cal, String.format("secs since %s%s%s", date, whitespace, time)); + assert(cleanUnit.getBaseDate().equals(testUnit.getBaseDate())); + } } - } - + } } From ec56d1e91c3e6922b7c13d8c3ab6634c9da108ef Mon Sep 17 00:00:00 2001 From: Megan Lerman Date: Tue, 25 Oct 2022 12:13:50 -0700 Subject: [PATCH 16/70] uncomment line that replaces all whitespace in the string with a single space --- cdm/core/src/main/java/ucar/nc2/time/CalendarDateUnit.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdm/core/src/main/java/ucar/nc2/time/CalendarDateUnit.java b/cdm/core/src/main/java/ucar/nc2/time/CalendarDateUnit.java index d77f9951d8..099d106bb9 100644 --- a/cdm/core/src/main/java/ucar/nc2/time/CalendarDateUnit.java +++ b/cdm/core/src/main/java/ucar/nc2/time/CalendarDateUnit.java @@ -111,7 +111,7 @@ public static CalendarDateUnit of(Calendar calt, CalendarPeriod.Field periodFiel private CalendarDateUnit(Calendar calt, String dateUnitString) { dateUnitString = dateUnitString.trim(); - // dateUnitString = dateUnitString.replaceAll("\\s+", " "); LOOK think about should we allow this ?? + dateUnitString = dateUnitString.replaceAll("\\s+", " "); dateUnitString = dateUnitString.toLowerCase(); isCalendarField = dateUnitString.startsWith(byCalendarString); From cb5791f3993cde62a634f661e29a2a8eff0b35ec Mon Sep 17 00:00:00 2001 From: Megan Lerman Date: Tue, 25 Oct 2022 13:47:29 -0700 Subject: [PATCH 17/70] rename variable and run spotless --- .../java/ucar/nc2/time/TestCalendars.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java b/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java index da6d8bb810..55ee99744d 100644 --- a/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java +++ b/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java @@ -149,17 +149,20 @@ public void testZeroYear() { } @Test - public void testWhiteSpaceCompatibilityWithUdunits(){ + public void testWhiteSpaceCompatibilityWithUdunits() { String date = "2022-1-1"; String time = "12:34:00Z"; - String[] udunitsWhitespace = new String[]{" ", " ", "\t", "\n", "\r", "\f"}; + String[] udunitsWhitespace = new String[] {" ", " ", "\t", "\n", "\r", "\f"}; for (Calendar cal : Calendar.values()) { - CalendarDateUnit cleanUnit = CalendarDateUnit.withCalendar(cal, String.format("secs since %sT%s", date, time)); - - for(String whitespace : udunitsWhitespace){ - CalendarDateUnit testUnit = CalendarDateUnit.withCalendar(cal, String.format("secs since %s%s%s", date, whitespace, time)); - assert(cleanUnit.getBaseDate().equals(testUnit.getBaseDate())); - } + // unit with time in ISO format to use for testing + CalendarDateUnit isoFormatUnit = + CalendarDateUnit.withCalendar(cal, String.format("secs since %sT%s", date, time)); + + for (String whitespace : udunitsWhitespace) { + CalendarDateUnit testUnit = + CalendarDateUnit.withCalendar(cal, String.format("secs since %s%s%s", date, whitespace, time)); + assert (isoFormatUnit.getBaseDate().equals(testUnit.getBaseDate())); + } } } From e4138dcbf23a7c18a1e7ab914cf4374ce08ca3f4 Mon Sep 17 00:00:00 2001 From: Megan Lerman Date: Tue, 25 Oct 2022 13:47:29 -0700 Subject: [PATCH 18/70] rename variable and run spotless --- .../java/ucar/nc2/time/TestCalendars.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java b/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java index da6d8bb810..55ee99744d 100644 --- a/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java +++ b/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java @@ -149,17 +149,20 @@ public void testZeroYear() { } @Test - public void testWhiteSpaceCompatibilityWithUdunits(){ + public void testWhiteSpaceCompatibilityWithUdunits() { String date = "2022-1-1"; String time = "12:34:00Z"; - String[] udunitsWhitespace = new String[]{" ", " ", "\t", "\n", "\r", "\f"}; + String[] udunitsWhitespace = new String[] {" ", " ", "\t", "\n", "\r", "\f"}; for (Calendar cal : Calendar.values()) { - CalendarDateUnit cleanUnit = CalendarDateUnit.withCalendar(cal, String.format("secs since %sT%s", date, time)); - - for(String whitespace : udunitsWhitespace){ - CalendarDateUnit testUnit = CalendarDateUnit.withCalendar(cal, String.format("secs since %s%s%s", date, whitespace, time)); - assert(cleanUnit.getBaseDate().equals(testUnit.getBaseDate())); - } + // unit with time in ISO format to use for testing + CalendarDateUnit isoFormatUnit = + CalendarDateUnit.withCalendar(cal, String.format("secs since %sT%s", date, time)); + + for (String whitespace : udunitsWhitespace) { + CalendarDateUnit testUnit = + CalendarDateUnit.withCalendar(cal, String.format("secs since %s%s%s", date, whitespace, time)); + assert (isoFormatUnit.getBaseDate().equals(testUnit.getBaseDate())); + } } } From a35b2289e55a78d24039cf8a6a3b941e071c3666 Mon Sep 17 00:00:00 2001 From: Megan Lerman Date: Tue, 25 Oct 2022 14:14:29 -0700 Subject: [PATCH 19/70] rename variable and run spotless --- cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java b/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java index 55ee99744d..eb684aa585 100644 --- a/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java +++ b/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java @@ -149,7 +149,7 @@ public void testZeroYear() { } @Test - public void testWhiteSpaceCompatibilityWithUdunits() { + public void testWhitespaceCompatibilityWithUdunits() { String date = "2022-1-1"; String time = "12:34:00Z"; String[] udunitsWhitespace = new String[] {" ", " ", "\t", "\n", "\r", "\f"}; From 6e9c2167a786bccf84600fd9c1c70db799f03db0 Mon Sep 17 00:00:00 2001 From: Megan Lerman Date: Tue, 25 Oct 2022 14:17:12 -0700 Subject: [PATCH 20/70] correct spelling of whitespace --- cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java b/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java index 55ee99744d..eb684aa585 100644 --- a/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java +++ b/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java @@ -149,7 +149,7 @@ public void testZeroYear() { } @Test - public void testWhiteSpaceCompatibilityWithUdunits() { + public void testWhitespaceCompatibilityWithUdunits() { String date = "2022-1-1"; String time = "12:34:00Z"; String[] udunitsWhitespace = new String[] {" ", " ", "\t", "\n", "\r", "\f"}; From 63e8da1537a67edf264942dfd9d74da608d73f6d Mon Sep 17 00:00:00 2001 From: Megan Lerman Date: Tue, 25 Oct 2022 17:23:49 -0700 Subject: [PATCH 21/70] use google truth assert instead of built in assert --- cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java b/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java index eb684aa585..a5f14ccd8d 100644 --- a/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java +++ b/cdm/core/src/test/java/ucar/nc2/time/TestCalendars.java @@ -37,6 +37,8 @@ import org.slf4j.LoggerFactory; import java.lang.invoke.MethodHandles; +import static com.google.common.truth.Truth.assertThat; + /** * Test on non-standard Calendars * @@ -161,7 +163,7 @@ public void testWhitespaceCompatibilityWithUdunits() { for (String whitespace : udunitsWhitespace) { CalendarDateUnit testUnit = CalendarDateUnit.withCalendar(cal, String.format("secs since %s%s%s", date, whitespace, time)); - assert (isoFormatUnit.getBaseDate().equals(testUnit.getBaseDate())); + assertThat(isoFormatUnit.getBaseDate()).isEqualTo(testUnit.getBaseDate()); } } From e178fc6f8a894a2c7a92ee004e1fbea1f4d10eeb Mon Sep 17 00:00:00 2001 From: Benjamin Root Date: Fri, 28 Oct 2022 23:09:42 -0400 Subject: [PATCH 22/70] Fix some misuses of substring(), with some unittests --- .../thredds/inventory/CollectionGlob.java | 2 +- .../thredds/inventory/TestCollectionGlob.java | 48 +++++++++++++++++++ .../java/ucar/unidata/io/s3/CdmS3Client.java | 5 +- 3 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 cdm/core/src/test/java/thredds/inventory/TestCollectionGlob.java diff --git a/cdm/core/src/main/java/thredds/inventory/CollectionGlob.java b/cdm/core/src/main/java/thredds/inventory/CollectionGlob.java index ff2147f59b..9766da8e31 100644 --- a/cdm/core/src/main/java/thredds/inventory/CollectionGlob.java +++ b/cdm/core/src/main/java/thredds/inventory/CollectionGlob.java @@ -58,7 +58,7 @@ public CollectionGlob(String collectionName, String glob, Logger logger) { // lets suppose the first "*" indicates the top dir int pos = glob.indexOf("*"); - this.root = glob.substring(0, pos - 1); + this.root = glob.substring(0, pos); String match = glob.substring(pos); // count how far to recurse. LAME!!! why doesnt java provide the right thing !!!! diff --git a/cdm/core/src/test/java/thredds/inventory/TestCollectionGlob.java b/cdm/core/src/test/java/thredds/inventory/TestCollectionGlob.java new file mode 100644 index 0000000000..ccecce81e6 --- /dev/null +++ b/cdm/core/src/test/java/thredds/inventory/TestCollectionGlob.java @@ -0,0 +1,48 @@ +package thredds.inventory; + +import static com.google.common.truth.Truth.assertThat; + +import java.util.Arrays; +import java.util.List; +import java.lang.invoke.MethodHandles; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@RunWith(Parameterized.class) +public class TestCollectionGlob { + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @Parameterized.Parameters(name = "{0}") + public static List getTestParameters() { + return Arrays.asList(new Object[][] { + + {"foo*.nc", "foo", 0}, + {"foo/*.nc", "foo/", 0}, + {"foo/*/bar.nc", "foo/", 1}, + {"**/bar.nc", "", 1}, + {"foo*/bar/baz.nc", "foo", 2}, + {"foo*/bar/**/baz.nc", "foo", Integer.MAX_VALUE}, + }); + } + + private final String spec; + private final String root; + private final int depth; + + public TestCollectionGlob(String spec, String root, int depth) { + this.spec = spec; + this.root = root; + this.depth = depth; + } + + @Test + public void nominalGlobbing() { + CollectionGlob globber; + globber = new CollectionGlob("fooName", spec, logger); + assertThat(globber.root).isEqualTo(root); + assertThat(globber.depth).isEqualTo(depth); + } +} diff --git a/cdm/s3/src/main/java/ucar/unidata/io/s3/CdmS3Client.java b/cdm/s3/src/main/java/ucar/unidata/io/s3/CdmS3Client.java index 1c11c9bf65..25ba9c897c 100644 --- a/cdm/s3/src/main/java/ucar/unidata/io/s3/CdmS3Client.java +++ b/cdm/s3/src/main/java/ucar/unidata/io/s3/CdmS3Client.java @@ -114,6 +114,9 @@ public static S3Client acquire(String uri) throws IOException { */ public static S3Client acquire(CdmS3Uri uri) throws IOException { CdmS3Uri key = makeKey(uri); + if (key == null) { + throw new IOException("Failed to make cache key from the given URI"); + } S3Client s3Client; try { s3Client = useCache ? s3ClientCache.get(key) : createS3Client(uri); @@ -137,7 +140,7 @@ private static CdmS3Uri makeKey(CdmS3Uri cdmS3Uri) { int chop = bucketUriString.lastIndexOf("?" + cdmS3Uri.getKey().get()); // remove everything after the ?key portion of the URI (retains authority and path (bucket) of URI) if (chop > 0) { - bucketUriString = bucketUriString.substring(0, chop - 1); + bucketUriString = bucketUriString.substring(0, chop); } } From 8f1427f66a626a4db937d72452a6d698b406d1dc Mon Sep 17 00:00:00 2001 From: Benjamin Root Date: Mon, 31 Oct 2022 11:49:02 -0400 Subject: [PATCH 23/70] Style fix to pass spotless --- .../src/test/java/thredds/inventory/TestCollectionGlob.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cdm/core/src/test/java/thredds/inventory/TestCollectionGlob.java b/cdm/core/src/test/java/thredds/inventory/TestCollectionGlob.java index ccecce81e6..4f618cfef1 100644 --- a/cdm/core/src/test/java/thredds/inventory/TestCollectionGlob.java +++ b/cdm/core/src/test/java/thredds/inventory/TestCollectionGlob.java @@ -20,11 +20,17 @@ public static List getTestParameters() { return Arrays.asList(new Object[][] { {"foo*.nc", "foo", 0}, + {"foo/*.nc", "foo/", 0}, + {"foo/*/bar.nc", "foo/", 1}, + {"**/bar.nc", "", 1}, + {"foo*/bar/baz.nc", "foo", 2}, + {"foo*/bar/**/baz.nc", "foo", Integer.MAX_VALUE}, + }); } From 8f26d853ca2ec53cdae592ed6b911b8a37fcd259 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 31 Oct 2022 10:31:18 -0600 Subject: [PATCH 24/70] Remove override of default function so that NetcdfFiles default_buffersize is not used for s3 --- .../src/main/java/ucar/unidata/io/s3/S3RandomAccessFile.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cdm/s3/src/main/java/ucar/unidata/io/s3/S3RandomAccessFile.java b/cdm/s3/src/main/java/ucar/unidata/io/s3/S3RandomAccessFile.java index ca4c2504d5..eace85da7b 100644 --- a/cdm/s3/src/main/java/ucar/unidata/io/s3/S3RandomAccessFile.java +++ b/cdm/s3/src/main/java/ucar/unidata/io/s3/S3RandomAccessFile.java @@ -163,10 +163,5 @@ public boolean isOwnerOf(String location) { public RandomAccessFile open(String location) throws IOException { return new S3RandomAccessFile(location); } - - @Override - public RandomAccessFile open(String location, int bufferSize) throws IOException { - return new S3RandomAccessFile(location, bufferSize); - } } } From c10fcbb817a690a8f8cb454d25326cb6015add22 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 31 Oct 2022 14:44:36 -0600 Subject: [PATCH 25/70] Remove commented out code --- .../nc2/grib/grib2/table/Grib2Tables.java | 109 ------------------ 1 file changed, 109 deletions(-) diff --git a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java index 40ec11e249..46f416b75c 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java +++ b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java @@ -489,67 +489,6 @@ public CalendarDate getForecastDate(Grib2Record gr) { } } - - /* - * Get the time interval in units of gr.getPDS().getTimeUnit() - * - * @param gr Grib record, must have pds that is a time interval. - * - * @return time interval in units of gr.getPDS().getTimeUnit() - * - * @Nullable - * public TimeCoordIntvDateValue getForecastTimeInterval(Grib2Record gr) { - * // note from Arthur Taylor (degrib): - * /* - * If there was a range I used: - * - * End of interval (EI) = (bytes 36-42 show an "end of overall time interval") - * C1) End of Interval = EI; - * Begin of Interval = EI - range - * - * and if there was no interval then I used: - * C2) End of Interval = Begin of Interval = Ref + ForeT. - * - * if (!gr.getPDS().isTimeInterval()) - * return null; - * Grib2Pds.PdsInterval pdsIntv = (Grib2Pds.PdsInterval) gr.getPDS(); - * int timeUnitOrg = gr.getPDS().getTimeUnit(); - * - * // calculate total "range" - * int range = 0; - * for (Grib2Pds.TimeInterval ti : pdsIntv.getTimeIntervals()) { - * if (ti.timeRangeUnit == 255) - * continue; - * if ((ti.timeRangeUnit != timeUnitOrg) - * || (ti.timeIncrementUnit != timeUnitOrg && ti.timeIncrementUnit != 255 && ti.timeIncrement != 0)) { - * if (!timeUnitWarnWasSent) { - * logger.warn( - * "TimeInterval has different units timeUnit org=" + timeUnitOrg + " TimeInterval=" + ti.timeIncrementUnit); - * timeUnitWarnWasSent = true; - * // throw new RuntimeException("TimeInterval(2) has different units"); - * } - * } - * - * range += ti.timeRangeLength; - * if (ti.timeIncrementUnit != 255) - * range += ti.timeIncrement; - * } - * - * CalendarPeriod unitPeriod = Grib2Utils.getCalendarPeriod(convertTimeUnit(timeUnitOrg)); - * if (unitPeriod == null) - * return null; - * CalendarPeriod period = unitPeriod.multiply(range); - * - * // End of Interval as date - * CalendarDate EI = pdsIntv.getIntervalTimeEnd(); - * if (EI == CalendarDate.UNKNOWN) { // all values were set to zero LOOK guessing! - * return new TimeCoordIntvDateValue(gr.getReferenceDate(), period); - * } else { - * return new TimeCoordIntvDateValue(period, EI); - * } - * } - */ - public TimeCoordIntvDateValue getForecastTimeInterval(Grib2Record gr) { if (!gr.getPDS().isTimeInterval()) return null; @@ -655,54 +594,6 @@ private TimeIntervalAndUnits getForecastTimeInterval(Grib2Pds.PdsInterval pdsInt return new TimeIntervalAndUnits(timeUnitIntv, range); } - /* - * Get interval size in units of hours. - * Only use in GribVariable to decide on variable identity when intvMerge = false. - * - * @param pds must be a Grib2Pds.PdsInterval - * - * @return interval size in units of hours - * - * public double getForecastTimeIntervalSizeInHours(Grib2Pds pds) { - * Grib2Pds.PdsInterval pdsIntv = (Grib2Pds.PdsInterval) pds; - * int timeUnitOrg = pds.getTimeUnit(); - * - * // calculate total "range" in units of timeUnit - * int range = 0; - * for (Grib2Pds.TimeInterval ti : pdsIntv.getTimeIntervals()) { - * if (ti.timeRangeUnit == 255) - * continue; - * if ((ti.timeRangeUnit != timeUnitOrg) - * || (ti.timeIncrementUnit != timeUnitOrg && ti.timeIncrementUnit != 255 && ti.timeIncrement != 0)) { - * logger.warn("TimeInterval(2) has different units timeUnit org=" + timeUnitOrg + " TimeInterval=" - * + ti.timeIncrementUnit); - * throw new RuntimeException("TimeInterval(2) has different units"); - * } - * - * range += ti.timeRangeLength; - * if (ti.timeIncrementUnit != 255) - * range += ti.timeIncrement; - * } - * - * // now convert that range to units of the requested period. - * CalendarPeriod timeUnitPeriod = Grib2Utils.getCalendarPeriod(convertTimeUnit(timeUnitOrg)); - * if (timeUnitPeriod == null) - * return GribNumbers.UNDEFINEDD; - * if (timeUnitPeriod.equals(CalendarPeriod.Hour)) - * return range; - * - * double fac; - * if (timeUnitPeriod.getField() == CalendarPeriod.Field.Month) { - * fac = 30.0 * 24.0; // nominal hours in a month - * } else if (timeUnitPeriod.getField() == CalendarPeriod.Field.Year) { - * fac = 365.0 * 24.0; // nominal hours in a year - * } else { - * fac = CalendarPeriod.Hour.getConvertFactor(timeUnitPeriod); - * } - * return fac * range; - * } - */ - /** * If this has a time interval coordinate, get time interval * From b95087146577b41b637f58e8eb18c94cce410ced Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 31 Oct 2022 14:48:21 -0600 Subject: [PATCH 26/70] Remove unused member --- grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java | 1 - 1 file changed, 1 deletion(-) diff --git a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java index 46f416b75c..b7a13531b9 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java +++ b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java @@ -139,7 +139,6 @@ public static ImmutableList getAllRegisteredTables() { /////////////////////////////////////////////////////////////// protected final Grib2TableConfig config; - private boolean timeUnitWarnWasSent; protected Grib2Tables(Grib2TableConfig config) { this.config = config; From fc610dff7164c127168bc52407a03330ff45f2d1 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 2 Nov 2022 15:26:23 -0600 Subject: [PATCH 27/70] Fix typos --- .../main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java index b7a13531b9..b03f4071a7 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java +++ b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java @@ -30,7 +30,7 @@ /** * Grib 2 Tables - allows local overrides and augmentation of WMO tables. * This class serves the standard WMO tables, local tables are subclasses that override. - * Methods are placed here because they may be overrided by local Tables. + * Methods are placed here because they may be overridden by local Tables. * * Tables include code, flag and parameter tables. * @@ -169,7 +169,7 @@ public String getVariableName(Grib2Record gr) { } /** - * Make a IOSP Variable name, using the Parameter name is available, otherwise a synthezized name. + * Make a IOSP Variable name, using the Parameter name is available, otherwise a synthesized name. */ public String getVariableName(int discipline, int category, int parameter) { String s = WmoParamTable.getParameterName(discipline, category, parameter); @@ -577,7 +577,7 @@ private TimeIntervalAndUnits getForecastTimeInterval(Grib2Pds.PdsInterval pdsInt } if (timeUnitIntv < 0) { timeUnitIntv = ti.timeRangeUnit; - } else if ((ti.timeRangeUnit != timeUnitIntv) // make sure it doesnt change + } else if ((ti.timeRangeUnit != timeUnitIntv) // make sure it doesn't change || (ti.timeIncrementUnit != timeUnitIntv && ti.timeIncrementUnit != 255 && ti.timeIncrement != 0)) { // LOOK // WTF? logger.warn("TimeInterval(2) has different units timeUnit first=" + timeUnitIntv + " TimeInterval=" From 368469e5097f0b4682f823894cb345ace8c6952a Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 2 Nov 2022 15:30:19 -0600 Subject: [PATCH 28/70] Add brackets to one line if statements --- .../nc2/grib/grib2/table/Grib2Tables.java | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java index b03f4071a7..f4de3f8ee1 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java +++ b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java @@ -54,8 +54,9 @@ public static Grib2Tables factory(Grib2Record gr) { public static Grib2Tables factory(int center, int subCenter, int masterVersion, int localVersion, int genProcessId) { Grib2TablesId id = new Grib2TablesId(center, subCenter, masterVersion, localVersion, genProcessId); Grib2Tables cust = tables.get(id); - if (cust != null) + if (cust != null) { return cust; + } // note that we match on id, so same Grib2Customizer may be mapped to multiple id's (eg match on -1) Grib2TableConfig config = Grib2TableConfig.matchTable(id); @@ -173,8 +174,9 @@ public String getVariableName(Grib2Record gr) { */ public String getVariableName(int discipline, int category, int parameter) { String s = WmoParamTable.getParameterName(discipline, category, parameter); - if (s == null) + if (s == null) { s = "U" + discipline + "-" + category + "-" + parameter; + } return s; } @@ -235,8 +237,9 @@ public String getCategory(int discipline, int category) { public String getStatisticName(int id) { String result = getCodeTableValue("4.10", id); // WMO - if (result == null) + if (result == null) { result = getStatisticNameShort(id); + } return result; } @@ -460,15 +463,17 @@ public String getLevelNameShort(int id) { private TimeUnitConverter timeUnitConverter; // LOOK not really immutable public void setTimeUnitConverter(TimeUnitConverter timeUnitConverter) { - if (this.timeUnitConverter != null) + if (this.timeUnitConverter != null) { throw new RuntimeException("Cant modify timeUnitConverter once its been set"); + } this.timeUnitConverter = timeUnitConverter; } @Override public int convertTimeUnit(int timeUnit) { - if (timeUnitConverter == null) + if (timeUnitConverter == null) { return timeUnit; + } return timeUnitConverter.convertTimeUnit(timeUnit); } @@ -489,8 +494,9 @@ public CalendarDate getForecastDate(Grib2Record gr) { } public TimeCoordIntvDateValue getForecastTimeInterval(Grib2Record gr) { - if (!gr.getPDS().isTimeInterval()) + if (!gr.getPDS().isTimeInterval()) { return null; + } Grib2Pds.PdsInterval pdsIntv = (Grib2Pds.PdsInterval) gr.getPDS(); // the time "range" in units of pdsIntv timeUnits @@ -541,10 +547,12 @@ public double getForecastTimeIntervalSizeInHours(Grib2Pds pds) { // convert that range to units of hours. CalendarPeriod timeUnitPeriod = Grib2Utils.getCalendarPeriod(convertTimeUnit(intvu.timeUnitIntv)); - if (timeUnitPeriod == null) + if (timeUnitPeriod == null) { return GribNumbers.UNDEFINEDD; - if (timeUnitPeriod.equals(CalendarPeriod.Hour)) + } + if (timeUnitPeriod.equals(CalendarPeriod.Hour)) { return intvu.timeRange; + } // LOOK leave this until we fix getConvertFactor() double fac; @@ -608,8 +616,9 @@ public int[] getForecastTimeIntervalOffset(Grib2Record gr) { Grib2Pds pds = gr.getPDS(); int unit = convertTimeUnit(pds.getTimeUnit()); TimeCoordIntvValue tinv = tinvd.convertReferenceDate(gr.getReferenceDate(), Grib2Utils.getCalendarPeriod(unit)); - if (tinv == null) + if (tinv == null) { return null; + } int[] result = new int[2]; result[0] = tinv.getBounds1(); result[1] = tinv.getBounds2(); From da01cca4ca9b4265e6d7fdf6f312d9cf7d0589d3 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 2 Nov 2022 12:07:10 -0600 Subject: [PATCH 29/70] Leave file path out of error message --- .../main/java/ucar/nc2/ft2/coverage/CoverageDatasetFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdm/core/src/main/java/ucar/nc2/ft2/coverage/CoverageDatasetFactory.java b/cdm/core/src/main/java/ucar/nc2/ft2/coverage/CoverageDatasetFactory.java index 8e68104abc..a3ab1aba73 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft2/coverage/CoverageDatasetFactory.java +++ b/cdm/core/src/main/java/ucar/nc2/ft2/coverage/CoverageDatasetFactory.java @@ -79,7 +79,7 @@ public static Optional openCoverageDataset(String endpoi return Optional.empty(errlog.toString()); } - return Optional.empty("Could not open as Coverage: " + endpoint); + return Optional.empty("Could not open as a coverage dataset"); } /** From fd7c8ccc03e7c122144f27b2f6761a8baa669d51 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 10 Oct 2022 11:26:01 -0600 Subject: [PATCH 30/70] Add tests for rounding of time interval offsets --- .../ucar/nc2/grib/coord/TestTimeCoord.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/grib/src/test/java/ucar/nc2/grib/coord/TestTimeCoord.java b/grib/src/test/java/ucar/nc2/grib/coord/TestTimeCoord.java index 112ba1cf3a..53860dcdd7 100644 --- a/grib/src/test/java/ucar/nc2/grib/coord/TestTimeCoord.java +++ b/grib/src/test/java/ucar/nc2/grib/coord/TestTimeCoord.java @@ -1,5 +1,6 @@ package ucar.nc2.grib.coord; +import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertEquals; import org.junit.Test; import org.junit.runner.RunWith; @@ -26,5 +27,34 @@ public void testTinvDate() { assertEquals("2010-03-29T00:00:00Z", refDate.toString()); } + @Test + public void shouldPreserveTimeIntervalLengthWithStartAfterRefDate() { + final CalendarDate start = CalendarDate.parseISOformat(null, "2022-08-16T01:00:00Z"); + final CalendarDate end = CalendarDate.parseISOformat(null, "2022-08-16T12:00:00Z"); + final TimeCoordIntvDateValue timeCoordIntvDateValue = new TimeCoordIntvDateValue(start, end); + + final CalendarDate refDate = CalendarDate.parseISOformat(null, "2022-08-16T00:30:00Z"); + final CalendarPeriod timeUnit = CalendarPeriod.of("Hour"); + + final TimeCoordIntvValue timeCoordIntvValue = timeCoordIntvDateValue.convertReferenceDate(refDate, timeUnit); + assertThat(timeCoordIntvValue.getBounds1()).isEqualTo(0); + assertThat(timeCoordIntvValue.getBounds2()).isEqualTo(11); + assertThat(timeCoordIntvValue.getIntervalSize()).isEqualTo(11); + } + + @Test + public void shouldPreserveTimeIntervalLengthWithStartBeforeRefDate() { + final CalendarDate start = CalendarDate.parseISOformat(null, "2022-08-16T01:00:00Z"); + final CalendarDate end = CalendarDate.parseISOformat(null, "2022-08-16T12:00:00Z"); + final TimeCoordIntvDateValue timeCoordIntvDateValue = new TimeCoordIntvDateValue(start, end); + + final CalendarDate refDate = CalendarDate.parseISOformat(null, "2022-08-16T01:30:00Z"); + final CalendarPeriod timeUnit = CalendarPeriod.of("Hour"); + + final TimeCoordIntvValue timeCoordIntvValue = timeCoordIntvDateValue.convertReferenceDate(refDate, timeUnit); + assertThat(timeCoordIntvValue.getBounds1()).isEqualTo(-1); + assertThat(timeCoordIntvValue.getBounds2()).isEqualTo(10); + assertThat(timeCoordIntvValue.getIntervalSize()).isEqualTo(11); + } } From 31a7f920e9881b2638aa868735eacee3439c2692 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 10 Oct 2022 11:27:38 -0600 Subject: [PATCH 31/70] Round time offsets using floor instead of truncation to zero (what joda time was doing) --- .../main/java/ucar/nc2/time/CalendarPeriod.java | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/time/CalendarPeriod.java b/cdm/core/src/main/java/ucar/nc2/time/CalendarPeriod.java index 2b5cff9762..e5927e7dcc 100644 --- a/cdm/core/src/main/java/ucar/nc2/time/CalendarPeriod.java +++ b/cdm/core/src/main/java/ucar/nc2/time/CalendarPeriod.java @@ -7,8 +7,6 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import org.joda.time.DurationFieldType; -import org.joda.time.Period; import org.joda.time.PeriodType; import ucar.nc2.units.TimeDuration; import ucar.unidata.util.StringUtil2; @@ -169,7 +167,7 @@ public Field getField() { /** * Subtract two dates, return difference in units of this period. - * If not even, will round down and log a warning + * If not even, will round down (floor) and log a warning * * @param start start date * @param end end date @@ -180,7 +178,7 @@ public int subtract(CalendarDate start, CalendarDate end) { int thislen = millisecs(); if ((diff % thislen != 0)) log.warn("roundoff error"); - return (int) (diff / thislen); + return (int) Math.floor(diff / (float) thislen); } /** @@ -238,20 +236,13 @@ else if (field == CalendarPeriod.Field.Day) // offset from start to end, in these units // start + offset = end + // takes the floor when rounding public int getOffset(CalendarDate start, CalendarDate end) { if (start.equals(end)) { return 0; } - Period p = new Period(start.getMillis(), end.getMillis(), getPeriodType()); - return p.get(getDurationFieldType()); - } - - PeriodType getPeriodType() { - return getField().p; - } - DurationFieldType getDurationFieldType() { - return getField().p.getFieldType(0); + return subtract(start, end); } @Override From c1d4f31ca7fcb26df29bc5bc9775352696bd11c3 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 13 Oct 2022 15:36:38 -0600 Subject: [PATCH 32/70] Remove verbose warning about rounding --- cdm/core/src/main/java/ucar/nc2/time/CalendarPeriod.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/time/CalendarPeriod.java b/cdm/core/src/main/java/ucar/nc2/time/CalendarPeriod.java index e5927e7dcc..c55c797644 100644 --- a/cdm/core/src/main/java/ucar/nc2/time/CalendarPeriod.java +++ b/cdm/core/src/main/java/ucar/nc2/time/CalendarPeriod.java @@ -167,7 +167,7 @@ public Field getField() { /** * Subtract two dates, return difference in units of this period. - * If not even, will round down (floor) and log a warning + * If not even, will round down (take the floor) * * @param start start date * @param end end date @@ -176,8 +176,6 @@ public Field getField() { public int subtract(CalendarDate start, CalendarDate end) { long diff = end.getDifferenceInMsecs(start); int thislen = millisecs(); - if ((diff % thislen != 0)) - log.warn("roundoff error"); return (int) Math.floor(diff / (float) thislen); } From 4f928d6e14118a6acd8b8014e568c6321728ab81 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 13 Oct 2022 13:46:54 -0600 Subject: [PATCH 33/70] Add test for the grib2 forecast intervals with a unit converter --- .../nc2/grib/grib2/table/TestLocalTables.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/grib/src/test/java/ucar/nc2/grib/grib2/table/TestLocalTables.java b/grib/src/test/java/ucar/nc2/grib/grib2/table/TestLocalTables.java index 17cf0b75fa..b585269ab8 100644 --- a/grib/src/test/java/ucar/nc2/grib/grib2/table/TestLocalTables.java +++ b/grib/src/test/java/ucar/nc2/grib/grib2/table/TestLocalTables.java @@ -6,14 +6,31 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.io.IOException; +import java.lang.invoke.MethodHandles; import java.util.Iterator; +import java.util.List; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import thredds.featurecollection.FeatureCollectionConfig.TimeUnitConverterHash; +import thredds.inventory.CollectionUpdateType; +import thredds.inventory.MFile; +import thredds.inventory.MFiles; +import ucar.nc2.grib.GribIndex; import ucar.nc2.grib.GribTables; +import ucar.nc2.grib.coord.TimeCoordIntvDateValue; +import ucar.nc2.grib.grib2.Grib2Index; +import ucar.nc2.grib.grib2.Grib2Record; +import ucar.nc2.time.CalendarDate; +import ucar.unidata.util.test.TestDir; +import ucar.unidata.util.test.category.NeedsCdmUnitTest; @RunWith(JUnit4.class) public class TestLocalTables { + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @Test public void testLocalTables() throws IOException { @@ -57,4 +74,43 @@ public void testEcmwfCodeTables() throws IOException { } } } + + @Test + @Category(NeedsCdmUnitTest.class) + public void shouldUseUnitConverterWhenCreatingForecastInterval() throws IOException { + final MFile file = + MFiles.create(TestDir.cdmUnitTestDir + "/tds/ncep/NDFD_NWS_CONUS_conduit_2p5km_20220816_0030.grib2"); + assertThat(file).isNotNull(); + + final Grib2Index gribIndex = + (Grib2Index) GribIndex.readOrCreateIndexFromSingleFile(false, file, CollectionUpdateType.always, logger); + final List records = gribIndex.getRecords(); + assertThat(records).isNotEmpty(); + final Grib2Record record = records.get(0); + final Grib2Tables tables = Grib2Tables.factory(record); + + final CalendarDate start = CalendarDate.parseISOformat(null, "2022-08-18T00:00:00Z"); + final CalendarDate end = CalendarDate.parseISOformat(null, "2022-08-18T12:00:00Z"); + final TimeCoordIntvDateValue expectedInterval = new TimeCoordIntvDateValue(start, end); + + // default units + final TimeCoordIntvDateValue interval = tables.getForecastTimeInterval(record); + assertThat(interval).isEqualTo(expectedInterval); + + // convert hours to minutes + final TimeUnitConverterHash hoursToMinutesConverter = new TimeUnitConverterHash(); + hoursToMinutesConverter.map.put(1, 0); + tables.setTimeUnitConverter(hoursToMinutesConverter); + + final TimeCoordIntvDateValue intervalHoursToMinutes = tables.getForecastTimeInterval(record); + assertThat(intervalHoursToMinutes).isEqualTo(expectedInterval); + + // convert minutes to hour + final TimeUnitConverterHash minutesToHoursConverter = new TimeUnitConverterHash(); + minutesToHoursConverter.map.put(0, 1); + tables.setTimeUnitConverter(minutesToHoursConverter); + + final TimeCoordIntvDateValue intervalMinutesToHours = tables.getForecastTimeInterval(record); + assertThat(intervalMinutesToHours).isEqualTo(expectedInterval); + } } From c6ef9702cb1e7103542f26f73cf277b4d67da14e Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 10 Oct 2022 14:59:36 -0600 Subject: [PATCH 34/70] Fix calendar have period-- don't convert to user configured time unit --- grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java index 40ec11e249..058d6b9c01 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java +++ b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java @@ -564,7 +564,7 @@ public TimeCoordIntvDateValue getForecastTimeInterval(Grib2Record gr) { if (wantPeriod == null) { return null; } - CalendarPeriod havePeriod = Grib2Utils.getCalendarPeriod(convertTimeUnit(intvu.timeUnitIntv)); + CalendarPeriod havePeriod = Grib2Utils.getCalendarPeriod(intvu.timeUnitIntv); double fac2 = intvu.timeRange * wantPeriod.getConvertFactor(havePeriod); CalendarPeriod period = wantPeriod.multiply((int) fac2); // LOOK needs to be an int From b57ff8fbd26ff3e33056d748ccca87735e37088e Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 13 Oct 2022 15:42:06 -0600 Subject: [PATCH 35/70] Add tests for unit conversion for a grib collection with mixed time units --- .../nc2/grib/TestGribCollectionTimeUnits.java | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionTimeUnits.java diff --git a/cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionTimeUnits.java b/cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionTimeUnits.java new file mode 100644 index 0000000000..028115619f --- /dev/null +++ b/cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionTimeUnits.java @@ -0,0 +1,98 @@ +/* + * Copyright (c) 1998-2022 University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. + */ +package ucar.nc2.grib; + +import static com.google.common.truth.Truth.assertThat; + +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import thredds.featurecollection.FeatureCollectionConfig; +import thredds.featurecollection.FeatureCollectionType; +import thredds.inventory.*; +import ucar.nc2.Variable; +import ucar.nc2.dataset.CoordinateAxis; +import ucar.nc2.dataset.CoordinateAxis1D; +import ucar.nc2.dataset.NetcdfDataset; +import ucar.nc2.dataset.NetcdfDatasets; +import ucar.nc2.grib.collection.GribCdmIndex; +import ucar.unidata.util.test.TestDir; +import ucar.unidata.util.test.category.NeedsCdmUnitTest; +import java.io.IOException; +import java.lang.invoke.MethodHandles; + +public class TestGribCollectionTimeUnits { + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + // Grib collection with mixture of hours and minutes + private static final String NAME = "testCollectionUnits"; + private static final String MIXED_UNITS_SPEC = + TestDir.cdmUnitTestDir + "/tds/ncep/NDFD_NWS_CONUS_conduit_2p5km_#yyyyMMdd_HHmm#\\.grib2$"; + private static final String MIXED_UNITS_INDEX = + TestDir.cdmUnitTestDir + "/tds/ncep/testCollectionUnits" + GribCdmIndex.NCX_SUFFIX; + + @Test + @Category(NeedsCdmUnitTest.class) + public void shouldUseDefaultUnitsInGrib2Collection() throws IOException { + final FeatureCollectionConfig config = new FeatureCollectionConfig(NAME, NAME, FeatureCollectionType.GRIB2, + MIXED_UNITS_SPEC, null, null, null, "file", null); + + assertThat(config.gribConfig.getTimeUnitConverter()).isEqualTo(null); + final double[] valuesInHoursAndMinutes = new double[] {6.0, 12.0, 18.0, 24.0, 30.0, 36.0, 42.0, 48.0, 54.0, 60.0, + 66.0, 72.0, 360.0, 720.0, 1080.0, 1440.0, 1800.0, 2160.0, 2520.0, 2880.0, 3240.0, 3600.0, 3960.0, 4320.0}; + checkVariableNameAndTimeAxis(config, "Mixed_intervals", "time3", "Hour", valuesInHoursAndMinutes); + } + + @Test + @Category(NeedsCdmUnitTest.class) + public void shouldConvertTimeToMinutesInGrib2Collection() throws IOException { + final FeatureCollectionConfig config = new FeatureCollectionConfig(NAME, NAME, FeatureCollectionType.GRIB2, + MIXED_UNITS_SPEC, null, null, null, "file", null); + + setTimeUnitConversionFromHoursToMinutes(config); + final double[] valuesInMinutes = + new double[] {360.0, 720.0, 1080.0, 1440.0, 1800.0, 2160.0, 2520.0, 2880.0, 3240.0, 3600.0, 3960.0, 4320.0}; + checkVariableNameAndTimeAxis(config, "360_Minute", "time2", "Minute", valuesInMinutes); + } + + @Test + @Category(NeedsCdmUnitTest.class) + public void shouldConvertTimeToHoursInGrib2Collection() throws IOException { + final FeatureCollectionConfig config = new FeatureCollectionConfig(NAME, NAME, FeatureCollectionType.GRIB2, + MIXED_UNITS_SPEC, null, null, null, "file", null); + + setTimeUnitConversionFromMinutesToHours(config); + // hours get rounded down due to refdate being on the half hour + final double[] valuesInHours = new double[] {5.0, 11.0, 17.0, 23.0, 29.0, 35.0, 41.0, 47.0, 53.0, 59.0, 65.0, 71.0}; + checkVariableNameAndTimeAxis(config, "6_Hour", "time2", "Hour", valuesInHours); + } + + private void setTimeUnitConversionFromMinutesToHours(FeatureCollectionConfig config) { + config.gribConfig.addTimeUnitConvert("0", "1"); + } + + private void setTimeUnitConversionFromHoursToMinutes(FeatureCollectionConfig config) { + config.gribConfig.addTimeUnitConvert("1", "0"); + } + + private void checkVariableNameAndTimeAxis(FeatureCollectionConfig config, String intervalName, String timeAxis, + String units, double[] values) throws IOException { + final boolean changed = GribCdmIndex.updateGribCollection(config, CollectionUpdateType.always, logger); + assertThat(changed).isTrue(); + + try (NetcdfDataset netcdfDataset = NetcdfDatasets.openDataset(MIXED_UNITS_INDEX)) { + final Variable variable = + netcdfDataset.findVariable("Best/Total_precipitation_surface_" + intervalName + "_Accumulation"); + assertThat((Iterable) variable).isNotNull(); + assertThat(variable.getDimensionsString()).isEqualTo(timeAxis + " y x"); + final CoordinateAxis coordinateAxis = netcdfDataset.findCoordinateAxis("Best/" + timeAxis); + assertThat((Iterable) coordinateAxis).isNotNull(); + assertThat(coordinateAxis.getUnitsString()).contains(units); + assertThat(((CoordinateAxis1D) coordinateAxis).getCoordValues()).isEqualTo(values); + } + } +} + From 7cd38a2384dd9defa0211c2105c98e23e485324b Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 13 Oct 2022 15:48:45 -0600 Subject: [PATCH 36/70] Remove restriction on updating time unit converter as this will fail when a collection is updated --- grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java index 058d6b9c01..00ee1079b8 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java +++ b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java @@ -460,9 +460,8 @@ public String getLevelNameShort(int id) { private TimeUnitConverter timeUnitConverter; // LOOK not really immutable + // Do not modify timeUnitConverter once it's been set public void setTimeUnitConverter(TimeUnitConverter timeUnitConverter) { - if (this.timeUnitConverter != null) - throw new RuntimeException("Cant modify timeUnitConverter once its been set"); this.timeUnitConverter = timeUnitConverter; } From 0c405583ee901aa7a7f2a67170267d10a28b2d2d Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 17 Oct 2022 15:19:56 -0600 Subject: [PATCH 37/70] Revert "Remove restriction on updating time unit converter as this will fail when a collection is updated" This reverts commit 62e66ad2c7073aace8e955308efe83e2ac1e05a2. --- grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java index 00ee1079b8..058d6b9c01 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java +++ b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java @@ -460,8 +460,9 @@ public String getLevelNameShort(int id) { private TimeUnitConverter timeUnitConverter; // LOOK not really immutable - // Do not modify timeUnitConverter once it's been set public void setTimeUnitConverter(TimeUnitConverter timeUnitConverter) { + if (this.timeUnitConverter != null) + throw new RuntimeException("Cant modify timeUnitConverter once its been set"); this.timeUnitConverter = timeUnitConverter; } From 0747c74280f523e0b7b7aac1a9e5b507bbc2df75 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 31 Oct 2022 14:59:59 -0600 Subject: [PATCH 38/70] Deprecate TimeUnitConverter functionality from Grib2Tables and move to Grib2CollectionBuilder --- .../grib/collection/Grib2CollectionBuilder.java | 14 ++++++++++++-- .../ucar/nc2/grib/grib2/table/Grib2Tables.java | 9 +++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/grib/src/main/java/ucar/nc2/grib/collection/Grib2CollectionBuilder.java b/grib/src/main/java/ucar/nc2/grib/collection/Grib2CollectionBuilder.java index 1fd48a0ba8..79dfb95389 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/Grib2CollectionBuilder.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/Grib2CollectionBuilder.java @@ -7,6 +7,7 @@ import javax.annotation.Nonnull; import thredds.featurecollection.FeatureCollectionConfig; +import thredds.featurecollection.TimeUnitConverter; import thredds.inventory.CollectionUpdateType; import thredds.inventory.MCollection; import thredds.inventory.MFile; @@ -40,6 +41,7 @@ */ class Grib2CollectionBuilder extends GribCollectionBuilder { private final FeatureCollectionConfig.GribConfig gribConfig; + private final TimeUnitConverter timeUnitConverter; private Grib2Tables cust; // LOOK prob name could be dcm.getCollectionName() @@ -48,6 +50,15 @@ class Grib2CollectionBuilder extends GribCollectionBuilder { FeatureCollectionConfig config = (FeatureCollectionConfig) dcm.getAuxInfo(FeatureCollectionConfig.AUX_CONFIG); gribConfig = config.gribConfig; + timeUnitConverter = + gribConfig.getTimeUnitConverter() != null ? gribConfig.getTimeUnitConverter() : new TrivialTimeUnitConverter(); + } + + private static class TrivialTimeUnitConverter implements TimeUnitConverter { + @Override + public int convertTimeUnit(int timeUnit) { + return timeUnit; + } } // read all records in all files, @@ -101,7 +112,6 @@ public List makeGroups(List allFiles, boolea for (Grib2Record gr : index.getRecords()) { // we are using entire Grib2Record - memory limitations if (this.cust == null) { this.cust = Grib2Tables.factory(gr); - cust.setTimeUnitConverter(gribConfig.getTimeUnitConverter()); } if (filterIntervals(gr, gribConfig.intvFilter)) { statsAll.filter++; @@ -275,7 +285,7 @@ public void make(FeatureCollectionConfig.GribConfig config, GribRecordStats coun // create coordinates for each variable for (VariableBag vb : gribvars) { Grib2Pds pdsFirst = vb.first.getPDS(); - int code = cust.convertTimeUnit(pdsFirst.getTimeUnit()); + int code = timeUnitConverter.convertTimeUnit(pdsFirst.getTimeUnit()); vb.timeUnit = userTimeUnit == null ? Grib2Utils.getCalendarPeriod(code) : userTimeUnit; // so can override the // code in config // "timeUnit" diff --git a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java index 058d6b9c01..6a1ceaf0c5 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java +++ b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java @@ -38,6 +38,7 @@ * @since 4/3/11 */ @Immutable +// TODO version 6: remove implements TimeUnitConverter public class Grib2Tables implements ucar.nc2.grib.GribTables, TimeUnitConverter { private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Grib2Tables.class); private static final Map tables = new HashMap<>(); @@ -460,12 +461,20 @@ public String getLevelNameShort(int id) { private TimeUnitConverter timeUnitConverter; // LOOK not really immutable + /** + * @deprecated Remove in version 6, functionality moved to Grib2CollectionBuilder + */ + @Deprecated public void setTimeUnitConverter(TimeUnitConverter timeUnitConverter) { if (this.timeUnitConverter != null) throw new RuntimeException("Cant modify timeUnitConverter once its been set"); this.timeUnitConverter = timeUnitConverter; } + /** + * @deprecated Remove in version 6, functionality moved to Grib2CollectionBuilder + */ + @Deprecated @Override public int convertTimeUnit(int timeUnit) { if (timeUnitConverter == null) From eefaeac96f32ebbd62396510151f34ed9b1f4770 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 31 Oct 2022 16:38:53 -0600 Subject: [PATCH 39/70] Do not use TimeUnitConverter in Grib2Tables --- .../main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java index 6a1ceaf0c5..115f8fa983 100644 --- a/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java +++ b/grib/src/main/java/ucar/nc2/grib/grib2/table/Grib2Tables.java @@ -491,7 +491,7 @@ public CalendarDate getForecastDate(Grib2Record gr) { } else { int val = pds.getForecastTime(); - CalendarPeriod period = Grib2Utils.getCalendarPeriod(convertTimeUnit(pds.getTimeUnit())); + CalendarPeriod period = Grib2Utils.getCalendarPeriod(pds.getTimeUnit()); if (period == null) return null; return gr.getReferenceDate().add(period.multiply(val)); @@ -569,7 +569,7 @@ public TimeCoordIntvDateValue getForecastTimeInterval(Grib2Record gr) { // convert time "range" to units of pds timeUnits int timeUnitOrg = gr.getPDS().getTimeUnit(); - CalendarPeriod wantPeriod = Grib2Utils.getCalendarPeriod(convertTimeUnit(timeUnitOrg)); + CalendarPeriod wantPeriod = Grib2Utils.getCalendarPeriod(timeUnitOrg); if (wantPeriod == null) { return null; } @@ -611,7 +611,7 @@ public double getForecastTimeIntervalSizeInHours(Grib2Pds pds) { TimeIntervalAndUnits intvu = getForecastTimeInterval(pdsIntv); // convert that range to units of hours. - CalendarPeriod timeUnitPeriod = Grib2Utils.getCalendarPeriod(convertTimeUnit(intvu.timeUnitIntv)); + CalendarPeriod timeUnitPeriod = Grib2Utils.getCalendarPeriod(intvu.timeUnitIntv); if (timeUnitPeriod == null) return GribNumbers.UNDEFINEDD; if (timeUnitPeriod.equals(CalendarPeriod.Hour)) @@ -725,7 +725,7 @@ public int[] getForecastTimeIntervalOffset(Grib2Record gr) { return null; Grib2Pds pds = gr.getPDS(); - int unit = convertTimeUnit(pds.getTimeUnit()); + int unit = pds.getTimeUnit(); TimeCoordIntvValue tinv = tinvd.convertReferenceDate(gr.getReferenceDate(), Grib2Utils.getCalendarPeriod(unit)); if (tinv == null) return null; From fc6cd3db6a4526cf7c0e3b1b831e5bb4b47d21cd Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Wed, 2 Nov 2022 14:29:49 -0600 Subject: [PATCH 40/70] Revert "Add test for the grib2 forecast intervals with a unit converter" This reverts commit c8abd5efadb82655a3c5be1ae12f9b68bdb5b2a7. --- .../nc2/grib/grib2/table/TestLocalTables.java | 56 ------------------- 1 file changed, 56 deletions(-) diff --git a/grib/src/test/java/ucar/nc2/grib/grib2/table/TestLocalTables.java b/grib/src/test/java/ucar/nc2/grib/grib2/table/TestLocalTables.java index b585269ab8..17cf0b75fa 100644 --- a/grib/src/test/java/ucar/nc2/grib/grib2/table/TestLocalTables.java +++ b/grib/src/test/java/ucar/nc2/grib/grib2/table/TestLocalTables.java @@ -6,31 +6,14 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.io.IOException; -import java.lang.invoke.MethodHandles; import java.util.Iterator; -import java.util.List; import org.junit.Test; -import org.junit.experimental.categories.Category; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import thredds.featurecollection.FeatureCollectionConfig.TimeUnitConverterHash; -import thredds.inventory.CollectionUpdateType; -import thredds.inventory.MFile; -import thredds.inventory.MFiles; -import ucar.nc2.grib.GribIndex; import ucar.nc2.grib.GribTables; -import ucar.nc2.grib.coord.TimeCoordIntvDateValue; -import ucar.nc2.grib.grib2.Grib2Index; -import ucar.nc2.grib.grib2.Grib2Record; -import ucar.nc2.time.CalendarDate; -import ucar.unidata.util.test.TestDir; -import ucar.unidata.util.test.category.NeedsCdmUnitTest; @RunWith(JUnit4.class) public class TestLocalTables { - private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @Test public void testLocalTables() throws IOException { @@ -74,43 +57,4 @@ public void testEcmwfCodeTables() throws IOException { } } } - - @Test - @Category(NeedsCdmUnitTest.class) - public void shouldUseUnitConverterWhenCreatingForecastInterval() throws IOException { - final MFile file = - MFiles.create(TestDir.cdmUnitTestDir + "/tds/ncep/NDFD_NWS_CONUS_conduit_2p5km_20220816_0030.grib2"); - assertThat(file).isNotNull(); - - final Grib2Index gribIndex = - (Grib2Index) GribIndex.readOrCreateIndexFromSingleFile(false, file, CollectionUpdateType.always, logger); - final List records = gribIndex.getRecords(); - assertThat(records).isNotEmpty(); - final Grib2Record record = records.get(0); - final Grib2Tables tables = Grib2Tables.factory(record); - - final CalendarDate start = CalendarDate.parseISOformat(null, "2022-08-18T00:00:00Z"); - final CalendarDate end = CalendarDate.parseISOformat(null, "2022-08-18T12:00:00Z"); - final TimeCoordIntvDateValue expectedInterval = new TimeCoordIntvDateValue(start, end); - - // default units - final TimeCoordIntvDateValue interval = tables.getForecastTimeInterval(record); - assertThat(interval).isEqualTo(expectedInterval); - - // convert hours to minutes - final TimeUnitConverterHash hoursToMinutesConverter = new TimeUnitConverterHash(); - hoursToMinutesConverter.map.put(1, 0); - tables.setTimeUnitConverter(hoursToMinutesConverter); - - final TimeCoordIntvDateValue intervalHoursToMinutes = tables.getForecastTimeInterval(record); - assertThat(intervalHoursToMinutes).isEqualTo(expectedInterval); - - // convert minutes to hour - final TimeUnitConverterHash minutesToHoursConverter = new TimeUnitConverterHash(); - minutesToHoursConverter.map.put(0, 1); - tables.setTimeUnitConverter(minutesToHoursConverter); - - final TimeCoordIntvDateValue intervalMinutesToHours = tables.getForecastTimeInterval(record); - assertThat(intervalMinutesToHours).isEqualTo(expectedInterval); - } } From 2491fcb04c40a7565cbd3aad89911178c00c7e45 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Thu, 3 Nov 2022 15:59:18 -0600 Subject: [PATCH 41/70] Fix index location in tests --- .../test/java/ucar/nc2/grib/TestGribCollectionTimeUnits.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionTimeUnits.java b/cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionTimeUnits.java index 028115619f..ef42b1dc6c 100644 --- a/cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionTimeUnits.java +++ b/cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionTimeUnits.java @@ -31,8 +31,6 @@ public class TestGribCollectionTimeUnits { private static final String NAME = "testCollectionUnits"; private static final String MIXED_UNITS_SPEC = TestDir.cdmUnitTestDir + "/tds/ncep/NDFD_NWS_CONUS_conduit_2p5km_#yyyyMMdd_HHmm#\\.grib2$"; - private static final String MIXED_UNITS_INDEX = - TestDir.cdmUnitTestDir + "/tds/ncep/testCollectionUnits" + GribCdmIndex.NCX_SUFFIX; @Test @Category(NeedsCdmUnitTest.class) @@ -82,8 +80,9 @@ private void checkVariableNameAndTimeAxis(FeatureCollectionConfig config, String String units, double[] values) throws IOException { final boolean changed = GribCdmIndex.updateGribCollection(config, CollectionUpdateType.always, logger); assertThat(changed).isTrue(); + final String topLevelIndex = GribCdmIndex.getTopIndexFileFromConfig(config).getAbsolutePath(); - try (NetcdfDataset netcdfDataset = NetcdfDatasets.openDataset(MIXED_UNITS_INDEX)) { + try (NetcdfDataset netcdfDataset = NetcdfDatasets.openDataset(topLevelIndex)) { final Variable variable = netcdfDataset.findVariable("Best/Total_precipitation_surface_" + intervalName + "_Accumulation"); assertThat((Iterable) variable).isNotNull(); From dbf98d2d7dcd50a0a553c7d7025436ff914cc203 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 4 Nov 2022 11:26:49 -0600 Subject: [PATCH 42/70] Remove commented out code --- .../ucar/nc2/ncml/TestAggDatasetIsCached.java | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java index a8da6ea714..fb67afee99 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java @@ -47,7 +47,6 @@ public static void cleanupClass() { public void TestAggCached() throws IOException, InvalidRangeException { String filename = TestDir.cdmUnitTestDir + "agg/caching/wqb.ncml"; DatasetUrl durl = DatasetUrl.findDatasetUrl(filename); - // String filename = "file:./"+TestNcMLRead.topDir + "aggExisting.xml"; boolean ok = true; System.out.printf("==========%n"); @@ -116,14 +115,6 @@ public void TestAggCached2() throws IOException, InvalidRangeException { assert fd3 != null; System.out.printf("---FeatureDataset not failed %d%n", i); - /* - * out = new Formatter(); - * ok = CompareNetcdf2.compareFiles(nc1, nc3, out, false, false, false); - * allok &= ok; - * System.out.printf("---fd compare ok %s iter %d%n", ok, i); - * System.out.printf("--------------%nfile=%s%n%s%n", filename, out); - */ - NetcdfDataset nc4 = NetcdfDataset.wrap(nc1, null); System.out.printf("---NetcdfDataset.wrap(nc1, null) object == %d%n", nc4.hashCode()); FeatureDataset fd4 = ucar.nc2.ft.FeatureDatasetFactoryManager.wrap(ucar.nc2.constants.FeatureType.STATION, nc4, @@ -137,16 +128,4 @@ public void TestAggCached2() throws IOException, InvalidRangeException { FileCacheIF cache = NetcdfDataset.getNetcdfFileCache(); cache.showCache(); } - - - /* - * @Test - * public void testProblem() throws Exception { - * show = true; - * Aggregation.setPersistenceCache(new DiskCache2("/.unidata/aggCache", true, -1, -1)); - * openWithEnhance( TestDir.cdmUnitTestDir+"/ft/grid/cg/cg.ncml"); - * Aggregation.setPersistenceCache(null); - * } - */ - } From 599d886ca6138d44352216559495458194043492 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 4 Nov 2022 11:27:33 -0600 Subject: [PATCH 43/70] Remove unused exception from signatures --- .../src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java index fb67afee99..9e96833172 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java @@ -7,7 +7,6 @@ import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ucar.ma2.InvalidRangeException; import ucar.nc2.dataset.DatasetUrl; import ucar.nc2.dataset.NetcdfDataset; import ucar.nc2.dataset.NetcdfDataset.Enhance; @@ -44,7 +43,7 @@ public static void cleanupClass() { } @Test - public void TestAggCached() throws IOException, InvalidRangeException { + public void TestAggCached() throws IOException { String filename = TestDir.cdmUnitTestDir + "agg/caching/wqb.ncml"; DatasetUrl durl = DatasetUrl.findDatasetUrl(filename); boolean ok = true; @@ -88,7 +87,7 @@ private void showModes(Set modes) { @Test // check if caching works - public void TestAggCached2() throws IOException, InvalidRangeException { + public void TestAggCached2() throws IOException { String filename = TestDir.cdmUnitTestDir + "agg/caching/wqb.ncml"; // joinExisting DatasetUrl durl = DatasetUrl.findDatasetUrl(filename); From bba7d57b7171c484b64429a190915196fa0887e4 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 4 Nov 2022 11:38:03 -0600 Subject: [PATCH 44/70] Use google truth asserts --- .../ucar/nc2/ncml/TestAggDatasetIsCached.java | 22 ++++++++----------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java index 9e96833172..f46a5c8e29 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java @@ -1,6 +1,9 @@ package ucar.nc2.ncml; +import static com.google.common.truth.Truth.assertThat; + import java.util.Set; +import java.util.stream.Collectors; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -46,14 +49,13 @@ public static void cleanupClass() { public void TestAggCached() throws IOException { String filename = TestDir.cdmUnitTestDir + "agg/caching/wqb.ncml"; DatasetUrl durl = DatasetUrl.findDatasetUrl(filename); - boolean ok = true; System.out.printf("==========%n"); for (int i = 0; i < 2; i++) { NetcdfDataset ncd = NetcdfDataset.acquireDataset(durl, true, null); NetcdfDataset ncd2 = NetcdfDataset.wrap(ncd, NetcdfDataset.getEnhanceAll()); Formatter out = new Formatter(); - ok &= CompareNetcdf2.compareFiles(ncd, ncd2, out, false, false, false); + assertThat(CompareNetcdf2.compareFiles(ncd, ncd2, out, false, false, false)).isTrue(); System.out.printf("----------------%nfile=%s%n%s%n", filename, out); Set modes = ncd2.getEnhanceMode(); @@ -61,7 +63,6 @@ public void TestAggCached() throws IOException { ncd2.close(); System.out.printf("==========%n"); } - assert ok; Formatter f = new Formatter(); FileCacheIF cache = NetcdfDataset.getNetcdfFileCache(); @@ -69,13 +70,8 @@ public void TestAggCached() throws IOException { System.out.printf("%s%n", f); List cacheFiles = cache.showCache(); - assert cacheFiles.size() == 6; - boolean gotit = false; - for (String name : cacheFiles) { - if (name.endsWith("wqb.ncml")) - gotit = true; - } - assert gotit; + assertThat(cacheFiles.size()).isEqualTo(6); + assertThat(cacheFiles.stream().filter(cacheFile -> cacheFile.endsWith("wqb.ncml")).collect(Collectors.toList())).isNotEmpty(); } private void showModes(Set modes) { @@ -100,7 +96,7 @@ public void TestAggCached2() throws IOException { System.out.printf("---new NetcdfDataset(nc1) object == %d%n", nc2.hashCode()); FeatureDataset fd2 = ucar.nc2.ft.FeatureDatasetFactoryManager.wrap(ucar.nc2.constants.FeatureType.STATION, nc2, null, new Formatter(System.out)); - assert fd2 != null; // no longer fails + assertThat(fd2).isNotNull(); System.out.printf("---FeatureDataset not failed%n"); Formatter out = new Formatter(); @@ -111,14 +107,14 @@ public void TestAggCached2() throws IOException { System.out.printf("---NetcdfDataset.wrap(nc1, enhance) object == %d%n", nc3.hashCode()); FeatureDataset fd3 = ucar.nc2.ft.FeatureDatasetFactoryManager.wrap(ucar.nc2.constants.FeatureType.STATION, nc3, null, new Formatter(System.out)); - assert fd3 != null; + assertThat(fd3).isNotNull(); System.out.printf("---FeatureDataset not failed %d%n", i); NetcdfDataset nc4 = NetcdfDataset.wrap(nc1, null); System.out.printf("---NetcdfDataset.wrap(nc1, null) object == %d%n", nc4.hashCode()); FeatureDataset fd4 = ucar.nc2.ft.FeatureDatasetFactoryManager.wrap(ucar.nc2.constants.FeatureType.STATION, nc4, null, new Formatter(System.err)); - assert fd4 != null; + assertThat(fd4).isNotNull(); System.out.printf("---FeatureDataset not failed%n"); nc1.close(); From f7bc8a09f2a4123893c209ec802717084556294e Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 4 Nov 2022 11:51:09 -0600 Subject: [PATCH 45/70] Use logger --- .../ucar/nc2/ncml/TestAggDatasetIsCached.java | 35 ++++++++----------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java index f46a5c8e29..1c71137d42 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java @@ -50,37 +50,29 @@ public void TestAggCached() throws IOException { String filename = TestDir.cdmUnitTestDir + "agg/caching/wqb.ncml"; DatasetUrl durl = DatasetUrl.findDatasetUrl(filename); - System.out.printf("==========%n"); for (int i = 0; i < 2; i++) { NetcdfDataset ncd = NetcdfDataset.acquireDataset(durl, true, null); NetcdfDataset ncd2 = NetcdfDataset.wrap(ncd, NetcdfDataset.getEnhanceAll()); Formatter out = new Formatter(); assertThat(CompareNetcdf2.compareFiles(ncd, ncd2, out, false, false, false)).isTrue(); - System.out.printf("----------------%nfile=%s%n%s%n", filename, out); + logger.debug("file=" + filename); + logger.debug(out.toString()); Set modes = ncd2.getEnhanceMode(); - showModes(modes); + logger.debug(modes.toString()); ncd2.close(); - System.out.printf("==========%n"); } Formatter f = new Formatter(); FileCacheIF cache = NetcdfDataset.getNetcdfFileCache(); cache.showCache(f); - System.out.printf("%s%n", f); + logger.debug(f.toString()); List cacheFiles = cache.showCache(); assertThat(cacheFiles.size()).isEqualTo(6); assertThat(cacheFiles.stream().filter(cacheFile -> cacheFile.endsWith("wqb.ncml")).collect(Collectors.toList())).isNotEmpty(); } - private void showModes(Set modes) { - for (NetcdfDataset.Enhance mode : modes) { - System.out.printf("%s,", mode); - } - System.out.printf("%n"); - } - @Test // check if caching works public void TestAggCached2() throws IOException { @@ -88,34 +80,35 @@ public void TestAggCached2() throws IOException { DatasetUrl durl = DatasetUrl.findDatasetUrl(filename); for (int i = 0; i < 2; i++) { - System.out.printf("%n=====Iteration %d =====%n", i + 1); + logger.debug("=====Iteration {} =====", i + 1); NetcdfDataset nc1 = NetcdfDataset.acquireDataset(durl, true, null); // put/get in the cache - System.out.printf("-----------------------nc object == %d%n", nc1.hashCode()); + logger.debug("-----------------------nc object == {}", nc1.hashCode()); NetcdfDataset nc2 = new NetcdfDataset(nc1); - System.out.printf("---new NetcdfDataset(nc1) object == %d%n", nc2.hashCode()); + logger.debug("---new NetcdfDataset(nc1) object == {}", nc2.hashCode()); FeatureDataset fd2 = ucar.nc2.ft.FeatureDatasetFactoryManager.wrap(ucar.nc2.constants.FeatureType.STATION, nc2, null, new Formatter(System.out)); assertThat(fd2).isNotNull(); - System.out.printf("---FeatureDataset not failed%n"); + logger.debug("---FeatureDataset not failed"); Formatter out = new Formatter(); boolean ok = CompareNetcdf2.compareFiles(nc1, nc2, out, false, false, false); - System.out.printf("---fd compare ok %s%n%s%n", ok, out); + logger.debug("---fd compare ok {}", ok); + logger.debug(out.toString()); NetcdfDataset nc3 = NetcdfDataset.wrap(nc1, NetcdfDataset.getEnhanceAll()); - System.out.printf("---NetcdfDataset.wrap(nc1, enhance) object == %d%n", nc3.hashCode()); + logger.debug("---NetcdfDataset.wrap(nc1, enhance) object == {}", nc3.hashCode()); FeatureDataset fd3 = ucar.nc2.ft.FeatureDatasetFactoryManager.wrap(ucar.nc2.constants.FeatureType.STATION, nc3, null, new Formatter(System.out)); assertThat(fd3).isNotNull(); - System.out.printf("---FeatureDataset not failed %d%n", i); + logger.debug("---FeatureDataset not failed {}", i); NetcdfDataset nc4 = NetcdfDataset.wrap(nc1, null); - System.out.printf("---NetcdfDataset.wrap(nc1, null) object == %d%n", nc4.hashCode()); + logger.debug("---NetcdfDataset.wrap(nc1, null) object == {}", nc4.hashCode()); FeatureDataset fd4 = ucar.nc2.ft.FeatureDatasetFactoryManager.wrap(ucar.nc2.constants.FeatureType.STATION, nc4, null, new Formatter(System.err)); assertThat(fd4).isNotNull(); - System.out.printf("---FeatureDataset not failed%n"); + logger.debug("---FeatureDataset not failed"); nc1.close(); } From a33d3aca0cc32d8fe4ad3408e848daa0a035b020 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 4 Nov 2022 11:53:54 -0600 Subject: [PATCH 46/70] Add assert instead of logging --- .../src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java index 1c71137d42..7d9fdd733b 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java @@ -92,8 +92,7 @@ public void TestAggCached2() throws IOException { logger.debug("---FeatureDataset not failed"); Formatter out = new Formatter(); - boolean ok = CompareNetcdf2.compareFiles(nc1, nc2, out, false, false, false); - logger.debug("---fd compare ok {}", ok); + assertThat(CompareNetcdf2.compareFiles(nc1, nc2, out, false, false, false)).isTrue(); logger.debug(out.toString()); NetcdfDataset nc3 = NetcdfDataset.wrap(nc1, NetcdfDataset.getEnhanceAll()); From 9da80e6f58c19015e3bd0bb31846211cfa6d4c05 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 4 Nov 2022 12:00:19 -0600 Subject: [PATCH 47/70] Improve names --- .../java/ucar/nc2/ncml/TestAggDatasetIsCached.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java index 7d9fdd733b..04dad3f45e 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java @@ -32,6 +32,7 @@ @Category(NeedsCdmUnitTest.class) public class TestAggDatasetIsCached { private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static final String JOIN_EXISTING_AGGREGATION = TestDir.cdmUnitTestDir + "agg/caching/wqb.ncml"; @BeforeClass public static void setupClass() { @@ -46,9 +47,8 @@ public static void cleanupClass() { } @Test - public void TestAggCached() throws IOException { - String filename = TestDir.cdmUnitTestDir + "agg/caching/wqb.ncml"; - DatasetUrl durl = DatasetUrl.findDatasetUrl(filename); + public void testAggCached() throws IOException { + DatasetUrl durl = DatasetUrl.findDatasetUrl(JOIN_EXISTING_AGGREGATION); for (int i = 0; i < 2; i++) { NetcdfDataset ncd = NetcdfDataset.acquireDataset(durl, true, null); @@ -74,10 +74,8 @@ public void TestAggCached() throws IOException { } @Test - // check if caching works - public void TestAggCached2() throws IOException { - String filename = TestDir.cdmUnitTestDir + "agg/caching/wqb.ncml"; // joinExisting - DatasetUrl durl = DatasetUrl.findDatasetUrl(filename); + public void testAggCached2() throws IOException { + DatasetUrl durl = DatasetUrl.findDatasetUrl(JOIN_EXISTING_AGGREGATION); for (int i = 0; i < 2; i++) { logger.debug("=====Iteration {} =====", i + 1); From d4d6e8b0dd41e0de8f6c3eb1a13b15e6460adeb4 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 4 Nov 2022 12:00:41 -0600 Subject: [PATCH 48/70] Remove log message --- cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java | 1 - 1 file changed, 1 deletion(-) diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java index 04dad3f45e..4781ab4ab0 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java @@ -55,7 +55,6 @@ public void testAggCached() throws IOException { NetcdfDataset ncd2 = NetcdfDataset.wrap(ncd, NetcdfDataset.getEnhanceAll()); Formatter out = new Formatter(); assertThat(CompareNetcdf2.compareFiles(ncd, ncd2, out, false, false, false)).isTrue(); - logger.debug("file=" + filename); logger.debug(out.toString()); Set modes = ncd2.getEnhanceMode(); From 314817112f50e4e383aa27a2986d6b49b78b828d Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 4 Nov 2022 14:49:14 -0600 Subject: [PATCH 49/70] Remove unused import --- .../src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java | 1 - 1 file changed, 1 deletion(-) diff --git a/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java b/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java index ef111196b2..0cf047f879 100644 --- a/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java +++ b/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java @@ -12,7 +12,6 @@ import ucar.nc2.dataset.DatasetUrl; import ucar.nc2.dataset.NetcdfDatasets; import ucar.nc2.util.CancelTask; -import ucar.nc2.dataset.NetcdfDataset; import ucar.nc2.NetcdfFile; import ucar.unidata.util.test.TestDir; import ucar.unidata.util.StringUtil2; From ec8e98c0d5731f8978ee7f7ed3988f51fc79b77e Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 4 Nov 2022 14:50:09 -0600 Subject: [PATCH 50/70] Remove commented out code --- .../test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java b/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java index 0cf047f879..bea57c1894 100644 --- a/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java +++ b/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java @@ -52,14 +52,12 @@ private void loadFilesIntoCache(File dir, FileCache cache) { loadFilesIntoCache(f, cache); else if (f.getPath().endsWith(".nc") && f.length() > 0) { - // System.out.println(" open "+f.getPath()); try { String want = StringUtil2.replace(f.getPath(), '\\', "/"); DatasetUrl durl = DatasetUrl.create(null, want); cache.acquire(factory, durl); count++; } catch (IOException e) { - // e.printStackTrace(); System.out.println(" *** failed on " + f.getPath()); } } @@ -81,7 +79,6 @@ public void testNetcdfFileCache() throws IOException { for (Object key : map.keySet()) { FileCache.CacheElement elem = map.get(key); - // System.out.println(" "+key+" == "+elem); Assert.assertEquals(1, elem.list.size()); } @@ -95,7 +92,6 @@ public void testNetcdfFileCache() throws IOException { for (Object key : map.keySet()) { FileCache.CacheElement elem = map.get(key); - // System.out.println(" "+key+" == "+elem); Assert.assertEquals(2, elem.list.size()); checkAllSame(elem.list); } @@ -217,7 +213,6 @@ private void testPeriodicCleanup(FileCache cache) throws IOException { int N = 10000; int PROD_THREAD = 10; int CONS_THREAD = 10; - // int QSIZE = 10; int SKIP = 100; @Test @@ -323,9 +318,7 @@ void consume(Future x) throws ExecutionException, InterruptedException, IOExcept if (x.isDone()) { NetcdfFile ncfile = (NetcdfFile) x.get(); ncfile.close(); - // format.format(" closed qsize= %3d\n", queue.size()); } else { - // format.format(" lost file= %3d\n", queue.size()); queue.add(x); // put it back } From 444c55cd8f3f4967e4e52c90f29949545bc266e6 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 4 Nov 2022 14:54:32 -0600 Subject: [PATCH 51/70] Fail if cannot acquire from cache --- .../test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java b/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java index bea57c1894..1878fdb290 100644 --- a/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java +++ b/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java @@ -4,6 +4,8 @@ */ package ucar.nc2.util.cache; +import static org.junit.Assert.fail; + import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -58,7 +60,7 @@ else if (f.getPath().endsWith(".nc") && f.length() > 0) { cache.acquire(factory, durl); count++; } catch (IOException e) { - System.out.println(" *** failed on " + f.getPath()); + fail(" *** failed on " + f.getPath()); } } } From 6883a477f0cc7c1ebc0bedcd409333ce326b627c Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 4 Nov 2022 14:55:48 -0600 Subject: [PATCH 52/70] Remove print statements --- .../src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java b/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java index 1878fdb290..ce9ae681c0 100644 --- a/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java +++ b/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java @@ -68,10 +68,7 @@ else if (f.getPath().endsWith(".nc") && f.length() > 0) { @Test public void testNetcdfFileCache() throws IOException { - System.out.printf("TestNetcdfFileCache%n"); - loadFilesIntoCache(new File(TestDir.cdmLocalTestDataDir), cache); - System.out.println(" loaded " + count); cache.showCache(new Formatter(System.out)); From a5170f2eb1840847e760e50e688637eda7c2f6b1 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 4 Nov 2022 15:12:53 -0600 Subject: [PATCH 53/70] Use logger --- .../test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java b/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java index ce9ae681c0..baff416cfe 100644 --- a/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java +++ b/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java @@ -185,7 +185,7 @@ public void testPeriodicClear() throws IOException { private void testPeriodicCleanup(FileCache cache) throws IOException { loadFilesIntoCache(new File(TestDir.cdmLocalTestDataDir), cache); - System.out.println(" loaded " + count); + logger.debug(" loaded " + count); // close all Map map = cache.getCache(); @@ -197,7 +197,7 @@ private void testPeriodicCleanup(FileCache cache) throws IOException { files.add(file.ncfile); } } - System.out.println(" close " + files.size()); + logger.debug(" close " + files.size()); for (FileCacheable ncfile : files) { ncfile.close(); @@ -269,11 +269,10 @@ public void testConcurrentAccess() throws InterruptedException { if (file.isLocked.get()) locks++; } - // System.out.println(" key= "+key+ " size= "+elem.list.size()+" locks="+locks); total_locks += locks; total += elem.list.size(); } - System.out.println(" total=" + total + " total_locks=" + total_locks); + logger.debug(" total=" + total + " total_locks=" + total_locks); // assert total_locks == map.keySet().size(); cache.clearCache(false); From 7691ee32c0051038f11215786e8448b33afc0a84 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 4 Nov 2022 15:15:02 -0600 Subject: [PATCH 54/70] Use google truth asserts --- .../nc2/util/cache/TestNetcdfFileCache.java | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java b/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java index baff416cfe..bf1f6b65a9 100644 --- a/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java +++ b/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java @@ -4,9 +4,9 @@ */ package ucar.nc2.util.cache; +import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.fail; -import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; @@ -74,11 +74,11 @@ public void testNetcdfFileCache() throws IOException { // count cache size Map map = cache.getCache(); - Assert.assertEquals(count, map.values().size()); + assertThat(map.values().size()).isEqualTo(count); for (Object key : map.keySet()) { FileCache.CacheElement elem = map.get(key); - Assert.assertEquals(1, elem.list.size()); + assertThat(elem.list.size()).isEqualTo(1); } // load same files again - should be added to the list, rather than creating a new elem @@ -87,28 +87,28 @@ public void testNetcdfFileCache() throws IOException { map = cache.getCache(); cache.showCache(new Formatter(System.out)); - Assert.assertEquals(saveCount, map.values().size()); // problem + assertThat(map.values().size()).isEqualTo(saveCount); // problem for (Object key : map.keySet()) { FileCache.CacheElement elem = map.get(key); - Assert.assertEquals(2, elem.list.size()); + assertThat(elem.list.size()).isEqualTo(2); checkAllSame(elem.list); } cache.clearCache(true); map = cache.getCache(); - Assert.assertEquals(0, map.values().size()); + assertThat(map.values().size()).isEqualTo(0); // load again loadFilesIntoCache(new File(TestDir.cdmLocalTestDataDir), cache); map = cache.getCache(); - Assert.assertEquals(saveCount, map.values().size()); + assertThat(map.values().size()).isEqualTo(saveCount); // close all List files = new ArrayList<>(); for (Object key : map.keySet()) { FileCache.CacheElement elem = map.get(key); - Assert.assertEquals(1, elem.list.size()); + assertThat(elem.list.size()).isEqualTo(1); for (FileCache.CacheElement.CacheFile file : elem.list) { files.add(file.ncfile); } @@ -118,34 +118,34 @@ public void testNetcdfFileCache() throws IOException { } cache.clearCache(false); map = cache.getCache(); - Assert.assertEquals(0, map.values().size()); + assertThat(map.values().size()).isEqualTo(0); // load twice loadFilesIntoCache(new File(TestDir.cdmLocalTestDataDir), cache); loadFilesIntoCache(new File(TestDir.cdmLocalTestDataDir), cache); map = cache.getCache(); - Assert.assertEquals(saveCount, map.values().size()); + assertThat(map.values().size()).isEqualTo(saveCount); // close 1 of 2 for (Object key : map.keySet()) { FileCache.CacheElement elem = map.get(key); - Assert.assertEquals(2, elem.list.size()); + assertThat(elem.list.size()).isEqualTo(2); FileCache.CacheElement.CacheFile first = elem.list.get(0); first.ncfile.close(); - Assert.assertTrue(!first.isLocked.get()); - Assert.assertEquals(2, elem.list.size()); + assertThat(first.isLocked.get()).isFalse(); + assertThat(elem.list.size()).isEqualTo(2); } map = cache.getCache(); - Assert.assertEquals(saveCount, map.values().size()); + assertThat(map.values().size()).isEqualTo(saveCount); cache.clearCache(false); map = cache.getCache(); - Assert.assertEquals(saveCount, map.values().size()); + assertThat(map.values().size()).isEqualTo(saveCount); for (Object key : map.keySet()) { FileCache.CacheElement elem = map.get(key); - Assert.assertEquals(1, elem.list.size()); + assertThat(elem.list.size()).isEqualTo(1); } cache.clearCache(true); @@ -154,15 +154,15 @@ public void testNetcdfFileCache() throws IOException { void checkAllSame(List list) { FileCache.CacheElement.CacheFile first = null; for (FileCache.CacheElement.CacheFile file : list) { - Assert.assertTrue(file.isLocked.get()); - Assert.assertEquals(0, file.countAccessed); // countAccessed not incremented until its closed, so == 0 - Assert.assertNotEquals(0, file.lastAccessed); + assertThat(file.isLocked.get()).isTrue(); + assertThat(file.countAccessed).isEqualTo(0); // countAccessed not incremented until its closed, so == 0 + assertThat(file.lastAccessed).isNotEqualTo(0); if (first == null) first = file; else { - assert first.ncfile.getLocation().equals(file.ncfile.getLocation()); - assert first.lastAccessed < file.lastAccessed; + assertThat(first.ncfile.getLocation()).isEqualTo(file.ncfile.getLocation()); + assertThat(first.lastAccessed).isLessThan(file.lastAccessed); } } } @@ -175,12 +175,12 @@ public void testPeriodicClear() throws IOException { FileCache cache = new FileCache(0, 10, 60 * 60); testPeriodicCleanup(cache); Map map = cache.getCache(); - assert map.values().size() == 0 : map.values().size(); + assertThat(map.values().size()).isEqualTo(0); cache = new FileCache(5, 10, 60 * 60); testPeriodicCleanup(cache); map = cache.getCache(); - assert map.values().size() == 5 : map.values().size(); + assertThat(map.values().size()).isEqualTo(5); } private void testPeriodicCleanup(FileCache cache) throws IOException { @@ -192,7 +192,7 @@ private void testPeriodicCleanup(FileCache cache) throws IOException { List files = new ArrayList<>(); for (Object key : map.keySet()) { FileCache.CacheElement elem = map.get(key); - assert elem.list.size() == 1; + assertThat(elem.list.size()).isEqualTo(1); for (FileCache.CacheElement.CacheFile file : elem.list) { files.add(file.ncfile); } @@ -260,7 +260,7 @@ public void testConcurrentAccess() throws InterruptedException { HashSet checkUnique = new HashSet<>(); map = cache.getCache(); for (Object key : map.keySet()) { - assert !checkUnique.contains(key); + assertThat(checkUnique.contains(key)).isFalse(); checkUnique.add(key); int locks = 0; FileCache.CacheElement elem = map.get(key); From c23af0491a8ba04d755eb95137a3abf30f9aa017 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 4 Nov 2022 15:18:31 -0600 Subject: [PATCH 55/70] Add brackets to if/else --- .../test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java b/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java index bf1f6b65a9..87fb45ef2e 100644 --- a/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java +++ b/cdm/core/src/test/java/ucar/nc2/util/cache/TestNetcdfFileCache.java @@ -50,10 +50,9 @@ private void loadFilesIntoCache(File dir, FileCache cache) { return; for (File f : files) { - if (f.isDirectory()) + if (f.isDirectory()) { loadFilesIntoCache(f, cache); - - else if (f.getPath().endsWith(".nc") && f.length() > 0) { + } else if (f.getPath().endsWith(".nc") && f.length() > 0) { try { String want = StringUtil2.replace(f.getPath(), '\\', "/"); DatasetUrl durl = DatasetUrl.create(null, want); From cd36b71818d162a469330477e8e4c8e3e99c3cd5 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 7 Nov 2022 08:44:41 -0700 Subject: [PATCH 56/70] Spotless formatting within line length --- .../src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java index 4781ab4ab0..f3f865e1a1 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java @@ -69,7 +69,8 @@ public void testAggCached() throws IOException { List cacheFiles = cache.showCache(); assertThat(cacheFiles.size()).isEqualTo(6); - assertThat(cacheFiles.stream().filter(cacheFile -> cacheFile.endsWith("wqb.ncml")).collect(Collectors.toList())).isNotEmpty(); + assertThat(cacheFiles.stream().filter(cacheFile -> cacheFile.endsWith("wqb.ncml")).collect(Collectors.toList())) + .isNotEmpty(); } @Test From 3407ec836d0fb1971c5c1fc3b431c498a9213f2a Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 8 Nov 2022 14:08:33 -0700 Subject: [PATCH 57/70] Put file name instead of path in error message --- .../ucar/nc2/grib/collection/PartitionCollectionImmutable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/grib/src/main/java/ucar/nc2/grib/collection/PartitionCollectionImmutable.java b/grib/src/main/java/ucar/nc2/grib/collection/PartitionCollectionImmutable.java index 881b3793f4..91905bcfb3 100644 --- a/grib/src/main/java/ucar/nc2/grib/collection/PartitionCollectionImmutable.java +++ b/grib/src/main/java/ucar/nc2/grib/collection/PartitionCollectionImmutable.java @@ -229,7 +229,7 @@ public String getIndexFilenameInCache() throws FileNotFoundException { File existingFile = GribIndexCache.getExistingFileOrCache(file.getPath()); if (existingFile == null) { - throw new FileNotFoundException("No index filename for partition= " + this + " looking for " + file.getPath()); + throw new FileNotFoundException("No index filename for partition= " + this + " looking for " + file.getName()); } /* From 95ee09ffa2d2c851a3bce80d3bc1b33cbefb6d7c Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 11 Nov 2022 09:44:12 -0700 Subject: [PATCH 58/70] Fix typos and spelling in javadocs and comments --- .../thredds/client/catalog/ServiceType.java | 6 +++--- cdm/core/src/main/java/ucar/ma2/Array.java | 12 +++++------ cdm/core/src/main/java/ucar/nc2/Variable.java | 12 +++++------ .../main/java/opendap/dap/ServerVersion.java | 20 +++++++++---------- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/cdm/core/src/main/java/thredds/client/catalog/ServiceType.java b/cdm/core/src/main/java/thredds/client/catalog/ServiceType.java index 5691a248de..dec63a1ba1 100644 --- a/cdm/core/src/main/java/thredds/client/catalog/ServiceType.java +++ b/cdm/core/src/main/java/thredds/client/catalog/ServiceType.java @@ -25,15 +25,15 @@ public enum ServiceType { H5Service, // HTTPServer("HTTP file download.", AccessType.DataAccess, "httpserver"), // JupyterNotebook("Generate a Jupyter Notebook that uses Siphon to access this dataset.", AccessType.DataViewer, null), // - ISO("Provide ISO 19115 metdata representation of a dataset's structure and metadata.", AccessType.Metadata, null), // + ISO("Provide ISO 19115 metadata representation of a dataset's structure and metadata.", AccessType.Metadata, null), // LAS, // NcJSON, // NCML("Provide NCML representation of a dataset.", AccessType.Metadata, "ncml"), // NetcdfSubset("A web service for subsetting CDM scientific datasets.", AccessType.DataAccess, null), // - OPENDAP("Access dataset through OPeNDAP using the DAP2 protcol.", AccessType.DataAccess, "dods"), // + OPENDAP("Access dataset through OPeNDAP using the DAP2 protocol.", AccessType.DataAccess, "dods"), // OPENDAPG, // Resolver, // - THREDDS("Access dataset through thredds calalog enty", AccessType.DataAccess, "thredds"), // + THREDDS("Access dataset through thredds catalog entry", AccessType.DataAccess, "thredds"), // UDDC("An evaluation of how well the metadata contained in the dataset" + " conforms to the NetCDF Attribute Convention for Data Discovery (NACDD)", AccessType.Metadata, null), // WebForm, // ?? diff --git a/cdm/core/src/main/java/ucar/ma2/Array.java b/cdm/core/src/main/java/ucar/ma2/Array.java index b5a629b402..8fc516cc75 100644 --- a/cdm/core/src/main/java/ucar/ma2/Array.java +++ b/cdm/core/src/main/java/ucar/ma2/Array.java @@ -40,7 +40,7 @@ *

* The type, shape and backing storage of an Array are immutable. * The data itself is read or written using an Index or an IndexIterator, which stores any needed state information - * for efficient traversal. This makes use of Arrays thread-safe (as long as you dont share the Index or IndexIterator) + * for efficient traversal. This makes use of Arrays thread-safe (as long as you don't share the Index or IndexIterator) * except for the possibility of non-atomic read/write on long/doubles. If this is the case, you should probably * synchronize your calls. Presumably 64-bit CPUs will make those operations atomic also. * @@ -165,7 +165,7 @@ public static Array makeObjectArray(DataType dtype, Class classType, int[] shape */ public static Array factoryConstant(DataType dtype, int[] shape, Object storage) { Index index = new IndexConstant(shape); - // cant go though the factory, must call the general constructor + // can't go through the factory, must call the general constructor switch (dtype) { case BOOLEAN: return new ArrayBoolean(index, (boolean[]) storage); @@ -337,7 +337,7 @@ public static Array makeArray(DataType dtype, List stringValues) throws } else if (dtype == DataType.LONG) { if (dtype.isUnsigned()) { BigInteger biggy = new BigInteger(s); - dataI.setLongNext(biggy.longValue()); // > 63 bits will become "negetive". + dataI.setLongNext(biggy.longValue()); // > 63 bits will become "negative". } else { long val = Long.parseLong(s); @@ -693,7 +693,7 @@ public Object get1DJavaArray(Class wantType) { * It avoids copying if possible. * Only for numeric types (byte, short, int, long, double, float) * - * @return equivilent data in a ByteBuffer + * @return equivalent data in a ByteBuffer */ public ByteBuffer getDataAsByteBuffer() { throw new UnsupportedOperationException(); @@ -807,7 +807,7 @@ public Object copyTo1DJavaArray() { } /** - * Copy this array to a n-Dimensional Java primitive array of type getElementType() + * Copy this array to an n-Dimensional Java primitive array of type getElementType() * and rank getRank(). Makes a copy of the data. * * @return a Java ND array of type getElementType(). @@ -1258,7 +1258,7 @@ public short nextShort() { * Return the next int in the local iterator. * Uses the local iterator, which is not thread-safe. Use getIndexIterator if you need thread-safety. * - * @return next element as a int, same as IndexIterator.getIntNext(). + * @return next element as an int, same as IndexIterator.getIntNext(). */ public int nextInt() { return ii.getIntNext(); diff --git a/cdm/core/src/main/java/ucar/nc2/Variable.java b/cdm/core/src/main/java/ucar/nc2/Variable.java index ab3752979d..19d6b98060 100644 --- a/cdm/core/src/main/java/ucar/nc2/Variable.java +++ b/cdm/core/src/main/java/ucar/nc2/Variable.java @@ -130,7 +130,7 @@ public long getSize() { /** * Get the number of bytes for one element of this Variable. * For Variables of primitive type, this is equal to getDataType().getSize(). - * Variables of String type dont know their size, so what they return is undefined. + * Variables of String type don't know their size, so what they return is undefined. * Variables of Structure type return the total number of bytes for all the members of * one Structure, plus possibly some extra padding, depending on the underlying format. * Variables of Sequence type return the number of bytes of one element. @@ -149,7 +149,7 @@ public int getRank() { /** * Get the parent group, or if null, the root group. * - * @deprecated Will go away in ver6, shouldnt be needed when builders are used. + * @deprecated Will go away in ver6, shouldn't be needed when builders are used. */ @Deprecated public Group getParentGroupOrRoot() { @@ -302,7 +302,7 @@ public String getUnitsString() { } /** - * Get shape as an List of Range objects. + * Get shape as a List of Range objects. * The List is immutable. * * @return List of Ranges, one for each Dimension. @@ -344,7 +344,7 @@ public void setProxyReader(ProxyReader proxyReader) { * Each Range corresponds to a Dimension, and specifies the section of data to read in that Dimension. * A Range object may be null, which means use the entire dimension. * @return a new Variable which is a logical section of this Variable. - * @throws InvalidRangeException if shape and range list dont match + * @throws InvalidRangeException if shape and range list don't match */ public Variable section(List ranges) throws InvalidRangeException { return section(new Section(ranges, shape).makeImmutable()); @@ -585,7 +585,7 @@ public Array read(List ranges) throws IOException, InvalidRangeException * To read the data in all structures, use ncfile.readSectionSpec(). *

* Note this only allows you to specify a subset of this variable. - * If the variable is nested in a array of structures and you want to subset that, use + * If the variable is nested in an array of structures and you want to subset that, use * NetcdfFile.read(String sectionSpec, boolean flatten); * * @param section list of Range specifying the section of data to read. @@ -1608,7 +1608,7 @@ public boolean isCaching() { /** * Note that standalone Ncml caches data values set in the Ncml. - * So one cannont invalidate those caches. + * So one cannot invalidate those caches. * * @deprecated Use Variable.builder() */ diff --git a/opendap/src/main/java/opendap/dap/ServerVersion.java b/opendap/src/main/java/opendap/dap/ServerVersion.java index cba26e8697..0b9649efc3 100644 --- a/opendap/src/main/java/opendap/dap/ServerVersion.java +++ b/opendap/src/main/java/opendap/dap/ServerVersion.java @@ -48,12 +48,12 @@ import ucar.httpservices.HTTPMethod; /** - * Aprses and holds the Server Version information returned by a DAP server. + * Parses and holds the Server Version information returned by a DAP server. * This information is used to determine the version of the DAP protocol used to * by the DAP server to encode the data.
*
* Currently the Server Version can come from - * more than one source. An DAP server responding to to a client request + * more than one source. An DAP server responding to a client request * over HTTP must include either an XDAP header or an * XDODS-Server header.
* > @@ -81,7 +81,7 @@ * considered deprecated. Thus, clients seeking to read data from OPeNDAP * servers should first check the server response for the existence of an * XDAP header, if one is not found then the client should - * check for an XDODS-Server header. If the repsonse is missing + * check for an XDODS-Server header. If the response is missing * both headers then an exception should be thrown as the server response is * invalid. * @@ -124,7 +124,7 @@ public class ServerVersion implements java.io.Serializable { * * @param method The GetMethod containing the DAP2 headers. * @throws DAP2Exception When bad things happen (like the headers are - * missing or incorrectly constructed. + * missing or incorrectly constructed.) */ public ServerVersion(HTTPMethod method) throws DAP2Exception { @@ -158,9 +158,9 @@ public ServerVersion(HTTPMethod method) throws DAP2Exception { * Determines Server (Protocol) Version based on the headers associated * with the passed java.net.URLConnection. * - * @param connection The URLCOnnection containing the DAP2 headers + * @param connection The URLConnection containing the DAP2 headers * @throws DAP2Exception When bad things happen (like the headers are - * missing or incorrectly constructed. + * missing or incorrectly constructed.) */ public ServerVersion(URLConnection connection) throws DAP2Exception { @@ -186,7 +186,7 @@ public ServerVersion(URLConnection connection) throws DAP2Exception { // This is important! If neither of these headers (XDAP or - // XDODS-Server is present then we are not connected to a real + // XDODS-Server) is present then we are not connected to a real // OPeNDAP server. Period. Without the information contained // in these headers some data types (Such as Sequence) cannot // be correctly serialized/deserialized. @@ -199,7 +199,7 @@ public ServerVersion(URLConnection connection) throws DAP2Exception { /** * Construct a new ServerVersion, setting major and minor version based * on the full version string. Currently the Server Version can come from - * more than one source. An OPeNDAP server responding to to a client request + * more than one source. An OPeNDAP server responding to a client request * over HTTP must include either an XDAP header or an * XDODS-Server header.
* > @@ -227,7 +227,7 @@ public ServerVersion(URLConnection connection) throws DAP2Exception { * considered deprecated. Thus, clients seeking to read data from OPeNDAP * servers should first check the server response for the existence of an * XDAP header, if one is not found then the client should - * check for an XDODS-Server header. If the repsonse is missing + * check for an XDODS-Server header. If the response is missing * both headers then an exception should be thrown as the server response is * invalid. * @@ -271,7 +271,7 @@ private void processXDODSServerVersion(String ver) throws DAP2Exception { // If the identifying word is missing then we punt and try to // read the value as if it is just the Major.Minor number. // Which is really bullshit, but a bunch of servers got built that - // do corrrectly utilze this parameter. + // do correctly utilize this parameter. verIndex = 0; } From 503ff9bf0032c5ce723a963a5f6634b792520761 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 22 Nov 2022 10:32:44 -0700 Subject: [PATCH 59/70] Add tests for last modified of netcdfDataset --- .../ucar/nc2/dataset/TestNetcdfDataset.java | 17 +++++++++++++++++ .../unidata/io/s3/TestS3NetcdfDataset.java | 19 +++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 cdm/core/src/test/java/ucar/nc2/dataset/TestNetcdfDataset.java create mode 100644 cdm/s3/src/test/java/ucar/unidata/io/s3/TestS3NetcdfDataset.java diff --git a/cdm/core/src/test/java/ucar/nc2/dataset/TestNetcdfDataset.java b/cdm/core/src/test/java/ucar/nc2/dataset/TestNetcdfDataset.java new file mode 100644 index 0000000000..892b72bc71 --- /dev/null +++ b/cdm/core/src/test/java/ucar/nc2/dataset/TestNetcdfDataset.java @@ -0,0 +1,17 @@ +package ucar.nc2.dataset; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.IOException; +import org.junit.Test; + +public class TestNetcdfDataset { + + @Test + public void shouldGetLastModified() throws IOException { + final String location = "src/test/data/ncml/nc/jan.nc"; + try (NetcdfDataset netcdfDataset = NetcdfDatasets.openDataset(location)) { + assertThat(netcdfDataset.getLastModified()).isGreaterThan(0); + } + } +} diff --git a/cdm/s3/src/test/java/ucar/unidata/io/s3/TestS3NetcdfDataset.java b/cdm/s3/src/test/java/ucar/unidata/io/s3/TestS3NetcdfDataset.java new file mode 100644 index 0000000000..5700104ae7 --- /dev/null +++ b/cdm/s3/src/test/java/ucar/unidata/io/s3/TestS3NetcdfDataset.java @@ -0,0 +1,19 @@ +package ucar.unidata.io.s3; + +import static com.google.common.truth.Truth.assertThat; + +import java.io.IOException; +import org.junit.Test; +import ucar.nc2.dataset.NetcdfDataset; +import ucar.nc2.dataset.NetcdfDatasets; + +public class TestS3NetcdfDataset { + + @Test + public void shouldGetLastModifiedForS3File() throws IOException { + final String location = "cdms3:thredds-test-data?testData.nc"; + try (NetcdfDataset netcdfDataset = NetcdfDatasets.openDataset(location)) { + assertThat(netcdfDataset.getLastModified()).isGreaterThan(0); + } + } +} From 3378352cf671d8d7fe1446dbb884c5856775c5b2 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Tue, 22 Nov 2022 10:39:23 -0700 Subject: [PATCH 60/70] Use MFile for last modified so it works for s3 --- .../main/java/ucar/nc2/iosp/AbstractIOServiceProvider.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cdm/core/src/main/java/ucar/nc2/iosp/AbstractIOServiceProvider.java b/cdm/core/src/main/java/ucar/nc2/iosp/AbstractIOServiceProvider.java index fdfe7e3453..0dc9e8febf 100644 --- a/cdm/core/src/main/java/ucar/nc2/iosp/AbstractIOServiceProvider.java +++ b/cdm/core/src/main/java/ucar/nc2/iosp/AbstractIOServiceProvider.java @@ -6,6 +6,8 @@ package ucar.nc2.iosp; import javax.annotation.Nullable; +import thredds.inventory.MFile; +import thredds.inventory.MFiles; import ucar.ma2.Array; import ucar.ma2.InvalidRangeException; import ucar.ma2.Section; @@ -161,8 +163,8 @@ public boolean syncExtend() throws IOException { */ public long getLastModified() { if (location != null) { - File file = new File(location); - return file.lastModified(); + MFile file = MFiles.create(location); + return file != null ? file.getLastModified() : 0; } else { return 0; } From 30e6459e0c453bf07137e97d00650e4f3eb94724 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Mon, 28 Nov 2022 17:18:28 -0700 Subject: [PATCH 61/70] Write compact xml when it doesn't need to be human readable --- .../src/main/java/ucar/nc2/ft/fmrc/GridDatasetInv.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cdm/core/src/main/java/ucar/nc2/ft/fmrc/GridDatasetInv.java b/cdm/core/src/main/java/ucar/nc2/ft/fmrc/GridDatasetInv.java index 938541b823..ec222d6170 100644 --- a/cdm/core/src/main/java/ucar/nc2/ft/fmrc/GridDatasetInv.java +++ b/cdm/core/src/main/java/ucar/nc2/ft/fmrc/GridDatasetInv.java @@ -394,6 +394,16 @@ private EnsCoord getEnsCoordinate(int ens_id) { ////////////////////////////////////////////////////////////// + /** + * Write the XML representation to a compact String. + * + * @return the compact XML representation to a String. + */ + public String writeCompactXML(Date lastModified) { + XMLOutputter fmt = new XMLOutputter(Format.getCompactFormat()); + return fmt.outputString(writeDocument(lastModified)); + } + /** * Write the XML representation to a String. * From cae45e75960a61954c38b3aabf8aeacbb0318a24 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 11 Nov 2022 12:49:50 -0700 Subject: [PATCH 62/70] Use a temorary folder in the tests instead of renaming a test file --- .../ucar/nc2/ncml/TestOffAggUpdating.java | 68 +++++++++---------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java index 6965927bd2..15186fd5e2 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java @@ -33,8 +33,10 @@ package ucar.nc2.ncml; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import thredds.client.catalog.ServiceType; @@ -66,38 +68,38 @@ public class TestOffAggUpdating { private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - String dir = TestDir.cdmUnitTestDir + "agg/updating"; - String location = dir + "agg/updating.ncml"; - File dirFile = new File(dir); - String extraFile = dir + "/extra.nc"; - - String ncml = "\n" - + "\n" - + " \n" + " \n" + " \n" - + " \n" + " \n" - + " \n" + " \n" - + " \n" + " \n" - + " \n" + ""; + private static final String dir = TestDir.cdmUnitTestDir + "agg/updating"; + private static final String location = "test/location.ncml"; + private static final Path extraFile = Paths.get(dir, "extra.nc"); + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + private String getNcml() { + return "\n" + + "\n" + + " \n" + + " \n" + + " \n" + " \n" + + " \n" + " \n" + + " \n" + " \n" + + " \n" + " \n" + + ""; + } @Before - public void setup() { - assert dirFile.exists(); - assert dirFile.isDirectory(); - assert dirFile.listFiles() != null; + public void setup() throws IOException { + Files.copy(Paths.get(dir, "ds1.nc"), Paths.get(tempFolder.getRoot().toString(), "ds1.nc")); + Files.copy(Paths.get(dir, "ds2.nc"), Paths.get(tempFolder.getRoot().toString(), "ds2.nc")); } @Test public void testUpdateSync() throws IOException, InvalidRangeException, InterruptedException { - // make sure that the extra file is not in the agg - move(extraFile); - // open the agg - NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(ncml), location, null); + NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(getNcml()), location, null); check(ncfile, 12); - // now make sure that the extra file is in the agg - moveBack(extraFile); + addExtraFile(); // reread ncfile.syncExtend(); @@ -108,15 +110,11 @@ public void testUpdateSync() throws IOException, InvalidRangeException, Interrup @Test public void testUpdateLastModified() throws IOException, InvalidRangeException, InterruptedException { - // make sure that the extra file is not in the agg - move(extraFile); - // open the agg - NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(ncml), location, null); + NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(getNcml()), location, null); long start = ncfile.getLastModified(); - // now make sure that the extra file is in the agg - moveBack(extraFile); + addExtraFile(); // reread long end = ncfile.getLastModified(); @@ -131,9 +129,6 @@ public void testUpdateLastModified() throws IOException, InvalidRangeException, @Test public void testUpdateCache() throws IOException, InvalidRangeException, InterruptedException { - // make sure that the extra file is not in the agg - move(extraFile); - DatasetUrl durl = DatasetUrl.findDatasetUrl(location); // open the agg @@ -141,8 +136,7 @@ public void testUpdateCache() throws IOException, InvalidRangeException, Interru check(ncfile, 12); - // now make sure that the extra file is in the agg - moveBack(extraFile); + addExtraFile(); // reread ncfile.syncExtend(); @@ -167,10 +161,14 @@ private class NcmlStringFileFactory implements ucar.nc2.util.cache.FileFactory { @Override public FileCacheable open(DatasetUrl durl, int buffer_size, CancelTask cancelTask, Object iospMessage) throws IOException { - return NcMLReader.readNcML(new StringReader(ncml), durl.trueurl, null); + return NcMLReader.readNcML(new StringReader(getNcml()), durl.trueurl, null); } } + private void addExtraFile() throws IOException { + Files.copy(extraFile, Paths.get(tempFolder.getRoot().toString(), "extra.nc")); + } + public static boolean move(String filename) throws IOException { Path src = Paths.get(filename); if (!Files.exists(src)) From 38494226801d24aefcf257c0c36351e5d0370d55 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 11 Nov 2022 12:54:13 -0700 Subject: [PATCH 63/70] Remove unused imports --- cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java index 15186fd5e2..3f46071ccd 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java @@ -39,7 +39,6 @@ import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import thredds.client.catalog.ServiceType; import ucar.ma2.InvalidRangeException; import ucar.nc2.NetcdfFile; import ucar.nc2.Variable; @@ -49,7 +48,6 @@ import ucar.nc2.util.cache.FileCacheable; import ucar.unidata.util.test.category.NeedsCdmUnitTest; import ucar.unidata.util.test.TestDir; -import java.io.File; import java.io.IOException; import java.io.StringReader; import java.lang.invoke.MethodHandles; From ad0df0b485fac8a2f0f43f6ac2f795865fc04c5f Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 11 Nov 2022 12:55:34 -0700 Subject: [PATCH 64/70] Shorten copyright --- .../ucar/nc2/ncml/TestOffAggUpdating.java | 31 ++----------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java index 3f46071ccd..66d8111be8 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java @@ -1,33 +1,6 @@ /* - * Copyright (c) 1998 - 2009. University Corporation for Atmospheric Research/Unidata - * Portions of this software were developed by the Unidata Program at the - * University Corporation for Atmospheric Research. - * - * Access and use of this software shall impose the following obligations - * and understandings on the user. The user is granted the right, without - * any fee or cost, to use, copy, modify, alter, enhance and distribute - * this software, and any derivative works thereof, and its supporting - * documentation for any purpose whatsoever, provided that this entire - * notice appears in all copies of the software, derivative works and - * supporting documentation. Further, UCAR requests that the user credit - * UCAR/Unidata in any publications that result from the use of this - * software or in any product that includes this software. The names UCAR - * and/or Unidata, however, may not be used in any advertising or publicity - * to endorse or promote any products or commercial entity unless specific - * written permission is obtained from UCAR/Unidata. The user also - * understands that UCAR/Unidata is not obligated to provide the user with - * any support, consulting, training or assistance of any kind with regard - * to the use, operation and performance of this software nor to provide - * the user with any updates, revisions, new versions or "bug fixes." - * - * THIS SOFTWARE IS PROVIDED BY UCAR/UNIDATA "AS IS" AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL UCAR/UNIDATA BE LIABLE FOR ANY SPECIAL, - * INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING - * FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, - * NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION - * WITH THE ACCESS, USE OR PERFORMANCE OF THIS SOFTWARE. + * Copyright (c) 1998-2022 John Caron and University Corporation for Atmospheric Research/Unidata + * See LICENSE for license information. */ package ucar.nc2.ncml; From 755ebffccf675351dc5155255793f821059e866d Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 11 Nov 2022 12:58:30 -0700 Subject: [PATCH 65/70] Use google truth asserts --- .../java/ucar/nc2/ncml/TestOffAggUpdating.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java index 66d8111be8..edb6b8f0a6 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java @@ -5,6 +5,8 @@ package ucar.nc2.ncml; +import static com.google.common.truth.Truth.assertThat; + import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -89,11 +91,11 @@ public void testUpdateLastModified() throws IOException, InvalidRangeException, // reread long end = ncfile.getLastModified(); - assert (end > start); + assertThat(end).isGreaterThan(start); // again long end2 = ncfile.getLastModified(); - assert (end == end2); + assertThat(end).isEqualTo(end2); ncfile.close(); } @@ -118,13 +120,13 @@ public void testUpdateCache() throws IOException, InvalidRangeException, Interru private void check(NetcdfFile ncfile, int n) throws IOException, InvalidRangeException { Variable v = ncfile.findVariable("time"); - assert v != null; + assertThat((Object) v).isNotNull(); System.out.printf(" time= %s%n", v.getNameAndDimensions()); - assert v.getSize() == n : v.getSize(); + assertThat(v.getSize()).isEqualTo(n); v = ncfile.findVariable("eta"); - assert v != null; - assert v.getRank() == 3 : v.getRank(); + assertThat((Object) v).isNotNull(); + assertThat(v.getRank()).isEqualTo(3); } private class NcmlStringFileFactory implements ucar.nc2.util.cache.FileFactory { From 3c107fc50f60fb150295598904489c2b8d1465c8 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 11 Nov 2022 12:54:25 -0700 Subject: [PATCH 66/70] Remove unused throws declarations --- .../src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java index edb6b8f0a6..66936fe393 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java @@ -14,7 +14,6 @@ import org.junit.rules.TemporaryFolder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ucar.ma2.InvalidRangeException; import ucar.nc2.NetcdfFile; import ucar.nc2.Variable; import ucar.nc2.dataset.DatasetUrl; @@ -67,7 +66,7 @@ public void setup() throws IOException { } @Test - public void testUpdateSync() throws IOException, InvalidRangeException, InterruptedException { + public void testUpdateSync() throws IOException { // open the agg NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(getNcml()), location, null); check(ncfile, 12); @@ -82,7 +81,7 @@ public void testUpdateSync() throws IOException, InvalidRangeException, Interrup } @Test - public void testUpdateLastModified() throws IOException, InvalidRangeException, InterruptedException { + public void testUpdateLastModified() throws IOException { // open the agg NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(getNcml()), location, null); long start = ncfile.getLastModified(); @@ -101,7 +100,7 @@ public void testUpdateLastModified() throws IOException, InvalidRangeException, } @Test - public void testUpdateCache() throws IOException, InvalidRangeException, InterruptedException { + public void testUpdateCache() throws IOException { DatasetUrl durl = DatasetUrl.findDatasetUrl(location); // open the agg @@ -118,7 +117,7 @@ public void testUpdateCache() throws IOException, InvalidRangeException, Interru ncfile.close(); } - private void check(NetcdfFile ncfile, int n) throws IOException, InvalidRangeException { + private void check(NetcdfFile ncfile, int n) { Variable v = ncfile.findVariable("time"); assertThat((Object) v).isNotNull(); System.out.printf(" time= %s%n", v.getNameAndDimensions()); From ffcb81307156ed15cdaf066b2f0a48484d1d1e83 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 11 Nov 2022 13:01:57 -0700 Subject: [PATCH 67/70] Use logger --- cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java index 66936fe393..2a64f64625 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java @@ -120,7 +120,7 @@ public void testUpdateCache() throws IOException { private void check(NetcdfFile ncfile, int n) { Variable v = ncfile.findVariable("time"); assertThat((Object) v).isNotNull(); - System.out.printf(" time= %s%n", v.getNameAndDimensions()); + logger.debug(" time= {}", v.getNameAndDimensions()); assertThat(v.getSize()).isEqualTo(n); v = ncfile.findVariable("eta"); From 7be025a426fd8a22ee63460d866e1aec7138203c Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 11 Nov 2022 13:06:55 -0700 Subject: [PATCH 68/70] Improve variable names --- .../ucar/nc2/ncml/TestOffAggUpdating.java | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java index 2a64f64625..1b8032ba10 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java @@ -43,6 +43,8 @@ public class TestOffAggUpdating { private static final String dir = TestDir.cdmUnitTestDir + "agg/updating"; private static final String location = "test/location.ncml"; private static final Path extraFile = Paths.get(dir, "extra.nc"); + private static final int expectedSizeOfTime = 12; + private static final int expectedSizeOfTimeWithExtraFile = 18; @Rule public TemporaryFolder tempFolder = new TemporaryFolder(); @@ -69,13 +71,13 @@ public void setup() throws IOException { public void testUpdateSync() throws IOException { // open the agg NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(getNcml()), location, null); - check(ncfile, 12); + check(ncfile, expectedSizeOfTime); addExtraFile(); // reread ncfile.syncExtend(); - check(ncfile, 18); + check(ncfile, expectedSizeOfTimeWithExtraFile); ncfile.close(); } @@ -106,26 +108,26 @@ public void testUpdateCache() throws IOException { // open the agg NetcdfFile ncfile = NetcdfDataset.acquireDataset(new NcmlStringFileFactory(), durl, null, -1, null, null); - check(ncfile, 12); + check(ncfile, expectedSizeOfTime); addExtraFile(); // reread ncfile.syncExtend(); - check(ncfile, 18); + check(ncfile, expectedSizeOfTimeWithExtraFile); ncfile.close(); } - private void check(NetcdfFile ncfile, int n) { - Variable v = ncfile.findVariable("time"); - assertThat((Object) v).isNotNull(); - logger.debug(" time= {}", v.getNameAndDimensions()); - assertThat(v.getSize()).isEqualTo(n); + private void check(NetcdfFile ncfile, int expectedSizeOfTime) { + final Variable time = ncfile.findVariable("time"); + assertThat((Object) time).isNotNull(); + logger.debug(" time= {}", time.getNameAndDimensions()); + assertThat(time.getSize()).isEqualTo(expectedSizeOfTime); - v = ncfile.findVariable("eta"); - assertThat((Object) v).isNotNull(); - assertThat(v.getRank()).isEqualTo(3); + final Variable eta = ncfile.findVariable("eta"); + assertThat((Object) eta).isNotNull(); + assertThat(eta.getRank()).isEqualTo(3); } private class NcmlStringFileFactory implements ucar.nc2.util.cache.FileFactory { From e46146c45f55ae910c7b2ddf8ca25a576a695616 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 11 Nov 2022 13:22:39 -0700 Subject: [PATCH 69/70] Move helper functions into other class that still uses them --- .../java/ucar/nc2/ncml/TestOffAggNewSync.java | 37 +++++++++++++++---- .../ucar/nc2/ncml/TestOffAggUpdating.java | 20 ---------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggNewSync.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggNewSync.java index 6d2999d1ba..b12bfcf479 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggNewSync.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggNewSync.java @@ -4,6 +4,10 @@ */ package ucar.nc2.ncml; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -37,14 +41,14 @@ public class TestOffAggNewSync { @Ignore("file in use - testing artifact") public void testMove() throws IOException, InterruptedException { String fname = dataDir + "WEST-CONUS_4km_3.9_20050912_2130.gini"; - if (!TestOffAggUpdating.move(fname)) + if (!move(fname)) System.out.printf("Move failed on %s%n", fname); System.out.printf("%s%n", aggExistingSync); NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(aggExistingSync), "aggExistingSync", null); testAggCoordVar(ncfile, ntimes - 1); ncfile.close(); - if (!TestOffAggUpdating.moveBack(fname)) + if (!moveBack(fname)) System.out.printf("Move back failed on %s%n", fname); ncfile = NcMLReader.readNcML(new StringReader(aggExistingSync), "aggExistingSync", null); @@ -62,14 +66,14 @@ public void testRemove() throws IOException, InterruptedException { ncfile.close(); String fname = dataDir + "WEST-CONUS_4km_3.9_20050912_2130.gini"; - boolean ok = TestOffAggUpdating.move(fname); + boolean ok = move(fname); int nfiles = ok ? ntimes - 1 : ntimes; // sometimes fails ncfile = NcMLReader.readNcML(new StringReader(aggExistingSync), "aggExistingSync", null); testAggCoordVar(ncfile, nfiles); ncfile.close(); - TestOffAggUpdating.moveBack(fname); + moveBack(fname); System.out.printf("ok testRemove%n"); } @@ -77,13 +81,13 @@ public void testRemove() throws IOException, InterruptedException { @Ignore("file in use - testing artifact") public void testSync() throws IOException, InterruptedException { String fname = dataDir + "WEST-CONUS_4km_3.9_20050912_2130.gini"; - if (!TestOffAggUpdating.move(fname)) + if (!move(fname)) System.out.printf("Move failed on %s%n", fname); NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(aggExistingSync), "aggExistingSync", null); testAggCoordVar(ncfile, ntimes - 1); - if (!TestOffAggUpdating.moveBack(fname)) + if (!moveBack(fname)) System.out.printf("Move back failed on %s%n", fname); Thread.sleep(2000); @@ -102,7 +106,7 @@ public void testSyncRemove() throws IOException, InterruptedException { System.out.println(""); String fname = dataDir + "WEST-CONUS_4km_3.9_20050912_2130.gini"; - boolean ok = TestOffAggUpdating.move(fname); + boolean ok = move(fname); int nfiles = ok ? ntimes - 1 : ntimes; // sometimes fails Thread.sleep(2000); @@ -111,7 +115,7 @@ public void testSyncRemove() throws IOException, InterruptedException { ncfile.close(); // if (!moveBack(dataDir + fname )) - if (!TestOffAggUpdating.moveBack(fname)) + if (!moveBack(fname)) System.out.printf("Move back failed on %s%n", fname); else System.out.printf("ok testSyncRemove %n"); @@ -133,6 +137,23 @@ public void testAggCoordVar(NetcdfFile ncfile, int n) throws IOException { assert data.getShape()[0] == n; } + private static boolean move(String filename) throws IOException { + Path src = Paths.get(filename); + if (!Files.exists(src)) + return false; + Path dest = Paths.get(filename + ".save"); + Files.move(src, dest, StandardCopyOption.REPLACE_EXISTING); + return true; + } + + private static boolean moveBack(String filename) throws IOException { + Path src = Paths.get(filename + ".save"); + if (!Files.exists(src)) + return false; + Path dest = Paths.get(filename); + Files.move(src, dest, StandardCopyOption.REPLACE_EXISTING); + return true; + } } diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java index 1b8032ba10..0d920c2559 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java @@ -28,7 +28,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; /** * Updating aggregation @@ -142,24 +141,5 @@ public FileCacheable open(DatasetUrl durl, int buffer_size, CancelTask cancelTas private void addExtraFile() throws IOException { Files.copy(extraFile, Paths.get(tempFolder.getRoot().toString(), "extra.nc")); } - - public static boolean move(String filename) throws IOException { - Path src = Paths.get(filename); - if (!Files.exists(src)) - return false; - Path dest = Paths.get(filename + ".save"); - Files.move(src, dest, StandardCopyOption.REPLACE_EXISTING); - return true; - } - - - public static boolean moveBack(String filename) throws IOException { - Path src = Paths.get(filename + ".save"); - if (!Files.exists(src)) - return false; - Path dest = Paths.get(filename); - Files.move(src, dest, StandardCopyOption.REPLACE_EXISTING); - return true; - } } From 5324d8b8be4a7d9f8f64cc02e1e84698fbe63667 Mon Sep 17 00:00:00 2001 From: Tara Drwenski Date: Fri, 11 Nov 2022 13:26:42 -0700 Subject: [PATCH 70/70] Use undeprecated version of acquireDataset --- cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java index 0d920c2559..57c7d65bfd 100644 --- a/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java +++ b/cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggUpdating.java @@ -17,7 +17,7 @@ import ucar.nc2.NetcdfFile; import ucar.nc2.Variable; import ucar.nc2.dataset.DatasetUrl; -import ucar.nc2.dataset.NetcdfDataset; +import ucar.nc2.dataset.NetcdfDatasets; import ucar.nc2.util.CancelTask; import ucar.nc2.util.cache.FileCacheable; import ucar.unidata.util.test.category.NeedsCdmUnitTest; @@ -105,7 +105,7 @@ public void testUpdateCache() throws IOException { DatasetUrl durl = DatasetUrl.findDatasetUrl(location); // open the agg - NetcdfFile ncfile = NetcdfDataset.acquireDataset(new NcmlStringFileFactory(), durl, null, -1, null, null); + NetcdfFile ncfile = NetcdfDatasets.acquireDataset(new NcmlStringFileFactory(), durl, null, -1, null, null); check(ncfile, expectedSizeOfTime);