Skip to content

Commit

Permalink
Merge branch 'Unidata:maint-5.x' into handle-zero-dimension-2
Browse files Browse the repository at this point in the history
  • Loading branch information
@mnlerman
mnlerman authored Dec 2, 2022
2 parents 44a82ad + 5324d8b commit 6d44d10
Show file tree
Hide file tree
Showing 421 changed files with 735 additions and 594 deletions.
97 changes: 97 additions & 0 deletions cdm-test/src/test/java/ucar/nc2/grib/TestGribCollectionTimeUnits.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
/*
* Copyright (c) 1998-2022 University Corporation for Atmospheric Research/Unidata
* See LICENSE for license information.
*/
package ucar.nc2.grib;

import static com.google.common.truth.Truth.assertThat;

import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import thredds.featurecollection.FeatureCollectionConfig;
import thredds.featurecollection.FeatureCollectionType;
import thredds.inventory.*;
import ucar.nc2.Variable;
import ucar.nc2.dataset.CoordinateAxis;
import ucar.nc2.dataset.CoordinateAxis1D;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.dataset.NetcdfDatasets;
import ucar.nc2.grib.collection.GribCdmIndex;
import ucar.unidata.util.test.TestDir;
import ucar.unidata.util.test.category.NeedsCdmUnitTest;
import java.io.IOException;
import java.lang.invoke.MethodHandles;

public class TestGribCollectionTimeUnits {
private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

// Grib collection with mixture of hours and minutes
private static final String NAME = "testCollectionUnits";
private static final String MIXED_UNITS_SPEC =
TestDir.cdmUnitTestDir + "/tds/ncep/NDFD_NWS_CONUS_conduit_2p5km_#yyyyMMdd_HHmm#\\.grib2$";

@Test
@Category(NeedsCdmUnitTest.class)
public void shouldUseDefaultUnitsInGrib2Collection() throws IOException {
final FeatureCollectionConfig config = new FeatureCollectionConfig(NAME, NAME, FeatureCollectionType.GRIB2,
MIXED_UNITS_SPEC, null, null, null, "file", null);

assertThat(config.gribConfig.getTimeUnitConverter()).isEqualTo(null);
final double[] valuesInHoursAndMinutes = new double[] {6.0, 12.0, 18.0, 24.0, 30.0, 36.0, 42.0, 48.0, 54.0, 60.0,
66.0, 72.0, 360.0, 720.0, 1080.0, 1440.0, 1800.0, 2160.0, 2520.0, 2880.0, 3240.0, 3600.0, 3960.0, 4320.0};
checkVariableNameAndTimeAxis(config, "Mixed_intervals", "time3", "Hour", valuesInHoursAndMinutes);
}

@Test
@Category(NeedsCdmUnitTest.class)
public void shouldConvertTimeToMinutesInGrib2Collection() throws IOException {
final FeatureCollectionConfig config = new FeatureCollectionConfig(NAME, NAME, FeatureCollectionType.GRIB2,
MIXED_UNITS_SPEC, null, null, null, "file", null);

setTimeUnitConversionFromHoursToMinutes(config);
final double[] valuesInMinutes =
new double[] {360.0, 720.0, 1080.0, 1440.0, 1800.0, 2160.0, 2520.0, 2880.0, 3240.0, 3600.0, 3960.0, 4320.0};
checkVariableNameAndTimeAxis(config, "360_Minute", "time2", "Minute", valuesInMinutes);
}

@Test
@Category(NeedsCdmUnitTest.class)
public void shouldConvertTimeToHoursInGrib2Collection() throws IOException {
final FeatureCollectionConfig config = new FeatureCollectionConfig(NAME, NAME, FeatureCollectionType.GRIB2,
MIXED_UNITS_SPEC, null, null, null, "file", null);

setTimeUnitConversionFromMinutesToHours(config);
// hours get rounded down due to refdate being on the half hour
final double[] valuesInHours = new double[] {5.0, 11.0, 17.0, 23.0, 29.0, 35.0, 41.0, 47.0, 53.0, 59.0, 65.0, 71.0};
checkVariableNameAndTimeAxis(config, "6_Hour", "time2", "Hour", valuesInHours);
}

private void setTimeUnitConversionFromMinutesToHours(FeatureCollectionConfig config) {
config.gribConfig.addTimeUnitConvert("0", "1");
}

private void setTimeUnitConversionFromHoursToMinutes(FeatureCollectionConfig config) {
config.gribConfig.addTimeUnitConvert("1", "0");
}

private void checkVariableNameAndTimeAxis(FeatureCollectionConfig config, String intervalName, String timeAxis,
String units, double[] values) throws IOException {
final boolean changed = GribCdmIndex.updateGribCollection(config, CollectionUpdateType.always, logger);
assertThat(changed).isTrue();
final String topLevelIndex = GribCdmIndex.getTopIndexFileFromConfig(config).getAbsolutePath();

try (NetcdfDataset netcdfDataset = NetcdfDatasets.openDataset(topLevelIndex)) {
final Variable variable =
netcdfDataset.findVariable("Best/Total_precipitation_surface_" + intervalName + "_Accumulation");
assertThat((Iterable<?>) variable).isNotNull();
assertThat(variable.getDimensionsString()).isEqualTo(timeAxis + " y x");
final CoordinateAxis coordinateAxis = netcdfDataset.findCoordinateAxis("Best/" + timeAxis);
assertThat((Iterable<?>) coordinateAxis).isNotNull();
assertThat(coordinateAxis.getUnitsString()).contains(units);
assertThat(((CoordinateAxis1D) coordinateAxis).getCoordValues()).isEqualTo(values);
}
}
}

92 changes: 28 additions & 64 deletions cdm-test/src/test/java/ucar/nc2/ncml/TestAggDatasetIsCached.java
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
package ucar.nc2.ncml;

import static com.google.common.truth.Truth.assertThat;

import java.util.Set;
import java.util.stream.Collectors;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import ucar.ma2.InvalidRangeException;
import ucar.nc2.dataset.DatasetUrl;
import ucar.nc2.dataset.NetcdfDataset;
import ucar.nc2.dataset.NetcdfDataset.Enhance;
Expand All @@ -30,6 +32,7 @@
@Category(NeedsCdmUnitTest.class)
public class TestAggDatasetIsCached {
private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final String JOIN_EXISTING_AGGREGATION = TestDir.cdmUnitTestDir + "agg/caching/wqb.ncml";

@BeforeClass
public static void setupClass() {
Expand All @@ -44,109 +47,70 @@ public static void cleanupClass() {
}

@Test
public void TestAggCached() throws IOException, InvalidRangeException {
String filename = TestDir.cdmUnitTestDir + "agg/caching/wqb.ncml";
DatasetUrl durl = DatasetUrl.findDatasetUrl(filename);
// String filename = "file:./"+TestNcMLRead.topDir + "aggExisting.xml";
boolean ok = true;
public void testAggCached() throws IOException {
DatasetUrl durl = DatasetUrl.findDatasetUrl(JOIN_EXISTING_AGGREGATION);

System.out.printf("==========%n");
for (int i = 0; i < 2; i++) {
NetcdfDataset ncd = NetcdfDataset.acquireDataset(durl, true, null);
NetcdfDataset ncd2 = NetcdfDataset.wrap(ncd, NetcdfDataset.getEnhanceAll());
Formatter out = new Formatter();
ok &= CompareNetcdf2.compareFiles(ncd, ncd2, out, false, false, false);
System.out.printf("----------------%nfile=%s%n%s%n", filename, out);
assertThat(CompareNetcdf2.compareFiles(ncd, ncd2, out, false, false, false)).isTrue();
logger.debug(out.toString());

Set<Enhance> modes = ncd2.getEnhanceMode();
showModes(modes);
logger.debug(modes.toString());
ncd2.close();
System.out.printf("==========%n");
}
assert ok;

Formatter f = new Formatter();
FileCacheIF cache = NetcdfDataset.getNetcdfFileCache();
cache.showCache(f);
System.out.printf("%s%n", f);
logger.debug(f.toString());

List<String> cacheFiles = cache.showCache();
assert cacheFiles.size() == 6;
boolean gotit = false;
for (String name : cacheFiles) {
if (name.endsWith("wqb.ncml"))
gotit = true;
}
assert gotit;
}

private void showModes(Set<NetcdfDataset.Enhance> modes) {
for (NetcdfDataset.Enhance mode : modes) {
System.out.printf("%s,", mode);
}
System.out.printf("%n");
assertThat(cacheFiles.size()).isEqualTo(6);
assertThat(cacheFiles.stream().filter(cacheFile -> cacheFile.endsWith("wqb.ncml")).collect(Collectors.toList()))
.isNotEmpty();
}

@Test
// check if caching works
public void TestAggCached2() throws IOException, InvalidRangeException {
String filename = TestDir.cdmUnitTestDir + "agg/caching/wqb.ncml"; // joinExisting
DatasetUrl durl = DatasetUrl.findDatasetUrl(filename);
public void testAggCached2() throws IOException {
DatasetUrl durl = DatasetUrl.findDatasetUrl(JOIN_EXISTING_AGGREGATION);

for (int i = 0; i < 2; i++) {
System.out.printf("%n=====Iteration %d =====%n", i + 1);
logger.debug("=====Iteration {} =====", i + 1);
NetcdfDataset nc1 = NetcdfDataset.acquireDataset(durl, true, null); // put/get in the cache
System.out.printf("-----------------------nc object == %d%n", nc1.hashCode());
logger.debug("-----------------------nc object == {}", nc1.hashCode());

NetcdfDataset nc2 = new NetcdfDataset(nc1);
System.out.printf("---new NetcdfDataset(nc1) object == %d%n", nc2.hashCode());
logger.debug("---new NetcdfDataset(nc1) object == {}", nc2.hashCode());
FeatureDataset fd2 = ucar.nc2.ft.FeatureDatasetFactoryManager.wrap(ucar.nc2.constants.FeatureType.STATION, nc2,
null, new Formatter(System.out));
assert fd2 != null; // no longer fails
System.out.printf("---FeatureDataset not failed%n");
assertThat(fd2).isNotNull();
logger.debug("---FeatureDataset not failed");

Formatter out = new Formatter();
boolean ok = CompareNetcdf2.compareFiles(nc1, nc2, out, false, false, false);
System.out.printf("---fd compare ok %s%n%s%n", ok, out);
assertThat(CompareNetcdf2.compareFiles(nc1, nc2, out, false, false, false)).isTrue();
logger.debug(out.toString());

NetcdfDataset nc3 = NetcdfDataset.wrap(nc1, NetcdfDataset.getEnhanceAll());
System.out.printf("---NetcdfDataset.wrap(nc1, enhance) object == %d%n", nc3.hashCode());
logger.debug("---NetcdfDataset.wrap(nc1, enhance) object == {}", nc3.hashCode());
FeatureDataset fd3 = ucar.nc2.ft.FeatureDatasetFactoryManager.wrap(ucar.nc2.constants.FeatureType.STATION, nc3,
null, new Formatter(System.out));
assert fd3 != null;
System.out.printf("---FeatureDataset not failed %d%n", i);

/*
* out = new Formatter();
* ok = CompareNetcdf2.compareFiles(nc1, nc3, out, false, false, false);
* allok &= ok;
* System.out.printf("---fd compare ok %s iter %d%n", ok, i);
* System.out.printf("--------------%nfile=%s%n%s%n", filename, out);
*/
assertThat(fd3).isNotNull();
logger.debug("---FeatureDataset not failed {}", i);

NetcdfDataset nc4 = NetcdfDataset.wrap(nc1, null);
System.out.printf("---NetcdfDataset.wrap(nc1, null) object == %d%n", nc4.hashCode());
logger.debug("---NetcdfDataset.wrap(nc1, null) object == {}", nc4.hashCode());
FeatureDataset fd4 = ucar.nc2.ft.FeatureDatasetFactoryManager.wrap(ucar.nc2.constants.FeatureType.STATION, nc4,
null, new Formatter(System.err));
assert fd4 != null;
System.out.printf("---FeatureDataset not failed%n");
assertThat(fd4).isNotNull();
logger.debug("---FeatureDataset not failed");

nc1.close();
}

FileCacheIF cache = NetcdfDataset.getNetcdfFileCache();
cache.showCache();
}


/*
* @Test
* public void testProblem() throws Exception {
* show = true;
* Aggregation.setPersistenceCache(new DiskCache2("/.unidata/aggCache", true, -1, -1));
* openWithEnhance( TestDir.cdmUnitTestDir+"/ft/grid/cg/cg.ncml");
* Aggregation.setPersistenceCache(null);
* }
*/

}
37 changes: 29 additions & 8 deletions cdm-test/src/test/java/ucar/nc2/ncml/TestOffAggNewSync.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@
*/
package ucar.nc2.ncml;

import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
Expand Down Expand Up @@ -37,14 +41,14 @@ public class TestOffAggNewSync {
@Ignore("file in use - testing artifact")
public void testMove() throws IOException, InterruptedException {
String fname = dataDir + "WEST-CONUS_4km_3.9_20050912_2130.gini";
if (!TestOffAggUpdating.move(fname))
if (!move(fname))
System.out.printf("Move failed on %s%n", fname);
System.out.printf("%s%n", aggExistingSync);
NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(aggExistingSync), "aggExistingSync", null);
testAggCoordVar(ncfile, ntimes - 1);
ncfile.close();

if (!TestOffAggUpdating.moveBack(fname))
if (!moveBack(fname))
System.out.printf("Move back failed on %s%n", fname);

ncfile = NcMLReader.readNcML(new StringReader(aggExistingSync), "aggExistingSync", null);
Expand All @@ -62,28 +66,28 @@ public void testRemove() throws IOException, InterruptedException {
ncfile.close();

String fname = dataDir + "WEST-CONUS_4km_3.9_20050912_2130.gini";
boolean ok = TestOffAggUpdating.move(fname);
boolean ok = move(fname);
int nfiles = ok ? ntimes - 1 : ntimes; // sometimes fails

ncfile = NcMLReader.readNcML(new StringReader(aggExistingSync), "aggExistingSync", null);
testAggCoordVar(ncfile, nfiles);
ncfile.close();

TestOffAggUpdating.moveBack(fname);
moveBack(fname);
System.out.printf("ok testRemove%n");
}

@Test
@Ignore("file in use - testing artifact")
public void testSync() throws IOException, InterruptedException {
String fname = dataDir + "WEST-CONUS_4km_3.9_20050912_2130.gini";
if (!TestOffAggUpdating.move(fname))
if (!move(fname))
System.out.printf("Move failed on %s%n", fname);

NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(aggExistingSync), "aggExistingSync", null);
testAggCoordVar(ncfile, ntimes - 1);

if (!TestOffAggUpdating.moveBack(fname))
if (!moveBack(fname))
System.out.printf("Move back failed on %s%n", fname);

Thread.sleep(2000);
Expand All @@ -102,7 +106,7 @@ public void testSyncRemove() throws IOException, InterruptedException {
System.out.println("");

String fname = dataDir + "WEST-CONUS_4km_3.9_20050912_2130.gini";
boolean ok = TestOffAggUpdating.move(fname);
boolean ok = move(fname);
int nfiles = ok ? ntimes - 1 : ntimes; // sometimes fails
Thread.sleep(2000);

Expand All @@ -111,7 +115,7 @@ public void testSyncRemove() throws IOException, InterruptedException {
ncfile.close();

// if (!moveBack(dataDir + fname ))
if (!TestOffAggUpdating.moveBack(fname))
if (!moveBack(fname))
System.out.printf("Move back failed on %s%n", fname);
else
System.out.printf("ok testSyncRemove %n");
Expand All @@ -133,6 +137,23 @@ public void testAggCoordVar(NetcdfFile ncfile, int n) throws IOException {
assert data.getShape()[0] == n;
}

private static boolean move(String filename) throws IOException {
Path src = Paths.get(filename);
if (!Files.exists(src))
return false;
Path dest = Paths.get(filename + ".save");
Files.move(src, dest, StandardCopyOption.REPLACE_EXISTING);
return true;
}

private static boolean moveBack(String filename) throws IOException {
Path src = Paths.get(filename + ".save");
if (!Files.exists(src))
return false;
Path dest = Paths.get(filename);
Files.move(src, dest, StandardCopyOption.REPLACE_EXISTING);
return true;
}
}


Loading

0 comments on commit 6d44d10

Please sign in to comment.