Skip to content

Commit

Permalink
incorporate PR feedback
Browse files Browse the repository at this point in the history
  • Loading branch information
haileyajohnson committed Jul 3, 2023
1 parent 4552be7 commit 9dc7aab
Show file tree
Hide file tree
Showing 7 changed files with 74 additions and 66 deletions.
2 changes: 1 addition & 1 deletion cdm-test/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ tasks.register("testIndexCreation", Test) {
}

test {
// dependsOn 'testIndexCreation'
dependsOn 'testIndexCreation'

// In addition to preventing TestGribIndexCreation from running during cdm-test:test,
// this statement also excludes the results of TestGribIndexCreation from appearing in the cdm-test report:
Expand Down
84 changes: 42 additions & 42 deletions cdm-test/src/test/java/ucar/nc2/ft/point/TestPointDatasets.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,47 +108,47 @@ private static List<Object[]> getCFDatasets() {
private static List<Object[]> getPlugDatasets() {
List<Object[]> result = new ArrayList<>();

// // cosmic
// result
// .add(new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/cosmic/wetPrf_C005.2007.294.16.22.G17_0001.0002_nc",
// FeatureType.TRAJECTORY, 383});
// // ndbc
// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/ndbc/41001h1976.nc", FeatureType.STATION, 1405});
// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/suomi/suoHWV_2006.105.00.00.0060_nc",
// FeatureType.STATION, 124});
// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/suomi/gsuPWV_2006.105.00.00.1440_nc",
// FeatureType.STATION, 4848});
// // fsl wind profilers
// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_RASS_01hr_20091027_1500.nc",
// FeatureType.STATION_PROFILE, 198});
// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_RASS_06min_20091028_2318.nc",
// FeatureType.STATION_PROFILE, 198});
// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_wind_01hr_20091024_1200.nc",
// FeatureType.STATION_PROFILE, 1728});
// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_wind_06min_20091030_2330.nc",
// FeatureType.STATION_PROFILE, 2088});
// // netcdf buoy / synoptic / metars ( robb's perl decoder output)
// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_METAR_latest.nc", FeatureType.POINT, 7});
// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_Buoy_20090921_0000.nc",
// FeatureType.POINT, 32452});
// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_Synoptic_20090921_0000.nc",
// FeatureType.POINT, 1516});
// // RAF-Nimbus
// result.add(
// new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/aircraft/135_ordrd.nc", FeatureType.TRAJECTORY, 7741});
// result.add(
// new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/aircraft/raftrack.nc", FeatureType.TRAJECTORY, 8157});
// // Madis
// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/acars/acars_20091109_0800.nc",
// FeatureType.TRAJECTORY, 5063});
// cosmic
result
.add(new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/cosmic/wetPrf_C005.2007.294.16.22.G17_0001.0002_nc",
FeatureType.TRAJECTORY, 383});
// ndbc
result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/ndbc/41001h1976.nc", FeatureType.STATION, 1405});
result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/suomi/suoHWV_2006.105.00.00.0060_nc",
FeatureType.STATION, 124});
result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/suomi/gsuPWV_2006.105.00.00.1440_nc",
FeatureType.STATION, 4848});
// fsl wind profilers
result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_RASS_01hr_20091027_1500.nc",
FeatureType.STATION_PROFILE, 198});
result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_RASS_06min_20091028_2318.nc",
FeatureType.STATION_PROFILE, 198});
result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_wind_01hr_20091024_1200.nc",
FeatureType.STATION_PROFILE, 1728});
result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/stationProfile/PROFILER_wind_06min_20091030_2330.nc",
FeatureType.STATION_PROFILE, 2088});
// netcdf buoy / synoptic / metars ( robb's perl decoder output)
result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_METAR_latest.nc", FeatureType.POINT, 7});
result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_Buoy_20090921_0000.nc",
FeatureType.POINT, 32452});
result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/Surface_Synoptic_20090921_0000.nc",
FeatureType.POINT, 1516});
// RAF-Nimbus
result.add(
new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/aircraft/135_ordrd.nc", FeatureType.TRAJECTORY, 7741});
result.add(
new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/aircraft/raftrack.nc", FeatureType.TRAJECTORY, 8157});
// Madis
result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/trajectory/acars/acars_20091109_0800.nc",
FeatureType.TRAJECTORY, 5063});
result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/19981110_1200", FeatureType.POINT, 2499});
// result.add(
// new Object[] {TestDir.cdmUnitTestDir + "ft/station/madis2/hydro/20050729_1200", FeatureType.STATION, 1374});
// result.add(
// new Object[] {TestDir.cdmUnitTestDir + "ft/sounding/netcdf/20070612_1200", FeatureType.STATION_PROFILE, 1788});
// // unidata point obs
// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/200501q3h-gr.nc", FeatureType.STATION, 5023});
// result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/20080814_LMA.ncml", FeatureType.POINT, 277477});
result.add(
new Object[] {TestDir.cdmUnitTestDir + "ft/station/madis2/hydro/20050729_1200", FeatureType.STATION, 1374});
result.add(
new Object[] {TestDir.cdmUnitTestDir + "ft/sounding/netcdf/20070612_1200", FeatureType.STATION_PROFILE, 1788});
// unidata point obs
result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/station/200501q3h-gr.nc", FeatureType.STATION, 5023});
result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/point/netcdf/20080814_LMA.ncml", FeatureType.POINT, 277477});

// FslRaob
// assert 63 == checkPointDataset(TestDir.testdataDir + "sounding/netcdf/raob_soundings20216.cdf",
Expand All @@ -174,9 +174,9 @@ private static List<Object[]> getMiscDatasets() {
public static List<Object[]> getTestParameters() {
List<Object[]> result = new ArrayList<>();

// result.addAll(getCFDatasets());
result.addAll(getCFDatasets());
result.addAll(getPlugDatasets());
// result.addAll(getMiscDatasets());
result.addAll(getMiscDatasets());

return result;
}
Expand Down
2 changes: 1 addition & 1 deletion cdm/core/src/main/java/ucar/nc2/dataset/EnhanceScaleMissingUnsigned.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
*
* <ol>
* <li>If scale_factor and/or add_offset variable attributes are present, then this is a "packed" Variable.</li>
* <li>The data type of the variable will be set to the {@link EnhanceScaleMissingUnsignedImpl#largestOf largest of}:
* <li>The data type of the variable will be set to the {@link ucar.nc2.filter.FilterHelpers#largestOf largest of}:
* <ul>
* <li>the original data type</li>
* <li>the unsigned conversion type, if applicable</li>
Expand Down
14 changes: 7 additions & 7 deletions cdm/core/src/main/java/ucar/nc2/dataset/VariableDS.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -628,20 +628,20 @@ public double getOffset() {

@Override
public boolean hasMissing() {
return convertMissing != null ? convertMissing.hasMissing() : false;
return convertMissing != null && convertMissing.hasMissing();
}

@Override
public boolean isMissing(double val) {
if (Double.isNaN(val)) {
return true;
}
return convertMissing != null ? convertMissing.isMissing(val) : false;
return convertMissing != null && convertMissing.isMissing(val);
}

@Override
public boolean hasValidData() {
return convertMissing != null ? convertMissing.hasMissingValue() : false;
return convertMissing != null && convertMissing.hasMissingValue();
}

@Override
Expand All @@ -656,7 +656,7 @@ public double getValidMax() {

@Override
public boolean isInvalidData(double val) {
return convertMissing != null ? convertMissing.isInvalidData(val) : false;
return convertMissing != null && convertMissing.isInvalidData(val);
}

@Override
Expand All @@ -671,12 +671,12 @@ public double getFillValue() {

@Override
public boolean isFillValue(double val) {
return convertMissing != null ? convertMissing.isFillValue(val) : false;
return convertMissing != null && convertMissing.isFillValue(val);
}

@Override
public boolean hasMissingValue() {
return convertMissing != null ? convertMissing.hasMissingValue() : false;
return convertMissing != null && convertMissing.hasMissingValue();
}

@Override
Expand All @@ -686,7 +686,7 @@ public double[] getMissingValues() {

@Override
public boolean isMissingValue(double val) {
return convertMissing != null ? convertMissing.isMissingValue(val) : false;
return convertMissing != null && convertMissing.isMissingValue(val);
}

/** @deprecated Use NetcdfDataset.builder() */
Expand Down
3 changes: 1 addition & 2 deletions cdm/core/src/main/java/ucar/nc2/filter/ScaleOffset.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,6 @@ public byte[] decode(byte[] dataIn) {
return FilterHelpers.arrayToBytes(out, dtype, dtypeOrder);
}

// not used anywhere yet
public Array applyScaleOffset(Array in) {
if (scale == DEFAULT_SCALE && offset == DEFAULT_OFFSET) {
return in;
Expand Down Expand Up @@ -251,7 +250,7 @@ private static ByteOrder parseByteOrder(String dtype, ByteOrder defaultOrder) {

public double applyScaleOffset(Number value) {
double convertedValue = value.doubleValue();
return Math.round((convertedValue - offset) * scale);
return (convertedValue - offset) * scale;
}

public double removeScaleOffset(Number value) {
Expand Down
23 changes: 16 additions & 7 deletions cdm/core/src/test/java/ucar/nc2/filter/TestEnhancements.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,12 @@ public class TestEnhancements {

@BeforeClass
public static void setUp() throws IOException, InvalidRangeException {
final int data_len = 10;
final int dataLen = 10;
String filePath = tempFolder.newFile().getAbsolutePath();
NetcdfFormatWriter.Builder builder = NetcdfFormatWriter.createNewNetcdf3(filePath);
builder.addDimension("dim", data_len);
builder.addDimension("dim", dataLen);

Array signedData = Array.factory(DataType.SHORT, new int[] {data_len}, signedShorts);
Array signedData = Array.factory(DataType.SHORT, new int[] {dataLen}, signedShorts);
// signed shorts
builder.addVariable("signedVar", DataType.SHORT, "dim");
// unsigned shorts
Expand All @@ -51,8 +51,10 @@ public static void setUp() throws IOException, InvalidRangeException {
// scaled and offset data
builder.addVariable("scaleOffsetVar", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.SCALE_FACTOR, 10))
.addAttribute(new Attribute(CDM.ADD_OFFSET, 10));
// scaled data no offset
builder.addVariable("scaleNoOffsetVar", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.SCALE_FACTOR, 10));

Array missingDataArray = Array.factory(DataType.FLOAT, new int[] {data_len}, missingData);
Array missingDataArray = Array.factory(DataType.FLOAT, new int[] {dataLen}, missingData);
// Data with min
builder.addVariable("validMin", DataType.FLOAT, "dim").addAttribute(new Attribute(CDM.VALID_MIN, VALID_MIN));
// Data with min and max
Expand All @@ -65,7 +67,7 @@ public static void setUp() throws IOException, InvalidRangeException {
.addAttribute(Attribute.builder(CDM.FILL_VALUE).setNumericValue(FILL_VALUE, true).build());

// unsigned, scaled/offset, and missing value
Array enhanceAllArray = Array.factory(DataType.SHORT, new int[] {data_len}, signedShorts);
Array enhanceAllArray = Array.factory(DataType.SHORT, new int[] {dataLen}, signedShorts);
builder.addVariable("enhanceAll", DataType.SHORT, "dim").addAttribute(new Attribute(CDM.UNSIGNED, "true"))
.addAttribute(new Attribute(CDM.SCALE_FACTOR, 10.0)).addAttribute(new Attribute(CDM.ADD_OFFSET, 10))
.addAttribute(new Attribute(CDM.VALID_MAX, SIGNED_SCALED_MAX))
Expand All @@ -76,6 +78,7 @@ public static void setUp() throws IOException, InvalidRangeException {
writer.write(writer.findVariable("signedVar"), new int[1], signedData);
writer.write(writer.findVariable("unsignedVar"), new int[1], signedData);
writer.write(writer.findVariable("scaleOffsetVar"), new int[1], signedData);
writer.write(writer.findVariable("scaleNoOffsetVar"), new int[1], signedData);
writer.write(writer.findVariable("validMin"), new int[1], missingDataArray);
writer.write(writer.findVariable("validMinMax"), new int[1], missingDataArray);
writer.write(writer.findVariable("validRange"), new int[1], missingDataArray);
Expand Down Expand Up @@ -104,13 +107,19 @@ public void testUnsignedConversion() throws IOException {

@Test
public void testScaleOffset() throws IOException {
final int[] expected = new int[] {-40, -30, -20, -10, 0, 10, 20, 30, 40, 50};
// signed var
int[] expected = new int[] {-40, -30, -20, -10, 0, 10, 20, 30, 40, 50};
Variable v = ncd.findVariable("scaleOffsetVar");
Array data = v.read();
assertThat(data.isUnsigned()).isFalse();
assertThat(data.getDataType()).isEqualTo(DataType.INT);
assertThat((int[]) data.copyTo1DJavaArray()).isEqualTo(expected);

expected = new int[] {-50, -40, -30, -20, -10, 0, 10, 20, 30, 40};
v = ncd.findVariable("scaleNoOffsetVar");
data = v.read();
assertThat(data.isUnsigned()).isFalse();
assertThat(data.getDataType()).isEqualTo(DataType.INT);
assertThat((int[]) data.copyTo1DJavaArray()).isEqualTo(expected);
}

@Test
Expand Down
12 changes: 6 additions & 6 deletions cdm/core/src/test/java/ucar/nc2/filter/TestFilters.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,13 +98,13 @@ public void testScaleOffset() throws IOException {
assertThat(decoded).isEqualTo(input);

// test empty props
Map<String, Object> props2 = new HashMap<>();
props2.put("id", "fixedscaleoffset");
props2.put("dtype", "<i4");
Filter filter2 = new ScaleOffset(props2);
byte[] out2 = filter2.encode(decoded_data);
Map<String, Object> emptyProps = new HashMap<>();
emptyProps.put("id", "fixedscaleoffset");
emptyProps.put("dtype", "<i4");
Filter emptyFilter = new ScaleOffset(emptyProps);
byte[] out2 = emptyFilter.encode(decoded_data);
assertThat(out2).isEqualTo(decoded_data);
byte[] out3 = filter2.decode(decoded_data);
byte[] out3 = emptyFilter.decode(decoded_data);
assertThat(out3).isEqualTo(decoded_data);
}

Expand Down

0 comments on commit 9dc7aab

Please sign in to comment.