Skip to content

Commit

Permalink
Fix geotiff support for unsigned/signed data
Browse files Browse the repository at this point in the history 
* Expand unit tests for geotiff writing
* Reduce code duplication
  • Loading branch information
@WeatherGod @tdrwenski
WeatherGod authored and tdrwenski committed Jul 1, 2024
1 parent cb58717 commit eacea20
Show file tree
Hide file tree
Showing 4 changed files with 365 additions and 32 deletions.
25 changes: 25 additions & 0 deletions cdm/misc/src/main/java/ucar/nc2/geotiff/GeoTiff.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import java.nio.ByteOrder;
import java.nio.FloatBuffer;
import java.nio.IntBuffer;
import java.nio.ShortBuffer;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
Expand Down Expand Up @@ -192,6 +193,30 @@ int writeData(byte[] data, int imageNumber) throws IOException {
return nextOverflowData;
}

int writeData(short[] data, int imageNumber) throws IOException {
if (file == null)
init();

if (imageNumber == 1)
channel.position(headerSize);
else
channel.position(nextOverflowData);

// no way around making a copy
ByteBuffer direct = ByteBuffer.allocateDirect(2 * data.length);
ShortBuffer buffer = direct.asShortBuffer();
buffer.put(data);
// buffer.flip();
channel.write(direct);

if (imageNumber == 1)
firstIFD = headerSize + 2 * data.length;
else
firstIFD = 2 * data.length + nextOverflowData;

return nextOverflowData;
}

int writeData(int[] data, int imageNumber) throws IOException {
if (file == null)
init();
Expand Down
132 changes: 104 additions & 28 deletions cdm/misc/src/main/java/ucar/nc2/geotiff/GeotiffWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import ucar.ma2.ArrayByte;
import ucar.ma2.ArrayInt;
import ucar.ma2.ArrayFloat;
import ucar.ma2.ArrayShort;
import ucar.ma2.DataType;
import ucar.ma2.Index;
import ucar.ma2.IndexIterator;
Expand Down Expand Up @@ -211,19 +212,15 @@ void writeGrid(GridDatatype grid, Array data, boolean greyScale, double xStart,
// write the data first
MAMath.MinMax dataMinMax = grid.getMinMaxSkipMissingData(data);
if (greyScale) {
ArrayByte result = replaceMissingValuesAndScale(grid, data, dataMinMax);
nextStart = geotiff.writeData((byte[]) result.getStorage(), imageNumber);
data = replaceMissingValuesAndScale(grid, data, dataMinMax);
nextStart = writeData(data, DataType.UBYTE);
} else if (dtype == DataType.FLOAT) {
// Backwards compatibility shim
data = replaceMissingValues(grid, data, dataMinMax);
nextStart = writeData(data, dtype);
} else {
if (dtype == DataType.BYTE || dtype == DataType.UBYTE) {
ArrayByte result = coerceByte(data);
nextStart = geotiff.writeData((byte[]) result.getStorage(), imageNumber);
} else if (dtype.isIntegral()) {
ArrayInt result = coerceInt(data);
nextStart = geotiff.writeData((int[]) result.getStorage(), imageNumber);
} else {
ArrayFloat result = replaceMissingValues(grid, data, dataMinMax);
nextStart = geotiff.writeData((float[]) result.getStorage(), imageNumber);
}
data = coerceData(data, dtype);
nextStart = writeData(data, dtype);
}

// set the width and the height
Expand Down Expand Up @@ -472,8 +469,16 @@ public static HashMap<Integer, Color> createColorMap(int[] flag_values, String[]
return colorMap;
}

private ArrayByte coerceByte(Array data) {
ArrayByte array = (ArrayByte) Array.factory(DataType.BYTE, data.getShape());
/**
* Coerce a given data array into an array of bytes.
* Always returns a copy. No data safety check is performed.
*
* @param data input data array (of any data type)
* @param isUnsigned coerce to unsigned bytes
* @return integer data array
*/
static ArrayByte coerceByte(Array data, boolean isUnsigned) {
ArrayByte array = (ArrayByte) Array.factory(isUnsigned ? DataType.UBYTE : DataType.BYTE, data.getShape());
IndexIterator dataIter = data.getIndexIterator();
IndexIterator resultIter = array.getIndexIterator();

Expand All @@ -484,8 +489,56 @@ private ArrayByte coerceByte(Array data) {
return array;
}

private ArrayInt coerceInt(Array data) {
ArrayInt array = (ArrayInt) Array.factory(DataType.INT, data.getShape());
/**
* Coerce a given data array into an array of 16-bit integers.
* Always returns a copy. No data safety check is performed.
*
* @param data input data array (of any data type)
* @param isUnsigned coerce to unsigned integers
* @return integer data array
*/
static ArrayShort coerceShort(Array data, boolean isUnsigned) {
ArrayShort array = (ArrayShort) Array.factory(isUnsigned ? DataType.USHORT : DataType.SHORT, data.getShape());
IndexIterator dataIter = data.getIndexIterator();
IndexIterator resultIter = array.getIndexIterator();

while (dataIter.hasNext()) {
resultIter.setIntNext(dataIter.getIntNext());
}

return array;
}


/**
* Coerce a given data array into an array of 32-bit integers.
* Always returns a copy. No data safety check is performed.
*
* @param data input data array (of any data type)
* @param isUnsigned coerce to unsigned integers
* @return integer data array
*/
static ArrayInt coerceInt(Array data, boolean isUnsigned) {
ArrayInt array = (ArrayInt) Array.factory(isUnsigned ? DataType.UINT : DataType.INT, data.getShape());
IndexIterator dataIter = data.getIndexIterator();
IndexIterator resultIter = array.getIndexIterator();

while (dataIter.hasNext()) {
resultIter.setIntNext(dataIter.getIntNext());
}

return array;
}

/**
* Coerce a given data array into an array of 32-bit floats.
* Always returns a copy. No data safety check is performed.
*
* @param data input data array (of any data type)
* @return float data array
*/
static ArrayFloat coerceFloat(Array data) {
ArrayFloat array = (ArrayFloat) Array.factory(DataType.FLOAT, data.getShape());
IndexIterator dataIter = data.getIndexIterator();
IndexIterator resultIter = array.getIndexIterator();

Expand Down Expand Up @@ -845,19 +898,15 @@ public void writeGrid(GeoReferencedArray array, boolean greyScale, DataType dtyp
int nextStart;
MAMath.MinMax dataMinMax = MAMath.getMinMaxSkipMissingData(data, array);
if (greyScale) {
ArrayByte result = replaceMissingValuesAndScale(array, data, dataMinMax);
nextStart = geotiff.writeData((byte[]) result.getStorage(), pageNumber);
data = replaceMissingValuesAndScale(array, data, dataMinMax);
nextStart = writeData(data, DataType.UBYTE);
} else if (dtype == DataType.FLOAT) {
// Backwards compatibility shim
data = replaceMissingValues(array, data, dataMinMax);
nextStart = writeData(data, dtype);
} else {
if (dtype == DataType.BYTE || dtype == DataType.UBYTE) {
ArrayByte result = coerceByte(data);
nextStart = geotiff.writeData((byte[]) result.getStorage(), pageNumber);
} else if (dtype.isIntegral()) {
ArrayInt result = coerceInt(data);
nextStart = geotiff.writeData((int[]) result.getStorage(), pageNumber);
} else {
ArrayFloat result = replaceMissingValues(array, data, dataMinMax);
nextStart = geotiff.writeData((float[]) result.getStorage(), pageNumber);
}
data = coerceData(data, dtype);
nextStart = writeData(data, dtype);
}

// set the width and the height
Expand All @@ -867,5 +916,32 @@ public void writeGrid(GeoReferencedArray array, boolean greyScale, DataType dtyp
writeMetadata(greyScale, xStart, yStart, xInc, yInc, height, width, pageNumber, nextStart, dataMinMax, proj, dtype);
pageNumber++;
}

static Array coerceData(Array data, DataType dtype) {
if (dtype == DataType.BYTE || dtype == DataType.UBYTE) {
data = coerceByte(data, dtype.isUnsigned());
} else if (dtype == DataType.SHORT || dtype == DataType.USHORT) {
data = coerceShort(data, dtype.isUnsigned());
} else if (dtype == DataType.INT || dtype == DataType.UINT) {
data = coerceInt(data, dtype.isUnsigned());
} else if (dtype.isFloatingPoint()) {
data = coerceFloat(data);
}
return data;
}

private int writeData(Array data, DataType dtype) throws IOException {
int nextStart;
if (dtype == DataType.BYTE || dtype == DataType.UBYTE) {
nextStart = geotiff.writeData((byte[]) data.getStorage(), pageNumber);
} else if (dtype == DataType.SHORT || dtype == DataType.USHORT) {
nextStart = geotiff.writeData((short[]) data.getStorage(), pageNumber);
} else if (dtype == DataType.INT || dtype == DataType.UINT) {
nextStart = geotiff.writeData((int[]) data.getStorage(), pageNumber);
} else {
nextStart = geotiff.writeData((float[]) data.getStorage(), pageNumber);
}
return nextStart;
}
}

31 changes: 27 additions & 4 deletions cdm/misc/src/test/java/ucar/nc2/geotiff/TestGeoTiffWriter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
import java.util.List;

/**
* GeoTiffWriter2 writing geotiffs
* GeotiffWriter writing geotiffs
*
* @author caron
* @since 7/31/2014
Expand Down Expand Up @@ -65,10 +65,12 @@ public static List<Object[]> getTestParameters() {
result.add(new Object[] {TestDir.cdmUnitTestDir + "ft/coverage/testCFwriter.nc", FeatureType.GRID, "Temperature",
greyscale == 1});

// This file is unique in that it is lambert conformal with yaxis flipped.
// This file is unique in that it is lambert conformal with yaxis flipped (South to North).
// Also, this drought data is stored as integers (the other test data above are floats).
// This will check that the compatibility shim is still forcing the data
// to floats when greyscale is false and not specifying the dtype.
result.add(
new Object[] {"src/test/data/ucar/nc2/geotiff/categorical.nc", FeatureType.GRID, "drought", greyscale == 1});

}

return result;
Expand All @@ -93,6 +95,15 @@ public void testWriteCoverage() throws IOException, InvalidRangeException {
Array dtArray;
float missingVal;
MAMath.MinMax minmax;

// When not explicitly specifying the dtype in the writeGrid() call,
// the code will fall back to old behavior and cast the data as floats
// when greyscale is false. greyscale==true always produces unsigned bytes.
FieldType expectedFieldType = greyscale ? FieldType.BYTE : FieldType.FLOAT;

// Float, signed, unsigned (future-proofing for when we expand expected field types)
int expectedSampleFormat = expectedFieldType.code >= 6 ? expectedFieldType.code == 11 ? 3 : 2 : 1;

try (GridDataset gds = GridDataset.open(filename)) {
GridDatatype grid = gds.findGridByName(field);
assert grid != null;
Expand Down Expand Up @@ -143,8 +154,16 @@ public void testWriteCoverage() throws IOException, InvalidRangeException {
Assert.assertEquals(minmax.max, (float) sMaxTag.valueD[0], 0.0);

IFDEntry noDataTag = geotiff.findTag(Tag.GDALNoData);
// Technically, it would be ambiguous what was intended due to the
// backwards compatibility shim, so FLOATS still get NoData tags.
Assert.assertNotNull(noDataTag);
Assert.assertEquals(String.valueOf(missingVal), noDataTag.valueS);

// Right now, this is the only case where the SampleFormat tag is written.
// There is no reason why this has to be the case.
IFDEntry sTypeTag = geotiff.findTag(Tag.SampleFormat);
Assert.assertNotNull(sTypeTag);
Assert.assertEquals(expectedSampleFormat, sTypeTag.value[0]);
} else {
IFDEntry sMinTag = geotiff.findTag(Tag.SMinSampleValue);
Assert.assertNull(sMinTag);
Expand All @@ -156,6 +175,10 @@ public void testWriteCoverage() throws IOException, InvalidRangeException {
Assert.assertNull(noDataTag);
}

IFDEntry sSizeTag = geotiff.findTag(Tag.BitsPerSample);
Assert.assertNotNull(sSizeTag);
Assert.assertEquals(expectedFieldType.size * 8, sSizeTag.value[0]);

String gridOut2 = tempFolder.newFile().getAbsolutePath();
logger.debug("geotiff2 read coverage {} write {} greyscale {}", filename, gridOut2, greyscale);

Expand All @@ -169,11 +192,11 @@ public void testWriteCoverage() throws IOException, InvalidRangeException {
String covName = (pos > 0) ? field.substring(pos + 1) : field;

Coverage coverage = gcd.findCoverage(covName);
Assert.assertNotNull(covName, coverage);
CoverageCoordAxis1D z = (CoverageCoordAxis1D) coverage.getCoordSys().getZAxis();
SubsetParams params = new SubsetParams().set(SubsetParams.timePresent, true);
if (z != null)
params.set(SubsetParams.vertCoord, z.getCoordMidpoint(0));
Assert.assertNotNull(covName, coverage);
covArray = coverage.readData(params);

try (GeotiffWriter writer = new GeotiffWriter(gridOut2)) {
Expand Down
Loading

0 comments on commit eacea20

Please sign in to comment.