From 37af37b6b013040868696ec526e8f1f3e750f164 Mon Sep 17 00:00:00 2001 From: Rich Fecher Date: Thu, 21 Jul 2016 17:30:11 -0400 Subject: [PATCH] move GDAL GeoTiff reader to adapter so that it can be used for commandline ingest --- extensions/adapters/raster/pom.xml | 27 +++- .../raster/plugin}/GDALGeoTiffFormat.java | 2 +- .../plugin/GDALGeoTiffFormatFactory.java | 45 ++++++ .../raster/plugin}/GDALGeoTiffReader.java | 2 +- ...ools.coverage.grid.io.GridFormatFactorySpi | 1 + .../cli/geoserver/GeoServerConfig.java | 2 +- .../format/landsat8/RasterIngestRunner.java | 1 + .../GeoToolsRasterDataStoreIngestPlugin.java | 152 ++++++++++++++---- .../geotools/raster/RasterOptionProvider.java | 11 +- 9 files changed, 209 insertions(+), 34 deletions(-) rename extensions/{cli/landsat8/src/main/java/mil/nga/giat/geowave/format/landsat8 => adapters/raster/src/main/java/mil/nga/giat/geowave/adapter/raster/plugin}/GDALGeoTiffFormat.java (97%) create mode 100644 extensions/adapters/raster/src/main/java/mil/nga/giat/geowave/adapter/raster/plugin/GDALGeoTiffFormatFactory.java rename extensions/{cli/landsat8/src/main/java/mil/nga/giat/geowave/format/landsat8 => adapters/raster/src/main/java/mil/nga/giat/geowave/adapter/raster/plugin}/GDALGeoTiffReader.java (96%) diff --git a/extensions/adapters/raster/pom.xml b/extensions/adapters/raster/pom.xml index 4c4237ee1f9..e9920f988c7 100644 --- a/extensions/adapters/raster/pom.xml +++ b/extensions/adapters/raster/pom.xml @@ -9,7 +9,32 @@ geowave-adapter-raster Geowave Raster Adapter Geowave Data Adapter for Raster Data - + + it.geosolutions.imageio-ext + imageio-ext-gdalgeotiff + 1.1.13 + + + com.vividsolutions + jts + + + + + org.geotools + gt-imageio-ext-gdal + ${geotools.version} + + + com.vividsolutions + jts + + + gt-epsg-hsql + org.geotools + + + com.google.guava guava diff --git a/extensions/cli/landsat8/src/main/java/mil/nga/giat/geowave/format/landsat8/GDALGeoTiffFormat.java b/extensions/adapters/raster/src/main/java/mil/nga/giat/geowave/adapter/raster/plugin/GDALGeoTiffFormat.java similarity index 97% rename from extensions/cli/landsat8/src/main/java/mil/nga/giat/geowave/format/landsat8/GDALGeoTiffFormat.java rename to extensions/adapters/raster/src/main/java/mil/nga/giat/geowave/adapter/raster/plugin/GDALGeoTiffFormat.java index 20d56bc9c29..f2597755270 100644 --- a/extensions/cli/landsat8/src/main/java/mil/nga/giat/geowave/format/landsat8/GDALGeoTiffFormat.java +++ b/extensions/adapters/raster/src/main/java/mil/nga/giat/geowave/adapter/raster/plugin/GDALGeoTiffFormat.java @@ -1,4 +1,4 @@ -package mil.nga.giat.geowave.format.landsat8; +package mil.nga.giat.geowave.adapter.raster.plugin; import java.util.Collections; import java.util.HashMap; diff --git a/extensions/adapters/raster/src/main/java/mil/nga/giat/geowave/adapter/raster/plugin/GDALGeoTiffFormatFactory.java b/extensions/adapters/raster/src/main/java/mil/nga/giat/geowave/adapter/raster/plugin/GDALGeoTiffFormatFactory.java new file mode 100644 index 00000000000..1dea26f734f --- /dev/null +++ b/extensions/adapters/raster/src/main/java/mil/nga/giat/geowave/adapter/raster/plugin/GDALGeoTiffFormatFactory.java @@ -0,0 +1,45 @@ +package mil.nga.giat.geowave.adapter.raster.plugin; + +import java.awt.RenderingHints.Key; +import java.util.Map; + +import org.geotools.coverage.grid.io.AbstractGridFormat; +import org.geotools.coverageio.BaseGridFormatFactorySPI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import it.geosolutions.imageio.plugins.geotiff.GeoTiffImageReaderSpi; + +public class GDALGeoTiffFormatFactory extends + BaseGridFormatFactorySPI +{ + private final static Logger LOGGER = LoggerFactory.getLogger(GDALGeoTiffFormatFactory.class); + + @Override + public boolean isAvailable() { + boolean available = true; + + // if these classes are here, then the runtime environment has + // access to JAI and the JAI ImageI/O toolbox. + try { + Class.forName("it.geosolutions.imageio.plugins.geotiff.GeoTiffImageReaderSpi"); + available = new GeoTiffImageReaderSpi().isAvailable(); + + } + catch (final ClassNotFoundException cnf) { + if (LOGGER.isDebugEnabled()) { + LOGGER.debug("GDALGeoTiffFormatFactory is not availaible."); + } + + available = false; + } + + return available; + } + + @Override + public AbstractGridFormat createFormat() { + return new GDALGeoTiffFormat(); + } + +} diff --git a/extensions/cli/landsat8/src/main/java/mil/nga/giat/geowave/format/landsat8/GDALGeoTiffReader.java b/extensions/adapters/raster/src/main/java/mil/nga/giat/geowave/adapter/raster/plugin/GDALGeoTiffReader.java similarity index 96% rename from extensions/cli/landsat8/src/main/java/mil/nga/giat/geowave/format/landsat8/GDALGeoTiffReader.java rename to extensions/adapters/raster/src/main/java/mil/nga/giat/geowave/adapter/raster/plugin/GDALGeoTiffReader.java index 2b1fe5f2263..8b91af7f2d3 100644 --- a/extensions/cli/landsat8/src/main/java/mil/nga/giat/geowave/format/landsat8/GDALGeoTiffReader.java +++ b/extensions/adapters/raster/src/main/java/mil/nga/giat/geowave/adapter/raster/plugin/GDALGeoTiffReader.java @@ -1,4 +1,4 @@ -package mil.nga.giat.geowave.format.landsat8; +package mil.nga.giat.geowave.adapter.raster.plugin; import org.geotools.coverageio.gdal.BaseGDALGridCoverage2DReader; import org.geotools.data.DataSourceException; diff --git a/extensions/adapters/raster/src/main/resources/META-INF/services/org.geotools.coverage.grid.io.GridFormatFactorySpi b/extensions/adapters/raster/src/main/resources/META-INF/services/org.geotools.coverage.grid.io.GridFormatFactorySpi index 647ced18ed2..884b4de5733 100644 --- a/extensions/adapters/raster/src/main/resources/META-INF/services/org.geotools.coverage.grid.io.GridFormatFactorySpi +++ b/extensions/adapters/raster/src/main/resources/META-INF/services/org.geotools.coverage.grid.io.GridFormatFactorySpi @@ -1 +1,2 @@ mil.nga.giat.geowave.adapter.raster.plugin.GeoWaveGTRasterFormatFactory +mil.nga.giat.geowave.adapter.raster.plugin.GDALGeoTiffFormatFactory diff --git a/extensions/cli/geoserver/src/main/java/mil/nga/giat/geowave/cli/geoserver/GeoServerConfig.java b/extensions/cli/geoserver/src/main/java/mil/nga/giat/geowave/cli/geoserver/GeoServerConfig.java index 067d9d97653..60668e48424 100644 --- a/extensions/cli/geoserver/src/main/java/mil/nga/giat/geowave/cli/geoserver/GeoServerConfig.java +++ b/extensions/cli/geoserver/src/main/java/mil/nga/giat/geowave/cli/geoserver/GeoServerConfig.java @@ -104,7 +104,7 @@ public GeoServerConfig() { } public String getUrl() { - if (!url.contains("//")) { + if (url.contains("//")) { // assume exact URL return url; } diff --git a/extensions/cli/landsat8/src/main/java/mil/nga/giat/geowave/format/landsat8/RasterIngestRunner.java b/extensions/cli/landsat8/src/main/java/mil/nga/giat/geowave/format/landsat8/RasterIngestRunner.java index 381bea48884..09a7673e35f 100644 --- a/extensions/cli/landsat8/src/main/java/mil/nga/giat/geowave/format/landsat8/RasterIngestRunner.java +++ b/extensions/cli/landsat8/src/main/java/mil/nga/giat/geowave/format/landsat8/RasterIngestRunner.java @@ -45,6 +45,7 @@ import mil.nga.giat.geowave.adapter.raster.RasterUtils; import mil.nga.giat.geowave.adapter.raster.adapter.RasterDataAdapter; import mil.nga.giat.geowave.adapter.raster.adapter.merge.nodata.NoDataMergeStrategy; +import mil.nga.giat.geowave.adapter.raster.plugin.GDALGeoTiffReader; import mil.nga.giat.geowave.adapter.raster.plugin.GeoWaveGTRasterFormat; import mil.nga.giat.geowave.adapter.vector.plugin.ExtractGeometryFilterVisitor; import mil.nga.giat.geowave.adapter.vector.plugin.GeoWaveGTDataStore; diff --git a/extensions/formats/geotools-raster/src/main/java/mil/nga/giat/geowave/format/geotools/raster/GeoToolsRasterDataStoreIngestPlugin.java b/extensions/formats/geotools-raster/src/main/java/mil/nga/giat/geowave/format/geotools/raster/GeoToolsRasterDataStoreIngestPlugin.java index 629078de390..86b1e0eb0a7 100644 --- a/extensions/formats/geotools-raster/src/main/java/mil/nga/giat/geowave/format/geotools/raster/GeoToolsRasterDataStoreIngestPlugin.java +++ b/extensions/formats/geotools-raster/src/main/java/mil/nga/giat/geowave/format/geotools/raster/GeoToolsRasterDataStoreIngestPlugin.java @@ -6,15 +6,20 @@ import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.log4j.Logger; import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.coverage.grid.io.AbstractGridFormat; import org.geotools.coverage.grid.io.GridCoverage2DReader; import org.geotools.coverage.grid.io.GridFormatFinder; +import org.geotools.factory.Hints; +import org.geotools.referencing.CRS; import org.opengis.coverage.grid.GridCoverage; +import org.opengis.referencing.crs.CoordinateReferenceSystem; import mil.nga.giat.geowave.adapter.raster.RasterUtils; import mil.nga.giat.geowave.adapter.raster.adapter.RasterDataAdapter; @@ -38,7 +43,8 @@ public class GeoToolsRasterDataStoreIngestPlugin implements LocalFileIngestPlugin { - private final static Logger LOGGER = Logger.getLogger(GeoToolsRasterDataStoreIngestPlugin.class); + private final static Logger LOGGER = Logger.getLogger( + GeoToolsRasterDataStoreIngestPlugin.class); private final RasterOptionProvider optionProvider; public GeoToolsRasterDataStoreIngestPlugin() { @@ -63,48 +69,134 @@ public void init( @Override public boolean supportsFile( final File file ) { - - final AbstractGridFormat format = GridFormatFinder.findFormat(file); + AbstractGridFormat format = null; + try { + format = GridFormatFinder.findFormat( + file); + } + catch (final Exception e) { + LOGGER.info( + "Unable to support as raster file", + e); + } // the null check is enough and we don't need to check the format // accepts this file because the finder should have previously validated // this return (format != null); } + private static AbstractGridFormat prioritizedFindFormat( + final File input ) { + final AbstractGridFormat format = null; + try { + final Set formats = GridFormatFinder.findFormats( + input); + if ((formats == null) || formats.isEmpty()) { + LOGGER.warn( + "Unable to support raster file " + input.getAbsolutePath()); + return null; + } + // world image and geotiff can both open tif files, give + // priority to gdalgeotiff, followed by geotiff + for (final AbstractGridFormat f : formats) { + if ("GDALGeoTiff".equals( + f.getName())) { + return f; + } + } + for (final AbstractGridFormat f : formats) { + if ("GeoTIFF".equals( + f.getName())) { + return f; + } + } + + // otherwise just pick the first + final Iterator it = formats.iterator(); + if (it.hasNext()) { + return it.next(); + } + } + catch (final Exception e) { + LOGGER.warn( + "Error while trying read raster file", + e); + return null; + } + return format; + } + @Override public CloseableIterator> toGeoWaveData( final File input, final Collection primaryIndexIds, final String globalVisibility ) { - - final AbstractGridFormat format = GridFormatFinder.findFormat(input); - final GridCoverage2DReader reader = format.getReader(input); + final AbstractGridFormat format = prioritizedFindFormat( + input); + if (format == null) { + return new Wrapper( + Collections.emptyIterator()); + } + Hints hints = null; + if ((optionProvider.getCrs() != null) && !optionProvider.getCrs().trim().isEmpty()) { + try { + final CoordinateReferenceSystem crs = CRS.decode( + optionProvider.getCrs()); + if (crs != null) { + hints = new Hints(); + hints.put( + Hints.DEFAULT_COORDINATE_REFERENCE_SYSTEM, + crs); + } + } + catch (final Exception e) { + LOGGER.warn( + "Unable to find coordinate reference system, continuing without hint", + e); + } + } + final GridCoverage2DReader reader = format.getReader( + input, hints); if (reader == null) { - LOGGER.error("Unable to get reader instance, getReader returned null"); + LOGGER.error( + "Unable to get reader instance, getReader returned null"); return new Wrapper( Collections.emptyIterator()); } try { - final GridCoverage2D coverage = reader.read(null); + final GridCoverage2D coverage = reader.read( + null); if (coverage != null) { final Map metadata = new HashMap(); final String coverageName = coverage.getName().toString(); - final String[] mdNames = reader.getMetadataNames(coverageName); - if ((mdNames != null) && (mdNames.length > 0)) { - for (final String mdName : mdNames) { - metadata.put( - mdName, - reader.getMetadataValue( - coverageName, - mdName)); + try { + // wrapping with try-catch block because often the reader + // does not support operations on coverage name + // if not, we just don't have metadata, and continue + final String[] mdNames = reader.getMetadataNames( + coverageName); + if ((mdNames != null) && (mdNames.length > 0)) { + for (final String mdName : mdNames) { + metadata.put( + mdName, + reader.getMetadataValue( + coverageName, + mdName)); + } } } + catch (final Exception e) { + LOGGER.debug( + "Unable to find metadata from coverage reader", + e); + } final List> coverages = new ArrayList>(); if (optionProvider.isSeparateBands() && (coverage.getNumSampleDimensions() > 1)) { final String baseName = optionProvider.getCoverageName() != null ? optionProvider.getCoverageName() : input.getName(); - final double[][] nodata = optionProvider.getNodata(coverage.getNumSampleDimensions()); + final double[][] nodata = optionProvider.getNodata( + coverage.getNumSampleDimensions()); for (int b = 0; b < coverage.getNumSampleDimensions(); b++) { final RasterDataAdapter adapter = new RasterDataAdapter( baseName + "_B" + b, @@ -120,10 +212,11 @@ public CloseableIterator> toGeoWaveData( new double[][] { nodata[b] }); - coverages.add(new GeoWaveData( - adapter, - primaryIndexIds, - coverage)); + coverages.add( + new GeoWaveData( + adapter, + primaryIndexIds, + coverage)); } } else { @@ -135,11 +228,13 @@ public CloseableIterator> toGeoWaveData( optionProvider.getTileSize(), optionProvider.isBuildPyramid(), optionProvider.isBuildHistogream(), - optionProvider.getNodata(coverage.getNumSampleDimensions())); - coverages.add(new GeoWaveData( - adapter, - primaryIndexIds, - coverage)); + optionProvider.getNodata( + coverage.getNumSampleDimensions())); + coverages.add( + new GeoWaveData( + adapter, + primaryIndexIds, + coverage)); } return new Wrapper( coverages.iterator()) { @@ -152,8 +247,9 @@ public void close() }; } else { - LOGGER.warn("Null grid coverage from file '" + input.getAbsolutePath() - + "' for discovered geotools format '" + format.getName() + "'"); + LOGGER.warn( + "Null grid coverage from file '" + input.getAbsolutePath() + + "' for discovered geotools format '" + format.getName() + "'"); } } catch (final IOException e) { diff --git a/extensions/formats/geotools-raster/src/main/java/mil/nga/giat/geowave/format/geotools/raster/RasterOptionProvider.java b/extensions/formats/geotools-raster/src/main/java/mil/nga/giat/geowave/format/geotools/raster/RasterOptionProvider.java index ab5806fd041..4e9d5c3ec18 100644 --- a/extensions/formats/geotools-raster/src/main/java/mil/nga/giat/geowave/format/geotools/raster/RasterOptionProvider.java +++ b/extensions/formats/geotools-raster/src/main/java/mil/nga/giat/geowave/format/geotools/raster/RasterOptionProvider.java @@ -14,10 +14,13 @@ public class RasterOptionProvider implements { @Parameter(names = "--pyramid", description = "Build an image pyramid on ingest for quick reduced resolution query") - private boolean buildPyramid = RasterDataAdapter.DEFAULT_BUILD_PYRAMID; + private boolean buildPyramid = false; + @Parameter(names = "--crs", description = "A CRS override for the provided raster file") + private String crs = null; + @Parameter(names = "--histogram", description = "Build a histogram of samples per band on ingest for performing band equalization") - private boolean buildHistogream = RasterDataAdapter.DEFAULT_BUILD_HISTOGRAM; + private boolean buildHistogream = false; @Parameter(names = "--tileSize", description = "Optional parameter to set the tile size stored (default is 256)") private int tileSize = RasterDataAdapter.DEFAULT_TILE_SIZE; @@ -45,6 +48,10 @@ public boolean isSeparateBands() { return separateBands; } + public String getCrs() { + return crs; + } + public String getCoverageName() { if ((coverageName == null) || coverageName.trim().isEmpty()) { return null;