From f15f56ac5fdcdf1c1c1bb67699a2c82db3e13a76 Mon Sep 17 00:00:00 2001 From: rfecher Date: Sun, 26 Sep 2021 13:26:33 -0400 Subject: [PATCH] minor fixes for standalone installer (#1855) * minor fixes for standalone installer * added gdal to java.library.path * removed kerberos IT * minor updates * more minor updates --- .github/workflows/test.yml | 2 +- .../docker/build-src/build-geowave-common.sh | 2 +- .../standalone/standalone-installer.install4j | 23 +++++++++- .../quickstart-emr/015-vector-demo.adoc | 14 +++---- docs/content/quickstart/010-vector-demo.adoc | 14 +++---- docs/content/userguide/010-cli.adoc | 2 +- .../geowave/adapter/raster/RasterUtils.java | 42 +++++++++++++++---- .../raster/plugin/gdal/InstallGdal.java | 3 ++ pom.xml | 8 +++- test/pom.xml | 17 ++++++++ .../test/landsat/CustomCRSLandsatIT.java | 6 +-- 11 files changed, 103 insertions(+), 30 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index afd1e45c32b..39e7b3cb694 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -125,7 +125,7 @@ jobs: strategy: fail-fast: false matrix: - profile: [redis-it, rocksdb-it, accumulo-it-client, accumulo-it-server, hbase-it-client, hbase-it-server, dynamodb-it, bigtable-it, kudu-it, filesystem-it, accumulo-it-kerberos, 'filesystem-it,secondary-index-it', 'rocksdb-it,secondary-index-it', 'accumulo-it-server,compatibility','hbase-it-server,secondary-index-it','hbase-it-server,secondary-index-it'] + profile: [redis-it, rocksdb-it, accumulo-it-client, accumulo-it-server, hbase-it-client, hbase-it-server, dynamodb-it, bigtable-it, kudu-it, filesystem-it, 'filesystem-it,secondary-index-it', 'rocksdb-it,secondary-index-it', 'accumulo-it-server,compatibility','hbase-it-server,secondary-index-it','hbase-it-server,secondary-index-it'] include: - profile: cassandra-it retry_tests: true diff --git a/deploy/packaging/docker/build-src/build-geowave-common.sh b/deploy/packaging/docker/build-src/build-geowave-common.sh index cd572326d8e..69f57c3044e 100755 --- a/deploy/packaging/docker/build-src/build-geowave-common.sh +++ b/deploy/packaging/docker/build-src/build-geowave-common.sh @@ -60,6 +60,6 @@ fi if [ -d /opt/install4j7 ]; then # Build standalone installer echo '###### Building standalone installer' - mvn package -P build-installer-plugin $BUILD_ARGS "$@" + mvn -pl '!test' package -P build-installer-plugin $BUILD_ARGS "$@" mvn package -pl deploy -P build-installer-main -Dinstall4j.home=/opt/install4j7 $BUILD_ARGS "$@" fi \ No newline at end of file diff --git a/deploy/packaging/standalone/standalone-installer.install4j b/deploy/packaging/standalone/standalone-installer.install4j index 428cf6d95bc..5b44c90c01c 100644 --- a/deploy/packaging/standalone/standalone-installer.install4j +++ b/deploy/packaging/standalone/standalone-installer.install4j @@ -964,6 +964,27 @@ return console.askOkCancel(message, true); + + + + + + + com.install4j.runtime.beans.actions.misc.ModifyStringType + PREPEND + + + + ${installer:sys.installationDir}\lib\utilities\gdal + + + Path + + + + + + @@ -1715,7 +1736,7 @@ return console.askYesNo(message, true); - + diff --git a/docs/content/quickstart-emr/quickstart-emr/015-vector-demo.adoc b/docs/content/quickstart-emr/quickstart-emr/015-vector-demo.adoc index 58a987cff09..c61379c4bc2 100644 --- a/docs/content/quickstart-emr/quickstart-emr/015-vector-demo.adoc +++ b/docs/content/quickstart-emr/quickstart-emr/015-vector-demo.adoc @@ -64,7 +64,7 @@ Now that the data has been ingested, we can make queries against it. The GeoWav [source, bash] ---- -$ geowave vector query "SELECT * FROM gdelt.gdeltevent LIMIT 10" +$ geowave query gdelt "SELECT * FROM gdeltevent LIMIT 10" ---- This command tells GeoWave to select all attributes from the `gdeltevent` type in the `gdelt` data store, but limits the output to 10 features. After running this command, you should get a result that is similar to the following: @@ -87,11 +87,11 @@ This command tells GeoWave to select all attributes from the `gdeltevent` type i +-------------------------+-----------+------------------------------+----------+-----------+----------------+----------------+-------------+-------------------------------------------------------------------------------------------------------+ ---- -We can see right away that these results are tagged with the country code `EZ` which falls under Czech Republic. Since our area of interest is around Germany, perhaps we want to only see events that are tagged with the `GM` country code. We can do this by adding a CQL filter to the query. +We can see right away that these results are tagged with the country code `EZ` which falls under Czech Republic. Since our area of interest is around Germany, perhaps we want to only see events that are tagged with the `GM` country code. We can do this by adding a WHERE clause to the query. [source, bash] ---- -$ geowave vector query "SELECT * FROM gdelt.gdeltevent WHERE CQL(countryCode='GM') LIMIT 10" +$ geowave query gdelt "SELECT * FROM gdeltevent WHERE countryCode='GM' LIMIT 10" ---- Now the results show only events that have the `GM` country code. @@ -118,7 +118,7 @@ If we wanted to see how many events belong to to the `GM` country code, we can p [source, bash] ---- -$ geowave vector query "SELECT COUNT(*) FROM gdelt.gdeltevent WHERE CQL(countryCode='GM')" +$ geowave query gdelt "SELECT COUNT(*) FROM gdeltevent WHERE countryCode='GM'" ---- [literal%nowrap] @@ -134,7 +134,7 @@ We can also perform multiple aggregations on the same data in a single query. Th [source, bash] ---- -$ geowave vector query "SELECT COUNT(actor1Name), COUNT(actor2Name) FROM gdelt.gdeltevent" +$ geowave query gdelt "SELECT COUNT(actor1Name), COUNT(actor2Name) FROM gdeltevent" ---- [literal%nowrap] @@ -150,7 +150,7 @@ We can also do bounding box aggregations. For example, if we wanted to see the [source, bash] ---- -$ geowave vector query "SELECT BBOX(*), COUNT(*) AS total_events FROM gdelt.gdeltevent WHERE CQL(actor1Name='HUNGARY')" +$ geowave query gdelt "SELECT BBOX(*), COUNT(*) AS total_events FROM gdeltevent WHERE actor1Name='HUNGARY'" ---- [literal%nowrap] @@ -167,7 +167,7 @@ $ geowave vector query "SELECT BBOX(*), COUNT(*) AS total_events FROM gdelt.gdel In these examples each query was output to console, but there are options on the command that allow the query results to be output to several formats, including geojson, shapefile, and CSV. ==== -For more information about vector queries, see the link:userguide.html#vector-queries[vector queries, window="_blank"] section of the User Guide. +For more information about queries, see the link:userguide.html#queries[queries, window="_blank"] section of the User Guide. === Kernel Density Estimation (KDE) diff --git a/docs/content/quickstart/010-vector-demo.adoc b/docs/content/quickstart/010-vector-demo.adoc index 8e69cd6ef9a..6d0702ec318 100644 --- a/docs/content/quickstart/010-vector-demo.adoc +++ b/docs/content/quickstart/010-vector-demo.adoc @@ -48,7 +48,7 @@ Now that the data has been ingested, we can make queries against it. The GeoWav [source, bash] ---- -$ geowave vector query "SELECT * FROM gdelt.gdeltevent LIMIT 10" +$ geowave query gdelt "SELECT * FROM gdeltevent LIMIT 10" ---- This command tells GeoWave to select all attributes from the `gdeltevent` type in the `gdelt` data store, but limits the output to 10 features. After running this command, you should get a result that is similar to the following: @@ -71,11 +71,11 @@ This command tells GeoWave to select all attributes from the `gdeltevent` type i +-------------------------+-----------+------------------------------+----------+-----------+----------------+----------------+-------------+-------------------------------------------------------------------------------------------------------+ ---- -We can see right away that these results are tagged with the country code `EZ` which falls under Czech Republic. Since our area of interest is around Germany, perhaps we want to only see events that are tagged with the `GM` country code. We can do this by adding a CQL filter to the query. +We can see right away that these results are tagged with the country code `EZ` which falls under Czech Republic. Since our area of interest is around Germany, perhaps we want to only see events that are tagged with the `GM` country code. We can do this by adding a WHERE clause to the query. [source, bash] ---- -$ geowave vector query "SELECT * FROM gdelt.gdeltevent WHERE CQL(countryCode='GM') LIMIT 10" +$ geowave query gdelt "SELECT * FROM gdeltevent WHERE countryCode='GM' LIMIT 10" ---- Now the results show only events that have the `GM` country code. @@ -102,7 +102,7 @@ If we wanted to see how many events belong to to the `GM` country code, we can p [source, bash] ---- -$ geowave vector query "SELECT COUNT(*) FROM gdelt.gdeltevent WHERE CQL(countryCode='GM')" +$ geowave query gdelt "SELECT COUNT(*) FROM gdeltevent WHERE countryCode='GM'" ---- [literal%nowrap] @@ -118,7 +118,7 @@ We can also perform multiple aggregations on the same data in a single query. Th [source, bash] ---- -$ geowave vector query "SELECT COUNT(actor1Name), COUNT(actor2Name) FROM gdelt.gdeltevent" +$ geowave query gdelt "SELECT COUNT(actor1Name), COUNT(actor2Name) FROM gdeltevent" ---- [literal%nowrap] @@ -134,7 +134,7 @@ We can also do bounding box aggregations. For example, if we wanted to see the [source, bash] ---- -$ geowave vector query "SELECT BBOX(*), COUNT(*) AS total_events FROM gdelt.gdeltevent WHERE CQL(actor1Name='HUNGARY')" +$ geowave query gdelt "SELECT BBOX(*), COUNT(*) AS total_events FROM gdeltevent WHERE actor1Name='HUNGARY'" ---- [literal%nowrap] @@ -151,7 +151,7 @@ $ geowave vector query "SELECT BBOX(*), COUNT(*) AS total_events FROM gdelt.gdel In these examples each query was output to console, but there are options on the command that allow the query results to be output to several formats, including geojson, shapefile, and CSV. ==== -For more information about vector queries, see the link:userguide.html#vector-queries[vector queries, window="_blank"] section of the User Guide. +For more information about vector queries, see the link:userguide.html#queries[queries, window="_blank"] section of the User Guide. === Kernel Density Estimation (KDE) diff --git a/docs/content/userguide/010-cli.adoc b/docs/content/userguide/010-cli.adoc index 8edd68dbb94..b6f322da555 100644 --- a/docs/content/userguide/010-cli.adoc +++ b/docs/content/userguide/010-cli.adoc @@ -153,7 +153,7 @@ The link:commands.html#analytic-commands[`analytic`] command contains commands f ==== Vector Commands -The link:commands.html#vector-commands[`vector`] command contains commands that are specific to vector data, this includes the vector query language and various export options. +The link:commands.html#vector-commands[`vector`] command contains commands that are specific to vector data, this includes various export options. ==== Raster Commands diff --git a/extensions/adapters/raster/src/main/java/org/locationtech/geowave/adapter/raster/RasterUtils.java b/extensions/adapters/raster/src/main/java/org/locationtech/geowave/adapter/raster/RasterUtils.java index 7a0cbc63969..8e174f1643e 100644 --- a/extensions/adapters/raster/src/main/java/org/locationtech/geowave/adapter/raster/RasterUtils.java +++ b/extensions/adapters/raster/src/main/java/org/locationtech/geowave/adapter/raster/RasterUtils.java @@ -95,6 +95,9 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; public class RasterUtils { + private static int MAX_FILL_SIZE = 4_194_304; + private static int MAX_FILL_SIZE_WIDTH = 2048; + private static int MAX_FILL_SIZE_HEIGHT = 2048; private static final RenderingHints DEFAULT_RENDERING_HINTS = new RenderingHints( new ImmutableMap.Builder().put( @@ -399,19 +402,42 @@ public static void fillWithNoDataValues( final WritableRaster raster, final double[][] noDataValues) { if ((noDataValues != null) && (noDataValues.length >= raster.getNumBands())) { - final double[] noDataFilledArray = new double[raster.getWidth() * raster.getHeight()]; + final int fillSize = raster.getWidth() * raster.getHeight(); + final double[] noDataFilledArray; + if (fillSize > MAX_FILL_SIZE) { + noDataFilledArray = new double[MAX_FILL_SIZE]; + } else { + noDataFilledArray = new double[fillSize]; + } + for (int b = 0; b < raster.getNumBands(); b++) { if ((noDataValues[b] != null) && (noDataValues[b].length > 0)) { // just fill every sample in this band with the first no // data value for that band Arrays.fill(noDataFilledArray, noDataValues[b][0]); - raster.setSamples( - raster.getMinX(), - raster.getMinY(), - raster.getWidth(), - raster.getHeight(), - b, - noDataFilledArray); + if (fillSize > MAX_FILL_SIZE) { + final int maxX = (raster.getMinX() + raster.getWidth()); + final int maxY = (raster.getMinY() + raster.getHeight()); + for (int x = raster.getMinX(); x < maxX; x += MAX_FILL_SIZE_WIDTH) { + for (int y = raster.getMinY(); y < maxY; y += MAX_FILL_SIZE_HEIGHT) { + raster.setSamples( + x, + y, + ((x + MAX_FILL_SIZE_WIDTH) > maxX) ? maxX - x : MAX_FILL_SIZE_WIDTH, + ((y + MAX_FILL_SIZE_HEIGHT) > maxY) ? maxY - y : MAX_FILL_SIZE_HEIGHT, + b, + noDataFilledArray); + } + } + } else { + raster.setSamples( + raster.getMinX(), + raster.getMinY(), + raster.getWidth(), + raster.getHeight(), + b, + noDataFilledArray); + } } } } diff --git a/extensions/adapters/raster/src/main/java/org/locationtech/geowave/adapter/raster/plugin/gdal/InstallGdal.java b/extensions/adapters/raster/src/main/java/org/locationtech/geowave/adapter/raster/plugin/gdal/InstallGdal.java index 79eeaa66ab2..f40b95a2052 100644 --- a/extensions/adapters/raster/src/main/java/org/locationtech/geowave/adapter/raster/plugin/gdal/InstallGdal.java +++ b/extensions/adapters/raster/src/main/java/org/locationtech/geowave/adapter/raster/plugin/gdal/InstallGdal.java @@ -51,6 +51,7 @@ public static void main(final String[] args) throws IOException { if (gdalDir.exists() && gdalDir.isDirectory()) { final File[] files = gdalDir.listFiles(); if ((files != null) && (files.length > 1)) { + System.out.println("GDAL already exists"); return; } else { LOGGER.error( @@ -93,6 +94,7 @@ private static void install(final File gdalDir) throws IOException { "File '" + downloadFile.getAbsolutePath() + "' is corrupt and cannot be deleted"); } } + System.out.println("Downloading GDAL native libraries..."); if (!downloadFile.exists()) { boolean success = false; for (int i = 0; i < 3; i++) { @@ -155,6 +157,7 @@ public boolean accept(final File pathname) { if (!downloadFile.delete()) { LOGGER.warn("cannot delete " + downloadFile.getAbsolutePath()); } + System.out.println("GDAL installed in directory " + gdalDir.getAbsolutePath()); } private static boolean isWindows() { diff --git a/pom.xml b/pom.xml index e9214a06b38..735bbf66ef1 100644 --- a/pom.xml +++ b/pom.xml @@ -122,6 +122,7 @@ 2.9.6 0.10.8.1 3.0.1-b08 + 2.8 ${project.build.directory}/temp/gdal UTF-8 UTF-8 @@ -283,6 +284,11 @@ jai_imageio 1.1 + + com.thoughtworks.paranamer + paranamer + ${paranamer.version} + com.github.ben-manes.caffeine caffeine @@ -1396,7 +1402,7 @@ - build-installer-plugin + build-installer-plugin