minor fixes for standalone installer (#1855)

* minor fixes for standalone installer * added gdal to java.library.path * removed kerberos IT * minor updates * more minor updates
locationtech · Sep 26, 2021 · f15f56a · f15f56a
1 parent e2233cb
commit f15f56a
Show file tree

Hide file tree

Showing 11 changed files with 103 additions and 30 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -125,7 +125,7 @@ jobs:
         strategy:
             fail-fast: false
             matrix:
-                profile: [redis-it, rocksdb-it, accumulo-it-client, accumulo-it-server, hbase-it-client, hbase-it-server, dynamodb-it, bigtable-it, kudu-it, filesystem-it, accumulo-it-kerberos, 'filesystem-it,secondary-index-it', 'rocksdb-it,secondary-index-it', 'accumulo-it-server,compatibility','hbase-it-server,secondary-index-it','hbase-it-server,secondary-index-it']
+                profile: [redis-it, rocksdb-it, accumulo-it-client, accumulo-it-server, hbase-it-client, hbase-it-server, dynamodb-it, bigtable-it, kudu-it, filesystem-it, 'filesystem-it,secondary-index-it', 'rocksdb-it,secondary-index-it', 'accumulo-it-server,compatibility','hbase-it-server,secondary-index-it','hbase-it-server,secondary-index-it']
                 include:
                   - profile: cassandra-it
                     retry_tests: true

diff --git a/deploy/packaging/docker/build-src/build-geowave-common.sh b/deploy/packaging/docker/build-src/build-geowave-common.sh
@@ -60,6 +60,6 @@ fi
 if [ -d /opt/install4j7 ]; then
     # Build standalone installer
     echo '###### Building standalone installer'
-    mvn package -P build-installer-plugin $BUILD_ARGS "$@"
+    mvn -pl '!test' package -P build-installer-plugin $BUILD_ARGS "$@"
     mvn package -pl deploy -P build-installer-main -Dinstall4j.home=/opt/install4j7 $BUILD_ARGS "$@"
 fi
diff --git a/deploy/packaging/standalone/standalone-installer.install4j b/deploy/packaging/standalone/standalone-installer.install4j
@@ -964,6 +964,27 @@ return console.askOkCancel(message, true);
                 </serializedBean>
                 <condition />
               </action>
+              <action name="" id="1206" customizedId="" beanClass="com.install4j.runtime.beans.actions.misc.ModifyEnvironmentVariableAction" enabled="true" commentSet="false" comment="" actionElevationType="elevated" rollbackBarrier="false" rollbackBarrierExitCode="0" multiExec="false" failureStrategy="1" errorMessage="">
+                <serializedBean>
+                  <java class="java.beans.XMLDecoder">
+                    <object class="com.install4j.runtime.beans.actions.misc.ModifyEnvironmentVariableAction">
+                      <void property="type">
+                        <object class="java.lang.Enum" method="valueOf">
+                          <class>com.install4j.runtime.beans.actions.misc.ModifyStringType</class>
+                          <string>PREPEND</string>
+                        </object>
+                      </void>
+                      <void property="value">
+                        <string>${installer:sys.installationDir}\lib\utilities\gdal</string>
+                      </void>
+                      <void property="variableName">
+                        <string>Path</string>
+                      </void>
+                    </object>
+                  </java>
+                </serializedBean>
+                <condition />
+              </action>
             </actions>
             <formComponents>
               <formComponent name="" id="16" customizedId="" beanClass="com.install4j.runtime.beans.formcomponents.ProgressComponent" enabled="true" commentSet="false" comment="" insetTop="" insetLeft="" insetBottom="" insetRight="" resetInitOnPrevious="false" useExternalParametrization="false" externalParametrizationName="" externalParametrizationMode="all">
@@ -1715,7 +1736,7 @@ return console.askYesNo(message, true);
     </unixInstaller>
   </mediaSets>
   <buildIds buildAll="false">
-    <mediaSet refId="66" />
+    <mediaSet refId="1198" />
   </buildIds>
   <buildOptions verbose="false" faster="false" disableSigning="false" disableJreBundling="false" debug="false" />
 </install4j>
diff --git a/docs/content/quickstart-emr/quickstart-emr/015-vector-demo.adoc b/docs/content/quickstart-emr/quickstart-emr/015-vector-demo.adoc
@@ -64,7 +64,7 @@ Now that the data has been ingested, we can make queries against it.  The GeoWav
 
 [source, bash]
 ----
-$ geowave vector query "SELECT * FROM gdelt.gdeltevent LIMIT 10"
+$ geowave query gdelt "SELECT * FROM gdeltevent LIMIT 10"
 ----
 
 This command tells GeoWave to select all attributes from the `gdeltevent` type in the `gdelt` data store, but limits the output to 10 features.  After running this command, you should get a result that is similar to the following:
@@ -87,11 +87,11 @@ This command tells GeoWave to select all attributes from the `gdeltevent` type i
 +-------------------------+-----------+------------------------------+----------+-----------+----------------+----------------+-------------+-------------------------------------------------------------------------------------------------------+
 ----
 
-We can see right away that these results are tagged with the country code `EZ` which falls under Czech Republic.  Since our area of interest is around Germany, perhaps we want to only see events that are tagged with the `GM` country code.  We can do this by adding a CQL filter to the query.
+We can see right away that these results are tagged with the country code `EZ` which falls under Czech Republic.  Since our area of interest is around Germany, perhaps we want to only see events that are tagged with the `GM` country code.  We can do this by adding a WHERE clause to the query.
 
 [source, bash]
 ----
-$ geowave vector query "SELECT * FROM gdelt.gdeltevent WHERE CQL(countryCode='GM') LIMIT 10"
+$ geowave query gdelt "SELECT * FROM gdeltevent WHERE countryCode='GM' LIMIT 10"
 ----
 
 Now the results show only events that have the `GM` country code.
@@ -118,7 +118,7 @@ If we wanted to see how many events belong to to the `GM` country code, we can p
 
 [source, bash]
 ----
-$ geowave vector query "SELECT COUNT(*) FROM gdelt.gdeltevent WHERE CQL(countryCode='GM')"
+$ geowave query gdelt "SELECT COUNT(*) FROM gdeltevent WHERE countryCode='GM'"
 ----
 
 [literal%nowrap]
@@ -134,7 +134,7 @@ We can also perform multiple aggregations on the same data in a single query. Th
 
 [source, bash]
 ----
-$ geowave vector query "SELECT COUNT(actor1Name), COUNT(actor2Name) FROM gdelt.gdeltevent"
+$ geowave query gdelt "SELECT COUNT(actor1Name), COUNT(actor2Name) FROM gdeltevent"
 ----
 
 [literal%nowrap]
@@ -150,7 +150,7 @@ We can also do bounding box aggregations.  For example, if we wanted to see the
 
 [source, bash]
 ----
-$ geowave vector query "SELECT BBOX(*), COUNT(*) AS total_events FROM gdelt.gdeltevent WHERE CQL(actor1Name='HUNGARY')"
+$ geowave query gdelt "SELECT BBOX(*), COUNT(*) AS total_events FROM gdeltevent WHERE actor1Name='HUNGARY'"
 ----
 
 [literal%nowrap]
@@ -167,7 +167,7 @@ $ geowave vector query "SELECT BBOX(*), COUNT(*) AS total_events FROM gdelt.gdel
 In these examples each query was output to console, but there are options on the command that allow the query results to be output to several formats, including geojson, shapefile, and CSV.
 ====
 
-For more information about vector queries, see the link:userguide.html#vector-queries[vector queries, window="_blank"] section of the User Guide.
+For more information about queries, see the link:userguide.html#queries[queries, window="_blank"] section of the User Guide.
 
 === Kernel Density Estimation (KDE)
 

diff --git a/docs/content/quickstart/010-vector-demo.adoc b/docs/content/quickstart/010-vector-demo.adoc
@@ -48,7 +48,7 @@ Now that the data has been ingested, we can make queries against it.  The GeoWav
 
 [source, bash]
 ----
-$ geowave vector query "SELECT * FROM gdelt.gdeltevent LIMIT 10"
+$ geowave query gdelt "SELECT * FROM gdeltevent LIMIT 10"
 ----
 
 This command tells GeoWave to select all attributes from the `gdeltevent` type in the `gdelt` data store, but limits the output to 10 features.  After running this command, you should get a result that is similar to the following:
@@ -71,11 +71,11 @@ This command tells GeoWave to select all attributes from the `gdeltevent` type i
 +-------------------------+-----------+------------------------------+----------+-----------+----------------+----------------+-------------+-------------------------------------------------------------------------------------------------------+
 ----
 
-We can see right away that these results are tagged with the country code `EZ` which falls under Czech Republic.  Since our area of interest is around Germany, perhaps we want to only see events that are tagged with the `GM` country code.  We can do this by adding a CQL filter to the query.
+We can see right away that these results are tagged with the country code `EZ` which falls under Czech Republic.  Since our area of interest is around Germany, perhaps we want to only see events that are tagged with the `GM` country code.  We can do this by adding a WHERE clause to the query.
 
 [source, bash]
 ----
-$ geowave vector query "SELECT * FROM gdelt.gdeltevent WHERE CQL(countryCode='GM') LIMIT 10"
+$ geowave query gdelt "SELECT * FROM gdeltevent WHERE countryCode='GM' LIMIT 10"
 ----
 
 Now the results show only events that have the `GM` country code.
@@ -102,7 +102,7 @@ If we wanted to see how many events belong to to the `GM` country code, we can p
 
 [source, bash]
 ----
-$ geowave vector query "SELECT COUNT(*) FROM gdelt.gdeltevent WHERE CQL(countryCode='GM')"
+$ geowave query gdelt "SELECT COUNT(*) FROM gdeltevent WHERE countryCode='GM'"
 ----
 
 [literal%nowrap]
@@ -118,7 +118,7 @@ We can also perform multiple aggregations on the same data in a single query. Th
 
 [source, bash]
 ----
-$ geowave vector query "SELECT COUNT(actor1Name), COUNT(actor2Name) FROM gdelt.gdeltevent"
+$ geowave query gdelt "SELECT COUNT(actor1Name), COUNT(actor2Name) FROM gdeltevent"
 ----
 
 [literal%nowrap]
@@ -134,7 +134,7 @@ We can also do bounding box aggregations.  For example, if we wanted to see the
 
 [source, bash]
 ----
-$ geowave vector query "SELECT BBOX(*), COUNT(*) AS total_events FROM gdelt.gdeltevent WHERE CQL(actor1Name='HUNGARY')"
+$ geowave query gdelt "SELECT BBOX(*), COUNT(*) AS total_events FROM gdeltevent WHERE actor1Name='HUNGARY'"
 ----
 
 [literal%nowrap]
@@ -151,7 +151,7 @@ $ geowave vector query "SELECT BBOX(*), COUNT(*) AS total_events FROM gdelt.gdel
 In these examples each query was output to console, but there are options on the command that allow the query results to be output to several formats, including geojson, shapefile, and CSV.
 ====
 
-For more information about vector queries, see the link:userguide.html#vector-queries[vector queries, window="_blank"] section of the User Guide.
+For more information about vector queries, see the link:userguide.html#queries[queries, window="_blank"] section of the User Guide.
 
 === Kernel Density Estimation (KDE)
 

diff --git a/docs/content/userguide/010-cli.adoc b/docs/content/userguide/010-cli.adoc
@@ -153,7 +153,7 @@ The link:commands.html#analytic-commands[`analytic`] command contains commands f
 
 ==== Vector Commands
 
-The link:commands.html#vector-commands[`vector`] command contains commands that are specific to vector data, this includes the vector query language and various export options.
+The link:commands.html#vector-commands[`vector`] command contains commands that are specific to vector data, this includes various export options.
 
 ==== Raster Commands
 

diff --git a/...ns/adapters/raster/src/main/java/org/locationtech/geowave/adapter/raster/RasterUtils.java b/...ns/adapters/raster/src/main/java/org/locationtech/geowave/adapter/raster/RasterUtils.java
@@ -95,6 +95,9 @@
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 
 public class RasterUtils {
+  private static int MAX_FILL_SIZE = 4_194_304;
+  private static int MAX_FILL_SIZE_WIDTH = 2048;
+  private static int MAX_FILL_SIZE_HEIGHT = 2048;
   private static final RenderingHints DEFAULT_RENDERING_HINTS =
       new RenderingHints(
           new ImmutableMap.Builder().put(
@@ -399,19 +402,42 @@ public static void fillWithNoDataValues(
       final WritableRaster raster,
       final double[][] noDataValues) {
     if ((noDataValues != null) && (noDataValues.length >= raster.getNumBands())) {
-      final double[] noDataFilledArray = new double[raster.getWidth() * raster.getHeight()];
+      final int fillSize = raster.getWidth() * raster.getHeight();
+      final double[] noDataFilledArray;
+      if (fillSize > MAX_FILL_SIZE) {
+        noDataFilledArray = new double[MAX_FILL_SIZE];
+      } else {
+        noDataFilledArray = new double[fillSize];
+      }
+
       for (int b = 0; b < raster.getNumBands(); b++) {
         if ((noDataValues[b] != null) && (noDataValues[b].length > 0)) {
           // just fill every sample in this band with the first no
           // data value for that band
           Arrays.fill(noDataFilledArray, noDataValues[b][0]);
-          raster.setSamples(
-              raster.getMinX(),
-              raster.getMinY(),
-              raster.getWidth(),
-              raster.getHeight(),
-              b,
-              noDataFilledArray);
+          if (fillSize > MAX_FILL_SIZE) {
+            final int maxX = (raster.getMinX() + raster.getWidth());
+            final int maxY = (raster.getMinY() + raster.getHeight());
+            for (int x = raster.getMinX(); x < maxX; x += MAX_FILL_SIZE_WIDTH) {
+              for (int y = raster.getMinY(); y < maxY; y += MAX_FILL_SIZE_HEIGHT) {
+                raster.setSamples(
+                    x,
+                    y,
+                    ((x + MAX_FILL_SIZE_WIDTH) > maxX) ? maxX - x : MAX_FILL_SIZE_WIDTH,
+                    ((y + MAX_FILL_SIZE_HEIGHT) > maxY) ? maxY - y : MAX_FILL_SIZE_HEIGHT,
+                    b,
+                    noDataFilledArray);
+              }
+            }
+          } else {
+            raster.setSamples(
+                raster.getMinX(),
+                raster.getMinY(),
+                raster.getWidth(),
+                raster.getHeight(),
+                b,
+                noDataFilledArray);
+          }
         }
       }
     }

diff --git a/...raster/src/main/java/org/locationtech/geowave/adapter/raster/plugin/gdal/InstallGdal.java b/...raster/src/main/java/org/locationtech/geowave/adapter/raster/plugin/gdal/InstallGdal.java
@@ -51,6 +51,7 @@ public static void main(final String[] args) throws IOException {
     if (gdalDir.exists() && gdalDir.isDirectory()) {
       final File[] files = gdalDir.listFiles();
       if ((files != null) && (files.length > 1)) {
+        System.out.println("GDAL already exists");
         return;
       } else {
         LOGGER.error(
@@ -93,6 +94,7 @@ private static void install(final File gdalDir) throws IOException {
             "File '" + downloadFile.getAbsolutePath() + "' is corrupt and cannot be deleted");
       }
     }
+    System.out.println("Downloading GDAL native libraries...");
     if (!downloadFile.exists()) {
       boolean success = false;
       for (int i = 0; i < 3; i++) {
@@ -155,6 +157,7 @@ public boolean accept(final File pathname) {
     if (!downloadFile.delete()) {
       LOGGER.warn("cannot delete " + downloadFile.getAbsolutePath());
     }
+    System.out.println("GDAL installed in directory " + gdalDir.getAbsolutePath());
   }
 
   private static boolean isWindows() {

diff --git a/pom.xml b/pom.xml
@@ -122,6 +122,7 @@
 		<clearspring.version>2.9.6</clearspring.version>
 		<py4j.version>0.10.8.1</py4j.version>
 		<glassfish.el.version>3.0.1-b08</glassfish.el.version>
+		<paranamer.version>2.8</paranamer.version>
 		<gdal.dir>${project.build.directory}/temp/gdal</gdal.dir>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
@@ -283,6 +284,11 @@
 				<artifactId>jai_imageio</artifactId>
 				<version>1.1</version>
 			</dependency>
+			<dependency>
+				<groupId>com.thoughtworks.paranamer</groupId>
+				<artifactId>paranamer</artifactId>
+				<version>${paranamer.version}</version>
+			</dependency>
 			<dependency>
 				<groupId>com.github.ben-manes.caffeine</groupId>
 				<artifactId>caffeine</artifactId>
@@ -1396,7 +1402,7 @@
 	</reporting>
 	<profiles>
 		<profile>
-			<id>build-installer-plugin</id>			
+			<id>build-installer-plugin</id>
 			<properties>
 				<!-- standalone installer seems to work better at least on windows with 
 					hadoop2 (more compatible with local FS), hbase shaded client is packaged 

diff --git a/test/pom.xml b/test/pom.xml
@@ -430,6 +430,23 @@
 			<classifier>test</classifier>
 		</dependency>
 	</dependencies>
+	<build>
+		<plugins>
+			<plugin>
+				<groupId>org.apache.maven.plugins</groupId>
+				<artifactId>maven-dependency-plugin</artifactId>
+				<dependencies>
+					<dependency>
+						<groupId>org.apache.felix</groupId>
+						<artifactId>maven-bundle-plugin</artifactId>
+						<version>5.1.2</version>
+						<type>maven-plugin</type>
+					</dependency>
+				</dependencies>
+				<extensions>true</extensions>
+			</plugin>
+		</plugins>
+	</build>
 	<profiles>
 		<profile>
 			<id>prepare-tests</id>

diff --git a/test/src/test/java/org/locationtech/geowave/test/landsat/CustomCRSLandsatIT.java b/test/src/test/java/org/locationtech/geowave/test/landsat/CustomCRSLandsatIT.java
@@ -146,8 +146,8 @@ protected DataStorePluginOptions getDataStorePluginOptions() {
   @Test
   public void testMosaic() throws Exception {
     int i = 0;
-    boolean failed = false;
-    while (failed) {
+    boolean failed;
+    do {
       failed = false;
       try {
         internalTestMosaic();
@@ -158,7 +158,7 @@ public void testMosaic() throws Exception {
         failed = true;
         LOGGER.error("testMosaic failed, retry attempt #" + i, e);
       }
-    }
+    } while (failed);
   }
 
   public void internalTestMosaic() throws Exception {