Merge pull request #651 from ngageoint/GEOWAVE-650

Issue #650. Multiple bugs with storing adapter preferences.  Adapter …

analytics/mapreduce/src/main/java/mil/nga/giat/geowave/analytic/mapreduce/clustering/runner/ConvexHullJobRunner.java

@@ -18,6 +18,7 @@
 import mil.nga.giat.geowave.analytic.param.MapReduceParameters;
 import mil.nga.giat.geowave.analytic.param.ParameterEnum;
 import mil.nga.giat.geowave.analytic.param.StoreParameters;
+import mil.nga.giat.geowave.core.geotime.ingest.SpatialDimensionalityTypeProvider;
 import mil.nga.giat.geowave.mapreduce.input.GeoWaveInputKey;
 import mil.nga.giat.geowave.mapreduce.output.GeoWaveOutputKey;
 
@@ -75,7 +76,8 @@ public int run(
 				new ParameterEnum<?>[] {
 					HullParameters.Hull.WRAPPER_FACTORY_CLASS,
 					HullParameters.Hull.PROJECTION_CLASS,
-					HullParameters.Hull.DATA_TYPE_ID
+					HullParameters.Hull.DATA_TYPE_ID,
+					HullParameters.Hull.INDEX_ID
 				},
 				config,
 				getScope());
@@ -109,7 +111,7 @@ public int run(
 		checkIndex(
 				runTimeProperties,
 				HullParameters.Hull.INDEX_ID,
-				"hull_idx");
+				new SpatialDimensionalityTypeProvider().createPrimaryIndex().getId().getString());
 		return super.run(
 				config,
 				runTimeProperties);

analytics/mapreduce/src/main/java/mil/nga/giat/geowave/analytic/mapreduce/clustering/runner/GeoWaveInputLoadJobRunner.java

@@ -14,6 +14,7 @@
 import mil.nga.giat.geowave.analytic.param.MapReduceParameters;
 import mil.nga.giat.geowave.analytic.param.OutputParameters;
 import mil.nga.giat.geowave.analytic.param.ParameterEnum;
+import mil.nga.giat.geowave.core.geotime.ingest.SpatialDimensionalityTypeProvider;
 import mil.nga.giat.geowave.mapreduce.input.GeoWaveInputKey;
 import mil.nga.giat.geowave.mapreduce.output.GeoWaveOutputKey;
 
@@ -73,7 +74,7 @@ public int run(
 				OutputParameters.Output.INDEX_ID,
 				runTimeProperties.getPropertyAsString(
 						CentroidParameters.Centroid.INDEX_ID,
-						"hull_idx"));
+						new SpatialDimensionalityTypeProvider().createPrimaryIndex().getId().getString()));
 		OutputParameters.Output.INDEX_ID.getHelper().setValue(
 				config,
 				getScope(),

extensions/adapters/vector/src/main/java/mil/nga/giat/geowave/adapter/vector/FeatureDataAdapter.java

@@ -19,6 +19,7 @@
 import org.opengis.referencing.operation.MathTransform;
 
 import mil.nga.giat.geowave.adapter.vector.index.SecondaryIndexManager;
+import mil.nga.giat.geowave.adapter.vector.index.SimpleFeaturePrimaryIndexConfiguration;
 import mil.nga.giat.geowave.adapter.vector.plugin.GeoWaveGTDataStore;
 import mil.nga.giat.geowave.adapter.vector.plugin.visibility.AdaptorProxyFieldLevelVisibilityHandler;
 import mil.nga.giat.geowave.adapter.vector.plugin.visibility.JsonDefinitionColumnVisibilityManagement;
@@ -359,10 +360,10 @@ protected byte[] defaultTypeDataToBinary() {
 		byte[] attrBytes = new byte[0];
 
 		final SimpleFeatureUserDataConfigurationSet userDataConfiguration = new SimpleFeatureUserDataConfigurationSet();
-		userDataConfiguration.addConfigurations(new TimeDescriptorConfiguration(
-				persistedType));
-		userDataConfiguration.addConfigurations(new SimpleFeatureStatsConfigurationCollection(
-				persistedType));
+		userDataConfiguration.addConfigurations(new TimeDescriptorConfiguration());
+		userDataConfiguration.addConfigurations(new SimpleFeatureStatsConfigurationCollection());
+		userDataConfiguration.addConfigurations(new SimpleFeaturePrimaryIndexConfiguration());
+		userDataConfiguration.configureFromType(persistedType);
 		try {
 			attrBytes = StringUtils.stringToBinary(userDataConfiguration.asJsonString());
 		}
@@ -466,6 +467,8 @@ protected Object defaultTypeDataFromBinary(
 					myType));
 			userDataConfiguration.addConfigurations(new SimpleFeatureStatsConfigurationCollection(
 					myType));
+			userDataConfiguration.addConfigurations(new SimpleFeaturePrimaryIndexConfiguration(
+					myType));
 			try {
 				userDataConfiguration.fromJsonString(
 						StringUtils.stringFromBinary(attrBytes),

extensions/adapters/vector/src/main/java/mil/nga/giat/geowave/adapter/vector/KryoFeatureDataAdapter.java

@@ -6,6 +6,7 @@
 import java.util.List;
 
 import mil.nga.giat.geowave.adapter.vector.field.SimpleFeatureSerializationProvider;
+import mil.nga.giat.geowave.adapter.vector.index.SimpleFeaturePrimaryIndexConfiguration;
 import mil.nga.giat.geowave.adapter.vector.plugin.visibility.AdaptorProxyFieldLevelVisibilityHandler;
 import mil.nga.giat.geowave.adapter.vector.plugin.visibility.JsonDefinitionColumnVisibilityManagement;
 import mil.nga.giat.geowave.adapter.vector.stats.StatsConfigurationCollection.SimpleFeatureStatsConfigurationCollection;
@@ -81,6 +82,10 @@ public FieldWriter<SimpleFeature, Object> getWriter(
 		return (FieldWriter<SimpleFeature, Object>) FieldUtils.getDefaultWriterForClass(clazz);
 	}
 
+	public boolean hasTemporalConstraints() {
+		return getTimeDescriptors().hasTime();
+	}
+
 	@Override
 	public ByteArrayId getAdapterId() {
 		return adapterId;
@@ -107,7 +112,8 @@ public SimpleFeature decode(
 		final RowBuilder<SimpleFeature, Object> builder = newBuilder();
 		builder.setField(data.getAdapterExtendedData().getValues().get(
 				0));
-		return builder.buildRow(data.getDataId());
+		return builder.buildRow(
+				data.getDataId());
 	}
 
 	@Override
@@ -134,10 +140,10 @@ protected byte[] defaultTypeDataToBinary() {
 		byte[] attrBytes = new byte[0];
 
 		final SimpleFeatureUserDataConfigurationSet userDataConfiguration = new SimpleFeatureUserDataConfigurationSet();
-		userDataConfiguration.addConfigurations(new TimeDescriptorConfiguration(
-				featureType));
-		userDataConfiguration.addConfigurations(new SimpleFeatureStatsConfigurationCollection(
-				featureType));
+		userDataConfiguration.addConfigurations(new TimeDescriptorConfiguration());
+		userDataConfiguration.addConfigurations(new SimpleFeatureStatsConfigurationCollection());
+		userDataConfiguration.addConfigurations(new SimpleFeaturePrimaryIndexConfiguration());
+		userDataConfiguration.configureFromType(this.featureType);
 		try {
 			attrBytes = StringUtils.stringToBinary(userDataConfiguration.asJsonString());
 		}
@@ -146,7 +152,7 @@ protected byte[] defaultTypeDataToBinary() {
 					"Failure to encode simple feature user data configuration",
 					e);
 		}
-		
+
 		final ByteBuffer buf = ByteBuffer.allocate(encodedTypeBytes.length + typeNameBytes.length + visibilityManagementClassNameBytes.length + adapterId.getBytes().length + attrBytes.length + 24);
 
 		buf.putInt(0); // a signal for the new version
@@ -170,7 +176,7 @@ protected Object defaultTypeDataFromBinary(
 		final ByteBuffer buf = ByteBuffer.wrap(bytes);
 		final int initialBytes = buf.getInt();
 		// temporary hack for backward compatibility
-		boolean skipConfig =   (initialBytes > 0);
+		boolean skipConfig = (initialBytes > 0);
 		final byte[] typeNameBytes = skipConfig ? new byte[initialBytes] : new byte[buf.getInt()];
 		final byte[] visibilityManagementClassNameBytes = new byte[buf.getInt()];
 		final byte[] attrBytes = skipConfig ? new byte[0] : new byte[buf.getInt()];
@@ -216,6 +222,8 @@ protected Object defaultTypeDataFromBinary(
 				featureType));
 		userDataConfiguration.addConfigurations(new SimpleFeatureStatsConfigurationCollection(
 				featureType));
+		userDataConfiguration.addConfigurations(new SimpleFeaturePrimaryIndexConfiguration(
+				featureType));
 		try {
 			userDataConfiguration.fromJsonString(
 					StringUtils.stringFromBinary(attrBytes),
@@ -252,7 +260,8 @@ protected Object defaultTypeDataFromBinary(
 							this)));
 			return defaultHandlers;
 		}
-		// LOGGER.warn("Simple Feature Type could not be used for handling the indexed data");
+		// LOGGER.warn("Simple Feature Type could not be used for handling the
+		// indexed data");
 		return super.getDefaultTypeMatchingHandlers(featureType);
 	}
 

extensions/adapters/vector/src/main/java/mil/nga/giat/geowave/adapter/vector/index/ChooseBestMatchIndexQueryStrategy.java

@@ -14,6 +14,7 @@
 import mil.nga.giat.geowave.core.index.sfc.data.MultiDimensionalNumericData;
 import mil.nga.giat.geowave.core.store.CloseableIterator;
 import mil.nga.giat.geowave.core.store.adapter.statistics.DataStatistics;
+import mil.nga.giat.geowave.core.store.adapter.statistics.RowRangeHistogramStatistics;
 import mil.nga.giat.geowave.core.store.index.Index;
 import mil.nga.giat.geowave.core.store.index.PrimaryIndex;
 import mil.nga.giat.geowave.core.store.memory.DataStoreUtils;
@@ -68,6 +69,10 @@ public boolean hasNext() {
 						nextIdx = (PrimaryIndex) nextChoosenIdx;
 						if (nextIdx.getIndexStrategy().getOrderedDimensionDefinitions().length == 0) continue;
 						final List<MultiDimensionalNumericData> constraints = query.getIndexConstraints(nextIdx.getIndexStrategy());
+
+						// no stats, no data
+						if (! stats.containsKey(RowRangeHistogramStatistics.composeId(nextIdx.getId()))) continue;
+
 						if (isFullTableScan(constraints)) {
 							// keep this is as a default in case all indices
 							// result in a full table scan

extensions/adapters/vector/src/main/java/mil/nga/giat/geowave/adapter/vector/index/SimpleFeaturePrimaryIndexConfiguration.java

@@ -16,6 +16,14 @@ public class SimpleFeaturePrimaryIndexConfiguration implements
 	public static final String INDEX_NAME = "PrimaryIndexName";
 	private List<String> indexNames = null;
 
+	public SimpleFeaturePrimaryIndexConfiguration() {}
+
+	public SimpleFeaturePrimaryIndexConfiguration(
+			final SimpleFeatureType type ) {
+		super();
+		updateType(type);
+	}
+
 	public static final List<String> getIndexNames(
 			final SimpleFeatureType type ) {
 		Object obj = type.getUserData().get(
@@ -31,6 +39,7 @@ public static final List<String> getIndexNames(
 	public void updateType(
 			final SimpleFeatureType type ) {
 		final StringBuffer names = new StringBuffer();
+		if (indexNames == null) return;
 		for (String name : indexNames) {
 			if (names.length() > 0) names.append(",");
 			names.append(name);

extensions/adapters/vector/src/main/java/mil/nga/giat/geowave/adapter/vector/plugin/GeoWaveDataStoreComponents.java

@@ -1,18 +1,21 @@
 package mil.nga.giat.geowave.adapter.vector.plugin;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import mil.nga.giat.geowave.adapter.vector.FeatureDataAdapter;
+import org.opengis.feature.simple.SimpleFeature;
+
 import mil.nga.giat.geowave.adapter.vector.GeotoolsFeatureDataAdapter;
 import mil.nga.giat.geowave.adapter.vector.plugin.transaction.GeoWaveTransaction;
 import mil.nga.giat.geowave.adapter.vector.plugin.transaction.TransactionsAllocator;
 import mil.nga.giat.geowave.core.index.ByteArrayId;
 import mil.nga.giat.geowave.core.index.StringUtils;
 import mil.nga.giat.geowave.core.store.CloseableIterator;
+import mil.nga.giat.geowave.core.store.CloseableIteratorWrapper;
 import mil.nga.giat.geowave.core.store.DataStore;
 import mil.nga.giat.geowave.core.store.IndexWriter;
 import mil.nga.giat.geowave.core.store.adapter.statistics.DataStatistics;
@@ -22,13 +25,10 @@
 import mil.nga.giat.geowave.core.store.index.Index;
 import mil.nga.giat.geowave.core.store.index.IndexStore;
 import mil.nga.giat.geowave.core.store.index.PrimaryIndex;
-import mil.nga.giat.geowave.core.store.query.BasicQuery.Constraints;
 import mil.nga.giat.geowave.core.store.query.BasicQuery;
 import mil.nga.giat.geowave.core.store.query.DataIdQuery;
 import mil.nga.giat.geowave.core.store.query.QueryOptions;
 
-import org.opengis.feature.simple.SimpleFeature;
-
 public class GeoWaveDataStoreComponents
 {
 	private final GeotoolsFeatureDataAdapter adapter;
@@ -38,7 +38,7 @@ public class GeoWaveDataStoreComponents
 	private final GeoWaveGTDataStore gtStore;
 	private final TransactionsAllocator transactionAllocator;
 
-	private final List<PrimaryIndex> writeIndices;
+	private final List<PrimaryIndex> preferredIndices;
 
 	public GeoWaveDataStoreComponents(
 			final DataStore dataStore,
@@ -52,7 +52,7 @@ public GeoWaveDataStoreComponents(
 		this.indexStore = indexStore;
 		this.dataStatisticsStore = dataStatisticsStore;
 		this.gtStore = gtStore;
-		writeIndices = gtStore.getWriteIndices(adapter);
+		preferredIndices = gtStore.getPreferredIndices(adapter);
 		this.transactionAllocator = transactionAllocator;
 	}
 
@@ -72,8 +72,8 @@ public GeoWaveGTDataStore getGTstore() {
 		return gtStore;
 	}
 
-	public List<PrimaryIndex> getWriteIndices() {
-		return writeIndices;
+	public List<PrimaryIndex> getPreferredIndices() {
+		return preferredIndices;
 	}
 
 	public DataStatisticsStore getStatsStore() {
@@ -86,7 +86,14 @@ public DataStatisticsStore getStatsStore() {
 		return getGTstore().getIndexQueryStrategy().getIndices(
 				stats,
 				query,
-				getIndexStore().getIndices());
+				new CloseableIteratorWrapper(
+						new Closeable() {
+
+							@Override
+							public void close()
+									throws IOException {}
+						},
+						this.preferredIndices.iterator()));
 	}
 
 	public void remove(
@@ -97,7 +104,7 @@ public void remove(
 		final QueryOptions options = new QueryOptions(
 				adapter);
 		options.setAuthorizations(transaction.composeAuthorizations());
-		options.setIndices(writeIndices.toArray(new Index[writeIndices.size()]));
+		options.setIndices(preferredIndices.toArray(new Index[preferredIndices.size()]));
 
 		dataStore.delete(
 				options,
@@ -114,7 +121,7 @@ public void remove(
 		final QueryOptions options = new QueryOptions(
 				adapter);
 		options.setAuthorizations(transaction.composeAuthorizations());
-		options.setIndices(writeIndices.toArray(new Index[writeIndices.size()]));
+		options.setIndices(preferredIndices.toArray(new Index[preferredIndices.size()]));
 
 		dataStore.delete(
 				options,
@@ -131,7 +138,7 @@ public void write(
 			final Set<String> fidList,
 			final GeoWaveTransaction transaction )
 			throws IOException {
-		for (PrimaryIndex currentIndex : this.writeIndices) {
+		for (PrimaryIndex currentIndex : this.preferredIndices) {
 			try (IndexWriter indexWriter = dataStore.createIndexWriter(
 					currentIndex,
 					new UniformVisibilityWriter<SimpleFeature>(
@@ -152,7 +159,7 @@ public void writeCommit(
 			final SimpleFeature feature,
 			final GeoWaveTransaction transaction )
 			throws IOException {
-		for (PrimaryIndex currentIndex : this.writeIndices) {
+		for (PrimaryIndex currentIndex : this.preferredIndices) {
 			try (IndexWriter indexWriter = dataStore.createIndexWriter(
 					currentIndex,
 					new UniformVisibilityWriter<SimpleFeature>(

extensions/adapters/vector/src/main/java/mil/nga/giat/geowave/adapter/vector/plugin/GeoWaveFeatureReader.java

@@ -497,7 +497,7 @@ public CloseableIterator<SimpleFeature> getData(
 						fid));
 			}
 
-			final List<PrimaryIndex> writeIndices = components.getWriteIndices();
+			final List<PrimaryIndex> writeIndices = components.getPreferredIndices();
 			final PrimaryIndex queryIndex = ((writeIndices != null) && (writeIndices.size() > 0)) ? writeIndices.get(0) : null;
 
 			return components.getDataStore().query(