diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/MongoDBRyaDAO.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/MongoDBRyaDAO.java index ab153b37e..16825ab3f 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/MongoDBRyaDAO.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/MongoDBRyaDAO.java @@ -82,8 +82,7 @@ public synchronized void setConf(final StatefulMongoDBRdfConfiguration conf) { auths = conf.getAuthorizations(); flushEachUpdate.set(conf.flushEachUpdate()); } - - + public void setDB(final DB db) { this.db = db; } @@ -107,7 +106,7 @@ public void init() throws RyaDAOException { index.setConf(conf); } - db = mongoClient.getDB(conf.get(MongoDBRdfConfiguration.MONGO_DB_NAME)); + db = mongoClient.getDB(conf.getRyaInstanceName()); coll = db.getCollection(conf.getTriplesCollectionName()); nameSpaceManager = new SimpleMongoDBNamespaceManager(db.getCollection(conf.getNameSpacesCollectionName())); queryEngine = new MongoDBQueryEngine(); @@ -307,4 +306,4 @@ private void flushIndexers() throws RyaDAOException { } } } -} \ No newline at end of file +} diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/visibility/DocumentVisibilityAdapter.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/visibility/DocumentVisibilityAdapter.java index 50dc311ae..fcd3680e5 100644 --- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/visibility/DocumentVisibilityAdapter.java +++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/visibility/DocumentVisibilityAdapter.java @@ -113,6 +113,8 @@ public static DocumentVisibility toDocumentVisibility(final DBObject mongoObj) t documentVisibilityArray = (Object[]) documentVisibilityObject; } else if (documentVisibilityObject instanceof BasicDBList) { documentVisibilityArray = DocumentVisibilityUtil.convertBasicDBListToObjectArray((BasicDBList) documentVisibilityObject); + } else { + documentVisibilityArray = new String[] {""}; } final String documentVisibilityString = DocumentVisibilityUtil.multidimensionalArrayToBooleanString(documentVisibilityArray); diff --git a/dao/mongodb.rya/src/test/java/org/apache/rya/mongodb/MongoRyaITBase.java b/dao/mongodb.rya/src/test/java/org/apache/rya/mongodb/MongoRyaITBase.java index 760152e0d..33f2bdf07 100644 --- a/dao/mongodb.rya/src/test/java/org/apache/rya/mongodb/MongoRyaITBase.java +++ b/dao/mongodb.rya/src/test/java/org/apache/rya/mongodb/MongoRyaITBase.java @@ -41,10 +41,9 @@ protected void beforeTest() throws Exception { // Setup the configuration that will be used within the test. final MongoDBRdfConfiguration conf = new MongoDBRdfConfiguration( new Configuration() ); conf.setBoolean("sc.useMongo", true); - conf.setTablePrefix("test_"); - conf.setMongoDBName(conf.getRyaInstanceName()); - conf.setMongoHostname( super.getMongoHostname() ); - conf.setMongoPort("" + super.getMongoPort()); + conf.setRyaInstanceName("mongo_test"); + conf.setMongoHostname(getMongoHostname()); + conf.setMongoPort(getMongoPort() + ""); // Let tests update the configuration. updateConfiguration(conf); @@ -77,4 +76,4 @@ public MongoCollection getRyaCollection() { public DBCollection getRyaDbCollection() { return getMongoClient().getDB(conf.getMongoDBName()).getCollection(conf.getTriplesCollectionName()); } -} \ No newline at end of file +} diff --git a/extras/indexing/src/main/java/org/apache/rya/sail/config/RyaSailFactory.java b/extras/indexing/src/main/java/org/apache/rya/sail/config/RyaSailFactory.java index 15c89ebae..f6f135ffd 100644 --- a/extras/indexing/src/main/java/org/apache/rya/sail/config/RyaSailFactory.java +++ b/extras/indexing/src/main/java/org/apache/rya/sail/config/RyaSailFactory.java @@ -80,19 +80,25 @@ private static Sail getRyaSail(final Configuration config) throws InferenceEngin final String user; final String pswd; - // XXX Should(?) be MongoDBRdfConfiguration.MONGO_COLLECTION_PREFIX inside the if below. RYA-135 - final String ryaInstance = config.get(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX); - Objects.requireNonNull(ryaInstance, "RyaInstance or table prefix is missing from configuration."+RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX); + final String ryaInstance; if(ConfigUtils.getUseMongo(config)) { // Get a reference to a Mongo DB configuration object. - final MongoDBRdfConfiguration mongoConfig = (config instanceof MongoDBRdfConfiguration) ? + final MongoDBRdfConfiguration mongoConfig = config instanceof MongoDBRdfConfiguration ? (MongoDBRdfConfiguration)config : new MongoDBRdfConfiguration(config); + + ryaInstance = mongoConfig.getRyaInstanceName(); + + requireNonNull(ryaInstance, "RyaInstance is missing from configuration." + MongoDBRdfConfiguration.RYA_INSTANCE_NAME); + // Instantiate a Mongo client and Mongo DAO. dao = getMongoDAO(mongoConfig); // Then use the DAO's newly-created stateful conf in place of the original rdfConfig = dao.getConf(); } else { + ryaInstance = config.get(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX); + Objects.requireNonNull(ryaInstance, "RyaInstance or table prefix is missing from configuration."+RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX); + rdfConfig = new AccumuloRdfConfiguration(config); user = rdfConfig.get(ConfigUtils.CLOUDBASE_USER); pswd = rdfConfig.get(ConfigUtils.CLOUDBASE_PASSWORD); @@ -221,7 +227,7 @@ public static void updateAccumuloConfig(final AccumuloRdfConfiguration config, f * @return - MongoDBRyaDAO with Indexers configured according to user's specification * @throws RyaDAOException if the DAO can't be initialized */ - public static MongoDBRyaDAO getMongoDAO(MongoDBRdfConfiguration mongoConfig) throws RyaDAOException { + public static MongoDBRyaDAO getMongoDAO(final MongoDBRdfConfiguration mongoConfig) throws RyaDAOException { // Create the MongoClient that will be used by the Sail object's components. final MongoClient client = createMongoClient(mongoConfig); diff --git a/extras/rya.export/export.accumulo/pom.xml b/extras/rya.export/export.accumulo/pom.xml index 06201dceb..09f999988 100644 --- a/extras/rya.export/export.accumulo/pom.xml +++ b/extras/rya.export/export.accumulo/pom.xml @@ -84,46 +84,4 @@ under the License. libthrift - - - - org.codehaus.mojo - jaxb2-maven-plugin - - - xjc - - xjc - - - - - org.apache.rya.export - - - - - com.mycila - license-maven-plugin - -
${project.basedir}/src/license/header.txt
- - - - update-generated-source-headers - - ${project.build.directory}/generated-sources - - XML_STYLE - - - process-sources - - format - - - - - - diff --git a/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/AccumuloRyaStatementStore.java b/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/AccumuloRyaStatementStore.java index ddcdd4c32..8d944a343 100644 --- a/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/AccumuloRyaStatementStore.java +++ b/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/AccumuloRyaStatementStore.java @@ -50,7 +50,6 @@ import org.apache.rya.export.api.store.UpdateStatementException; import org.eclipse.rdf4j.common.iteration.CloseableIteration; -import com.google.common.base.Function; import com.google.common.collect.Iterators; /** @@ -74,6 +73,7 @@ public class AccumuloRyaStatementStore implements RyaStatementStore { private final Set iteratorSettings = new HashSet<>(); private final AccumuloParentMetadataRepository metadataRepo; + private final String ryaInstanceName; /** * Creates a new instance of {@link AccumuloRyaStatementStore}. * @param dao the {@AccumuloRyaDAO}. @@ -87,6 +87,7 @@ public AccumuloRyaStatementStore(final AccumuloRyaDAO dao, final String tablePre } accumuloRyaDao = dao; metadataRepo = new AccumuloParentMetadataRepository(dao); + ryaInstanceName = ryaInstance; } @Override @@ -105,9 +106,7 @@ public Iterator fetchStatements() throws FetchStatementException { } // Convert Entry iterator to RyaStatement iterator final Iterator> entryIter = scanner.iterator(); - final Iterator ryaStatementIter = Iterators.transform(entryIter, new Function, RyaStatement>() { - @Override - public RyaStatement apply(final Entry entry) { + final Iterator ryaStatementIter = Iterators.transform(entryIter, entry -> { final Key key = entry.getKey(); final Value value = entry.getValue(); RyaStatement ryaStatement = null; @@ -117,14 +116,19 @@ public RyaStatement apply(final Entry entry) { log.error("Unable to convert the key/value pair into a Rya Statement", e); } return ryaStatement; - } - }); + }); return ryaStatementIter; } catch (final Exception e) { throw new FetchStatementException("Failed to fetch statements.", e); } } + @Override + public long count() { + //accumulo cannot count. + return -1; + } + @Override public void addStatement(final RyaStatement statement) throws AddStatementException { try { @@ -141,6 +145,16 @@ public void addStatement(final RyaStatement statement) throws AddStatementExcept } } + @Override + public void addStatements(final Iterator statements) throws AddStatementException { + try { + accumuloRyaDao.add(statements); + accumuloRyaDao.flush(); + } catch (final RyaDAOException e) { + throw new AddStatementException("Unable to add the Rya Statement", e); + } + } + @Override public void removeStatement(final RyaStatement statement) throws RemoveStatementException { try { @@ -216,4 +230,9 @@ public void addIterator(final IteratorSetting iteratorSetting) { checkNotNull(iteratorSetting); iteratorSettings.add(iteratorSetting); } + + @Override + public String getRyaInstanceName() { + return ryaInstanceName; + } } \ No newline at end of file diff --git a/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/conf/AccumuloExportConstants.java b/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/conf/AccumuloExportConstants.java index 1fc9eaaea..51dec9bdb 100644 --- a/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/conf/AccumuloExportConstants.java +++ b/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/conf/AccumuloExportConstants.java @@ -27,7 +27,6 @@ import org.apache.log4j.Logger; import org.apache.rya.accumulo.mr.MRUtils; import org.apache.rya.api.RdfCloudTripleStoreConfiguration; -import org.apache.rya.export.InstanceType; import org.apache.rya.indexing.accumulo.ConfigUtils; import com.google.common.collect.ImmutableList; diff --git a/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/conf/InstanceType.java b/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/conf/InstanceType.java new file mode 100644 index 000000000..061ef4897 --- /dev/null +++ b/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/conf/InstanceType.java @@ -0,0 +1,7 @@ +package org.apache.rya.export.accumulo.conf; + +public enum InstanceType { + MOCK, + MINI, + DISTRIBUTION; +} diff --git a/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/parent/AccumuloParentMetadataRepository.java b/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/parent/AccumuloParentMetadataRepository.java index 5f50a43da..030728e56 100644 --- a/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/parent/AccumuloParentMetadataRepository.java +++ b/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/parent/AccumuloParentMetadataRepository.java @@ -141,7 +141,7 @@ private MergeParentMetadata getMetadataFromTable() throws ParentMetadataDoesNotE // Fetch the metadata from the entries. String ryaInstanceName = null; Date timestamp = null; - Date filterTimestamp = null; + long filterTimestamp = -1L; Long parentTimeOffset = null; while (entries.hasNext()) { @@ -154,7 +154,7 @@ private MergeParentMetadata getMetadataFromTable() throws ParentMetadataDoesNotE } else if (columnQualifier.equals(MERGE_PARENT_METADATA_TIMESTAMP)) { timestamp = DATE_LEXICODER.decode(value); } else if (columnQualifier.equals(MERGE_PARENT_METADATA_FILTER_TIMESTAMP)) { - filterTimestamp = DATE_LEXICODER.decode(value); + filterTimestamp = LONG_LEXICODER.decode(value); } else if (columnQualifier.equals(MERGE_PARENT_METADATA_PARENT_TIME_OFFSET)) { parentTimeOffset = LONG_LEXICODER.decode(value); } @@ -220,8 +220,8 @@ private static List makeWriteMetadataMutations(final MergeParentMetada mutations.add(timestampMutation); // Filter Timestamp - if (metadata.getFilterTimestamp() != null) { - final Mutation filterTimestampMutation = makeFieldMutation(metadata.getFilterTimestamp(), DATE_LEXICODER, MERGE_PARENT_METADATA_FILTER_TIMESTAMP); + if (metadata.getFilterTimestamp() != -1L) { + final Mutation filterTimestampMutation = makeFieldMutation(metadata.getFilterTimestamp(), LONG_LEXICODER, MERGE_PARENT_METADATA_FILTER_TIMESTAMP); mutations.add(filterTimestampMutation); } diff --git a/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/policy/TimestampPolicyAccumuloRyaStatementStore.java b/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/policy/TimestampPolicyAccumuloRyaStatementStore.java index ad2701014..13caa0332 100644 --- a/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/policy/TimestampPolicyAccumuloRyaStatementStore.java +++ b/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/policy/TimestampPolicyAccumuloRyaStatementStore.java @@ -18,14 +18,13 @@ */ package org.apache.rya.export.accumulo.policy; -import java.util.Date; import java.util.Iterator; import org.apache.accumulo.core.client.IteratorSetting; import org.apache.accumulo.core.iterators.user.TimestampFilter; import org.apache.rya.api.domain.RyaStatement; import org.apache.rya.export.accumulo.AccumuloRyaStatementStore; -import org.apache.rya.export.api.conf.policy.TimestampPolicyStatementStore; +import org.apache.rya.export.api.policy.TimestampPolicyStatementStore; import org.apache.rya.export.api.store.FetchStatementException; import org.apache.rya.export.api.store.RyaStatementStore; @@ -34,15 +33,15 @@ * filter statements based on a timestamp. */ public class TimestampPolicyAccumuloRyaStatementStore extends TimestampPolicyStatementStore { - + //an instance is held onto to be able to add iterators to. + private final AccumuloRyaStatementStore store; /** * Creates a new {@link TimestampPolicyAccumuloRyaStatementStore} * @param store - * @param timestamp */ - public TimestampPolicyAccumuloRyaStatementStore(final AccumuloRyaStatementStore store, final Date timestamp) { + public TimestampPolicyAccumuloRyaStatementStore(final AccumuloRyaStatementStore store, final long timestamp) { super(store, timestamp); - store.addIterator(getStartTimeSetting(timestamp)); + this.store = store; } /** @@ -50,15 +49,16 @@ public TimestampPolicyAccumuloRyaStatementStore(final AccumuloRyaStatementStore * @param time the start time of the filter. * @return the {@link IteratorSetting}. */ - private static IteratorSetting getStartTimeSetting(final Date time) { + private IteratorSetting getStartTimeSetting() { final IteratorSetting setting = new IteratorSetting(1, "startTimeIterator", TimestampFilter.class); - TimestampFilter.setStart(setting, time.getTime(), true); + TimestampFilter.setStart(setting, timestamp, true); TimestampFilter.setEnd(setting, Long.MAX_VALUE, true); return setting; } @Override public Iterator fetchStatements() throws FetchStatementException { + store.addIterator(getStartTimeSetting()); return store.fetchStatements(); } } diff --git a/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/util/AccumuloInstanceDriver.java b/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/util/AccumuloInstanceDriver.java index e077751dc..69232e5fc 100644 --- a/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/util/AccumuloInstanceDriver.java +++ b/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/accumulo/util/AccumuloInstanceDriver.java @@ -49,8 +49,8 @@ import org.apache.rya.api.RdfCloudTripleStoreConstants; import org.apache.rya.api.path.PathUtils; import org.apache.rya.api.persist.RyaDAOException; -import org.apache.rya.export.InstanceType; import org.apache.rya.export.accumulo.conf.AccumuloExportConstants; +import org.apache.rya.export.accumulo.conf.InstanceType; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -118,7 +118,7 @@ public class AccumuloInstanceDriver { /** * Creates a new instance of {@link AccumuloInstanceDriver}. - * + * * @param driverName * the name used to identify this driver in the logs. (not {@code null}) * @param instanceType diff --git a/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/api/conf/AccumuloConfigurationAdapter.java b/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/api/conf/AccumuloConfigurationAdapter.java deleted file mode 100644 index 63d093096..000000000 --- a/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/api/conf/AccumuloConfigurationAdapter.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.rya.export.api.conf; - -import org.apache.rya.export.AccumuloMergeToolConfiguration; -import org.apache.rya.export.DBType; -import org.apache.rya.export.InstanceType; -import org.apache.rya.export.MergeToolConfiguration; -import org.apache.rya.export.api.conf.AccumuloMergeConfiguration.AccumuloBuilder; - -/** - * Helper for creating the immutable application configuration that uses - * Accumulo. - */ -public class AccumuloConfigurationAdapter extends ConfigurationAdapter { - /** - * @param genConfig - The JAXB generated configuration. - * @return The {@link MergeConfiguration} used in the application - * @throws MergeConfigurationException - */ - @Override - public MergeConfiguration createConfig(final MergeToolConfiguration genConfig) throws MergeConfigurationException { - final AccumuloMergeToolConfiguration aConfig = (AccumuloMergeToolConfiguration) genConfig; - final DBType parentType = aConfig.getParentDBType(); - final DBType childType = aConfig.getChildDBType(); - final MergeConfiguration.Builder configBuilder = super.getBuilder(aConfig); - final AccumuloBuilder builder = new AccumuloBuilder(configBuilder); - if(parentType == DBType.ACCUMULO) { - verifyParentInstanceType(aConfig); - builder.setParentZookeepers(aConfig.getParentZookeepers()) - .setParentAuths(aConfig.getParentAuths()) - .setParentInstanceType(aConfig.getParentInstanceType()); - } - - if(childType == DBType.ACCUMULO) { - verifyChildInstanceType(aConfig); - builder.setChildZookeepers(aConfig.getChildZookeepers()) - .setChildAuths(aConfig.getChildAuths()) - .setChildInstanceType(aConfig.getChildInstanceType()); - } - - return builder.build(); - } - - private void verifyParentInstanceType(final AccumuloMergeToolConfiguration aConfig) throws MergeConfigurationException { - final InstanceType type = aConfig.getParentInstanceType(); - switch(type) { - case DISTRIBUTION: - final String auths = aConfig.getParentAuths(); - if(auths == null) { - throw new MergeConfigurationException("Missing authorization level for parent accumulo."); - } - final String zookeepers = aConfig.getParentZookeepers(); - if(zookeepers == null) { - throw new MergeConfigurationException("Missing zookeeper location(s) for parent accumulo."); - } - break; - case MINI: - case MOCK: - break; - } - } - - private void verifyChildInstanceType(final AccumuloMergeToolConfiguration aConfig) throws MergeConfigurationException { - final InstanceType type = aConfig.getChildInstanceType(); - switch(type) { - case DISTRIBUTION: - final String auths = aConfig.getChildAuths(); - if(auths == null) { - throw new MergeConfigurationException("Missing authorization level for child accumulo."); - } - final String zookeepers = aConfig.getChildZookeepers(); - if(zookeepers == null) { - throw new MergeConfigurationException("Missing zookeeper location(s) for child accumulo."); - } - break; - case MINI: - case MOCK: - break; - } - } -} diff --git a/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/api/conf/AccumuloMergeConfiguration.java b/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/api/conf/AccumuloMergeConfiguration.java deleted file mode 100644 index a35d5aab7..000000000 --- a/extras/rya.export/export.accumulo/src/main/java/org/apache/rya/export/api/conf/AccumuloMergeConfiguration.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.rya.export.api.conf; - -import org.apache.http.annotation.Immutable; -import org.apache.rya.export.InstanceType; - -/** - * Immutable configuration object to allow the MergeTool to connect to the parent and child - * databases for data merging. - */ -@Immutable -public class AccumuloMergeConfiguration extends MergeConfigurationDecorator { - /** - * Information needed to connect to the parent database - */ - private final String parentZookeepers; - private final String parentAuths; - private final InstanceType parentInstanceType; - - /** - * Information needed to connect to the child database - */ - private final String childZookeepers; - private final String childAuths; - private final InstanceType childInstanceType; - - /** - * Constructs a {@link AccumuloMergeConfiguration}. All fields are required. - */ - private AccumuloMergeConfiguration(final AccumuloMergeConfiguration.AccumuloBuilder builder) throws MergeConfigurationException { - super(builder); - try { - parentZookeepers = builder.parentZookeepers; - parentAuths = builder.parentAuths; - parentInstanceType = builder.parentInstanceType; - childZookeepers = builder.childZookeepers; - childAuths = builder.childAuths; - childInstanceType = builder.childInstanceType; - } catch(final NullPointerException npe) { - throw new MergeConfigurationException("The configuration was missing required field(s)", npe); - } - } - - /** - * @return the Zookeeper host names of the parent used by Accumulo. - */ - public String getParentZookeepers() { - return parentZookeepers; - } - - /** - * @return the Accumulo user authorizations of the parent. - */ - public String getParentAuths() { - return parentAuths; - } - - /** - * @return the Accumulo instance type of the parent. - */ - public InstanceType getParentInstanceType() { - return parentInstanceType; - } - - /** - * @return the Zookeeper host names of the child used by Accumulo. - */ - public String getChildZookeepers() { - return childZookeepers; - } - - /** - * @return the Accumulo user authorizations of the child. - */ - public String getChildAuths() { - return childAuths; - } - - /** - * @return the Accumulo instance type of the child. - */ - public InstanceType getChildInstanceType() { - return childInstanceType; - } - - /** - * Builder to help create {@link MergeConfiguration}s. - */ - public static class AccumuloBuilder extends MergeConfiguration.Builder { - private String parentZookeepers; - private String parentAuths; - private InstanceType parentInstanceType; - - private String childZookeepers; - private String childAuths; - private InstanceType childInstanceType; - - public AccumuloBuilder(final MergeConfiguration.Builder builder) { - super(builder); - } - - /** - * @param zookeepers - the Zookeeper host names of the parent used by - * Accumulo. - * @return the updated {@link AccumuloBuilder}. - */ - public AccumuloBuilder setParentZookeepers(final String zookeepers) { - parentZookeepers = zookeepers; - return this; - } - - /** - * @param auths - the Accumulo user authorizations of the parent. - * @return the updated {@link AccumuloBuilder}. - */ - public AccumuloBuilder setParentAuths(final String auths) { - parentAuths = auths; - return this; - } - - /** - * @param instanceType the Accumulo instance type of the parent. - * @return the updated {@link AccumuloBuilder}. - */ - public AccumuloBuilder setParentInstanceType(final InstanceType instanceType) { - parentInstanceType = instanceType; - return this; - } - - /** - * @param zookeepers - the Zookeeper host names of the child used by - * Accumulo. - * @return the updated {@link AccumuloBuilder}. - */ - public AccumuloBuilder setChildZookeepers(final String zookeepers) { - childZookeepers = zookeepers; - return this; - } - - /** - * @param auths - the Accumulo user authorizations of the child. - * @return the updated {@link AccumuloBuilder}. - */ - public AccumuloBuilder setChildAuths(final String auths) { - childAuths = auths; - return this; - } - - /** - * @params instanceType - the Accumulo instance type of the child. - * @return the updated {@link AccumuloBuilder}. - */ - public AccumuloBuilder setChildInstanceType(final InstanceType instanceType) { - childInstanceType = instanceType; - return this; - } - - @Override - public AccumuloMergeConfiguration build() throws MergeConfigurationException { - return new AccumuloMergeConfiguration(this); - } - } -} diff --git a/extras/rya.export/export.accumulo/src/main/xsd/AccumuloMergeConfiguration.xsd b/extras/rya.export/export.accumulo/src/main/xsd/AccumuloMergeConfiguration.xsd deleted file mode 100644 index e0cd5ce83..000000000 --- a/extras/rya.export/export.accumulo/src/main/xsd/AccumuloMergeConfiguration.xsd +++ /dev/null @@ -1,56 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/extras/rya.export/export.accumulo/src/test/java/org/apache/rya/export/accumulo/AccumuloRyaStatementStoreTest.java b/extras/rya.export/export.accumulo/src/test/java/org/apache/rya/export/accumulo/AccumuloRyaStatementStoreTest.java index cdbd0a60b..d8b2264bb 100644 --- a/extras/rya.export/export.accumulo/src/test/java/org/apache/rya/export/accumulo/AccumuloRyaStatementStoreTest.java +++ b/extras/rya.export/export.accumulo/src/test/java/org/apache/rya/export/accumulo/AccumuloRyaStatementStoreTest.java @@ -21,8 +21,6 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import java.util.Date; import java.util.Iterator; @@ -31,11 +29,9 @@ import org.apache.log4j.LogManager; import org.apache.log4j.Logger; import org.apache.rya.api.domain.RyaStatement; -import org.apache.rya.export.InstanceType; -import org.apache.rya.export.MergePolicy; +import org.apache.rya.export.accumulo.conf.InstanceType; import org.apache.rya.export.accumulo.util.AccumuloInstanceDriver; import org.apache.rya.export.api.MergerException; -import org.apache.rya.export.api.conf.AccumuloMergeConfiguration; import org.apache.rya.export.api.store.AddStatementException; import org.apache.rya.export.api.store.FetchStatementException; import org.apache.rya.export.api.store.RemoveStatementException; @@ -393,31 +389,7 @@ private static AccumuloInstanceDriver startAccumuloInstanceDriver() throws Excep return accumuloInstanceDriver; } - private static AccumuloMergeConfiguration createAccumuloMergeConfiguration() { - final AccumuloMergeConfiguration accumuloMergeConfiguration = mock(AccumuloMergeConfiguration.class); - - when(accumuloMergeConfiguration.getParentRyaInstanceName()).thenReturn(INSTANCE_NAME); - when(accumuloMergeConfiguration.getParentUsername()).thenReturn(USER_NAME); - when(accumuloMergeConfiguration.getParentPassword()).thenReturn(PASSWORD); - when(accumuloMergeConfiguration.getParentInstanceType()).thenReturn(INSTANCE_TYPE); - when(accumuloMergeConfiguration.getParentTablePrefix()).thenReturn(RYA_TABLE_PREFIX); - when(accumuloMergeConfiguration.getParentAuths()).thenReturn(AUTHS); - - // Other - when(accumuloMergeConfiguration.getMergePolicy()).thenReturn(MergePolicy.TIMESTAMP); - - return accumuloMergeConfiguration; - } - private static AccumuloRyaStatementStore createAccumuloRyaStatementStore() throws MergerException { - final AccumuloMergeConfiguration accumuloMergeConfiguration = createAccumuloMergeConfiguration(); - return createAccumuloRyaStatementStore(accumuloMergeConfiguration); - } - - private static AccumuloRyaStatementStore createAccumuloRyaStatementStore(final AccumuloMergeConfiguration accumuloMergeConfiguration) throws MergerException { - final String instance = accumuloMergeConfiguration.getParentRyaInstanceName(); - final String tablePrefix = accumuloMergeConfiguration.getParentTablePrefix(); - - return new AccumuloRyaStatementStore(accumuloInstanceDriver.getDao(), tablePrefix, instance); + return new AccumuloRyaStatementStore(accumuloInstanceDriver.getDao(), RYA_TABLE_PREFIX, INSTANCE_NAME); } } diff --git a/extras/rya.export/export.accumulo/src/test/java/org/apache/rya/export/accumulo/driver/AccumuloDualInstanceDriver.java b/extras/rya.export/export.accumulo/src/test/java/org/apache/rya/export/accumulo/driver/AccumuloDualInstanceDriver.java index 467476c03..7cb962f37 100644 --- a/extras/rya.export/export.accumulo/src/test/java/org/apache/rya/export/accumulo/driver/AccumuloDualInstanceDriver.java +++ b/extras/rya.export/export.accumulo/src/test/java/org/apache/rya/export/accumulo/driver/AccumuloDualInstanceDriver.java @@ -39,7 +39,7 @@ import org.apache.rya.accumulo.AccumuloRyaDAO; import org.apache.rya.api.domain.RyaStatement; import org.apache.rya.api.persist.RyaDAOException; -import org.apache.rya.export.InstanceType; +import org.apache.rya.export.accumulo.conf.InstanceType; import org.apache.rya.export.accumulo.util.AccumuloInstanceDriver; /** diff --git a/extras/rya.export/export.accumulo/src/test/java/org/apache/rya/export/accumulo/parent/AccumuloParentMetadataRepositoryAdapterTest.java b/extras/rya.export/export.accumulo/src/test/java/org/apache/rya/export/accumulo/parent/AccumuloParentMetadataRepositoryAdapterTest.java index a5adca19f..68d9244e2 100644 --- a/extras/rya.export/export.accumulo/src/test/java/org/apache/rya/export/accumulo/parent/AccumuloParentMetadataRepositoryAdapterTest.java +++ b/extras/rya.export/export.accumulo/src/test/java/org/apache/rya/export/accumulo/parent/AccumuloParentMetadataRepositoryAdapterTest.java @@ -22,7 +22,7 @@ import java.util.Date; -import org.apache.rya.export.InstanceType; +import org.apache.rya.export.accumulo.conf.InstanceType; import org.apache.rya.export.accumulo.util.AccumuloInstanceDriver; import org.apache.rya.export.api.metadata.MergeParentMetadata; import org.apache.rya.export.api.metadata.ParentMetadataDoesNotExistException; @@ -38,7 +38,7 @@ public class AccumuloParentMetadataRepositoryAdapterTest { private final static String TEST_INSTANCE = "test_instance"; private final static Date TEST_TIMESTAMP = new Date(8675309L); - private final static Date TEST_FILTER_TIMESTAMP = new Date(1234567L); + private final static long TEST_FILTER_TIMESTAMP = 1234567L; private final static long TEST_TIME_OFFSET = 123L; private static final InstanceType INSTANCE_TYPE = InstanceType.MOCK; diff --git a/extras/rya.export/export.accumulo/src/test/java/org/apache/rya/export/api/conf/AccumuloConfigurationAdapterTest.java b/extras/rya.export/export.accumulo/src/test/java/org/apache/rya/export/api/conf/AccumuloConfigurationAdapterTest.java deleted file mode 100644 index e62cae10d..000000000 --- a/extras/rya.export/export.accumulo/src/test/java/org/apache/rya/export/api/conf/AccumuloConfigurationAdapterTest.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.rya.export.api.conf; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.util.Date; - -import org.apache.rya.export.AccumuloMergeToolConfiguration; -import org.apache.rya.export.DBType; -import org.apache.rya.export.InstanceType; -import org.apache.rya.export.MergePolicy; -import org.apache.rya.export.accumulo.conf.AccumuloExportConstants; -import org.apache.rya.export.accumulo.driver.AccumuloDualInstanceDriver; -import org.apache.rya.export.accumulo.util.AccumuloInstanceDriver; -import org.junit.Test; - -/** - * Tests the methods of {@link AccumuloConfigurationAdapter}. - */ -public class AccumuloConfigurationAdapterTest { - private static final InstanceType INSTANCE_TYPE = InstanceType.MOCK; - - private static final boolean IS_MOCK = INSTANCE_TYPE == InstanceType.MOCK; - private static final boolean USE_TIME_SYNC = true; - - private static final String PARENT_HOST_NAME = "localhost:1234"; - private static final int PARENT_PORT = 1111; - private static final String PARENT_USER_NAME = IS_MOCK ? "parent_user" : AccumuloInstanceDriver.ROOT_USER_NAME; - private static final String PARENT_PASSWORD = AccumuloDualInstanceDriver.PARENT_PASSWORD; - private static final String PARENT_INSTANCE = AccumuloDualInstanceDriver.PARENT_INSTANCE; - private static final String PARENT_TABLE_PREFIX = AccumuloDualInstanceDriver.PARENT_TABLE_PREFIX; - private static final String PARENT_AUTH = AccumuloDualInstanceDriver.PARENT_AUTH; - private static final String PARENT_TOMCAT_URL = "http://localhost:8080"; - private static final String PARENT_ZOOKEEPERS = "http://rya-example-box:9090"; - - private static final String CHILD_HOST_NAME = "localhost:4321"; - private static final int CHILD_PORT = 2222; - private static final String CHILD_USER_NAME = IS_MOCK ? "child_user" : AccumuloInstanceDriver.ROOT_USER_NAME; - private static final String CHILD_PASSWORD = AccumuloDualInstanceDriver.CHILD_PASSWORD; - private static final String CHILD_INSTANCE = AccumuloDualInstanceDriver.CHILD_INSTANCE; - private static final String CHILD_TABLE_PREFIX = AccumuloDualInstanceDriver.CHILD_TABLE_PREFIX; - private static final String CHILD_AUTH = AccumuloDualInstanceDriver.CHILD_AUTH; - private static final String CHILD_TOMCAT_URL = "http://localhost:8080"; - private static final String CHILD_ZOOKEEPERS = "http://localhost:9999"; - - - private static final String TOOL_START_TIME = AccumuloExportConstants.convertDateToStartTimeString(new Date()); - private static final String TIME_SERVER = "time.nist.gov"; - - @Test - public void testCreateConfig() throws MergeConfigurationException { - final AccumuloMergeToolConfiguration jConfig = mock(AccumuloMergeToolConfiguration.class); - // Parent Properties - when(jConfig.getParentHostname()).thenReturn(PARENT_HOST_NAME); - when(jConfig.getParentPort()).thenReturn(PARENT_PORT); - when(jConfig.getParentRyaInstanceName()).thenReturn(PARENT_INSTANCE); - when(jConfig.getParentUsername()).thenReturn(PARENT_USER_NAME); - when(jConfig.getParentPassword()).thenReturn(PARENT_PASSWORD); - when(jConfig.getParentTablePrefix()).thenReturn(PARENT_TABLE_PREFIX); - when(jConfig.getParentDBType()).thenReturn(DBType.ACCUMULO); - when(jConfig.getParentTomcatUrl()).thenReturn(PARENT_TOMCAT_URL); - // Parent Accumulo Properties - when(jConfig.getParentInstanceType()).thenReturn(INSTANCE_TYPE); - when(jConfig.getParentAuths()).thenReturn(PARENT_AUTH); - when(jConfig.getParentZookeepers()).thenReturn(PARENT_ZOOKEEPERS); - - // Child Properties - when(jConfig.getChildHostname()).thenReturn(CHILD_HOST_NAME); - when(jConfig.getChildPort()).thenReturn(CHILD_PORT); - when(jConfig.getChildRyaInstanceName()).thenReturn(CHILD_INSTANCE); - when(jConfig.getChildUsername()).thenReturn(CHILD_USER_NAME); - when(jConfig.getChildPassword()).thenReturn(CHILD_PASSWORD); - when(jConfig.getChildTablePrefix()).thenReturn(CHILD_TABLE_PREFIX); - when(jConfig.getChildDBType()).thenReturn(DBType.MONGO); - when(jConfig.getChildTomcatUrl()).thenReturn(CHILD_TOMCAT_URL); - // Other Properties - when(jConfig.getMergePolicy()).thenReturn(MergePolicy.TIMESTAMP); - when(jConfig.getNtpServerHost()).thenReturn(TIME_SERVER); - when(jConfig.isUseNtpServer()).thenReturn(USE_TIME_SYNC); - - - final AccumuloConfigurationAdapter adapter = new AccumuloConfigurationAdapter(); - final AccumuloMergeConfiguration accumuloMergeConfiguration = (AccumuloMergeConfiguration) adapter.createConfig(jConfig); - - assertNotNull(accumuloMergeConfiguration); - assertEquals(AccumuloMergeConfiguration.class, accumuloMergeConfiguration.getClass()); - - // Parent Properties - assertEquals(PARENT_HOST_NAME, accumuloMergeConfiguration.getParentHostname()); - assertEquals(PARENT_USER_NAME, accumuloMergeConfiguration.getParentUsername()); - assertEquals(PARENT_PASSWORD, accumuloMergeConfiguration.getParentPassword()); - assertEquals(PARENT_INSTANCE, accumuloMergeConfiguration.getParentRyaInstanceName()); - assertEquals(PARENT_TABLE_PREFIX, accumuloMergeConfiguration.getParentTablePrefix()); - assertEquals(PARENT_TOMCAT_URL, accumuloMergeConfiguration.getParentTomcatUrl()); - assertEquals(DBType.ACCUMULO, accumuloMergeConfiguration.getParentDBType()); - assertEquals(PARENT_PORT, accumuloMergeConfiguration.getParentPort()); - // Parent Accumulo Properties - assertEquals(PARENT_ZOOKEEPERS, accumuloMergeConfiguration.getParentZookeepers()); - assertEquals(PARENT_AUTH, accumuloMergeConfiguration.getParentAuths()); - assertEquals(InstanceType.MOCK, accumuloMergeConfiguration.getParentInstanceType()); - - // Child Properties - assertEquals(CHILD_HOST_NAME, accumuloMergeConfiguration.getChildHostname()); - assertEquals(CHILD_USER_NAME, accumuloMergeConfiguration.getChildUsername()); - assertEquals(CHILD_PASSWORD, accumuloMergeConfiguration.getChildPassword()); - assertEquals(CHILD_INSTANCE, accumuloMergeConfiguration.getChildRyaInstanceName()); - assertEquals(CHILD_TABLE_PREFIX, accumuloMergeConfiguration.getChildTablePrefix()); - assertEquals(CHILD_TOMCAT_URL, accumuloMergeConfiguration.getChildTomcatUrl()); - assertEquals(DBType.MONGO, accumuloMergeConfiguration.getChildDBType()); - assertEquals(CHILD_PORT, accumuloMergeConfiguration.getChildPort()); - // Child Properties - assertNull(accumuloMergeConfiguration.getChildZookeepers()); - assertNull(accumuloMergeConfiguration.getChildAuths()); - assertNull(accumuloMergeConfiguration.getChildInstanceType()); - - // Other Properties - assertEquals(MergePolicy.TIMESTAMP, accumuloMergeConfiguration.getMergePolicy()); - assertTrue(accumuloMergeConfiguration.getUseNtpServer()); - assertEquals(TIME_SERVER, accumuloMergeConfiguration.getNtpServerHost()); - } -} diff --git a/extras/rya.export/export.api/pom.xml b/extras/rya.export/export.api/pom.xml index b8ad3d272..ff34a6ef7 100644 --- a/extras/rya.export/export.api/pom.xml +++ b/extras/rya.export/export.api/pom.xml @@ -70,225 +70,4 @@ under the License. slf4j-log4j12 - - - - - org.codehaus.mojo - jaxb2-maven-plugin - - - xjc - - xjc - - - - - org.apache.rya.export - - - - com.mycila - license-maven-plugin - -
${project.basedir}/src/license/header.txt
- - - - update-generated-source-headers - - ${project.build.directory}/generated-sources - - XML_STYLE - - - process-sources - - format - - - - - - - - org.apache.maven.plugins - maven-shade-plugin - - - package - - shade - - - - - - - - - - - - org.apache.accumulo:accumulo-native - - - - - *:* - - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - - - - - - - - - - - maven-dependency-plugin - - - install - - copy-dependencies - - - ${project.build.directory}/lib - false - false - true - - - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - - true - lib - - - - - - lib/ - - - - - - - - maven-resources-plugin - - - copy-resource-bin - install - - copy-resources - - - ${project.build.directory}/distribution/${project.artifactId}-${project.version}/bin - - - ${basedir}/startup_scripts - - - - - - copy-resource-config - install - - copy-resources - - - ${project.build.directory}/distribution/${project.artifactId}-${project.version}/config - - - ${basedir}/config - - - - - - copy-resource-lib - install - - copy-resources - - - ${project.build.directory}/distribution/${project.artifactId}-${project.version}/lib - - - ${project.build.directory}/lib - - *.jar - - - - - - - copy-resource-jar - install - - copy-resources - - - ${project.build.directory}/distribution/${project.artifactId}-${project.version}/ - - - ${project.build.directory} - - rya.merger-*.jar - README.md - - - rya.merger-*-shaded.jar - rya.merger-*-sources.jar - - - - - - - copy-resource-readme - install - - copy-resources - - - ${project.build.directory}/distribution/${project.artifactId}-${project.version}/ - - - ${basedir} - - README.md - - - - - - - - - diff --git a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/ConfigurationAdapter.java b/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/ConfigurationAdapter.java deleted file mode 100644 index 2699a96ef..000000000 --- a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/ConfigurationAdapter.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.rya.export.api.conf; - -import org.apache.rya.export.MergeToolConfiguration; -import org.apache.rya.export.api.conf.MergeConfiguration.Builder; - -/** - * Helper for creating the immutable application configuration. - */ -public class ConfigurationAdapter { - /** - * @param jConfig - The JAXB generated configuration. - * @return The {@link MergeConfiguration} used in the application - * @throws MergeConfigurationException - */ - public MergeConfiguration createConfig(final MergeToolConfiguration jConfig) throws MergeConfigurationException { - final Builder configBuilder = getBuilder(jConfig); - return configBuilder.build(); - } - - protected Builder getBuilder(final MergeToolConfiguration jConfig) { - final Builder configBuilder = new Builder() - .setParentHostname(jConfig.getParentHostname()) - .setParentUsername(jConfig.getParentUsername()) - .setParentPassword(jConfig.getParentPassword()) - .setParentTablePrefix(jConfig.getParentTablePrefix()) - .setParentRyaInstanceName(jConfig.getParentRyaInstanceName()) - .setParentTomcatUrl(jConfig.getParentTomcatUrl()) - .setParentDBType(jConfig.getParentDBType()) - .setParentPort(jConfig.getParentPort()) - .setChildHostname(jConfig.getChildHostname()) - .setChildUsername(jConfig.getChildUsername()) - .setChildPassword(jConfig.getChildPassword()) - .setChildTablePrefix(jConfig.getChildTablePrefix()) - .setChildRyaInstanceName(jConfig.getChildRyaInstanceName()) - .setChildTomcatUrl(jConfig.getChildTomcatUrl()) - .setChildDBType(jConfig.getChildDBType()) - .setChildPort(jConfig.getChildPort()) - .setMergePolicy(jConfig.getMergePolicy()) - .setUseNtpServer(jConfig.isUseNtpServer()) - .setNtpServerHost(jConfig.getNtpServerHost()); - return configBuilder; - } -} diff --git a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/MergeConfiguration.java b/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/MergeConfiguration.java deleted file mode 100644 index 96f7ed678..000000000 --- a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/MergeConfiguration.java +++ /dev/null @@ -1,466 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.rya.export.api.conf; - -import static com.google.common.base.Preconditions.checkNotNull; - -import org.apache.http.annotation.Immutable; -import org.apache.rya.export.DBType; -import org.apache.rya.export.MergePolicy; - -/** - * Immutable configuration object to allow the MergeTool to connect to the parent and child - * databases for data merging. - */ -@Immutable -public class MergeConfiguration { - /** - * Information needed to connect to the parent database - */ - private final String parentHostname; - private final String parentUsername; - private final String parentPassword; - private final String parentRyaInstanceName; - private final String parentTablePrefix; - private final String parentTomcatUrl; - private final DBType parentDBType; - private final int parentPort; - - /** - * Information needed to connect to the child database - */ - private final String childHostname; - private final String childUsername; - private final String childPassword; - private final String childRyaInstanceName; - private final String childTablePrefix; - private final String childTomcatUrl; - private final DBType childDBType; - private final int childPort; - - private final MergePolicy mergePolicy; - - private final boolean useNtpServer; - private final String ntpServerHost; - - /** - * Constructs a {@link MergeConfiguration}. - */ - protected MergeConfiguration(final Builder builder) throws MergeConfigurationException { - try { - parentHostname = checkNotNull(builder.parentHostname); - parentUsername = builder.parentUsername; - parentPassword = builder.parentPassword; - parentRyaInstanceName = checkNotNull(builder.parentRyaInstanceName); - parentTablePrefix = checkNotNull(builder.parentTablePrefix); - parentTomcatUrl = checkNotNull(builder.parentTomcatUrl); - parentDBType = checkNotNull(builder.parentDBType); - parentPort = checkNotNull(builder.parentPort); - childHostname = checkNotNull(builder.childHostname); - childUsername = builder.childUsername; - childPassword = builder.childPassword; - childRyaInstanceName = checkNotNull(builder.childRyaInstanceName); - childTablePrefix = checkNotNull(builder.childTablePrefix); - childTomcatUrl = checkNotNull(builder.childTomcatUrl); - childDBType = checkNotNull(builder.childDBType); - childPort = checkNotNull(builder.childPort); - mergePolicy = builder.mergePolicy; - useNtpServer = builder.useNtpServer; - ntpServerHost = builder.ntpServerHost; - } catch(final NullPointerException npe) { - //fix this. - throw new MergeConfigurationException("The configuration was missing required field(s)", npe); - } - } - - /** - * @return the hostname of the parent. - */ - public String getParentHostname() { - return parentHostname; - } - - /** - * @return the username of the parent. - */ - public String getParentUsername() { - return parentUsername; - } - - /** - * @return the password of the parent. - */ - public String getParentPassword() { - return parentPassword; - } - - /** - * @return the Rya Instance Name of the parent. - */ - public String getParentRyaInstanceName() { - return parentRyaInstanceName; - } - - /** - * @return the Rya table prefix of the parent. - */ - public String getParentTablePrefix() { - return parentTablePrefix; - } - - /** - * @return The URL of the Apache Tomcat server web page running on the parent machine. - */ - public String getParentTomcatUrl() { - return parentTomcatUrl; - } - - /** - * @return the Database Type of the parent. - */ - public DBType getParentDBType() { - return parentDBType; - } - - /** - * @return the port of the parent. - */ - public int getParentPort() { - return parentPort; - } - - /** - * @return the hostname of the child. - */ - public String getChildHostname() { - return childHostname; - } - - /** - * @return the username of the child. - */ - public String getChildUsername() { - return childUsername; - } - - /** - * @return the password of the child. - */ - public String getChildPassword() { - return childPassword; - } - - /** - * @return the Rya Instance Name of the child. - */ - public String getChildRyaInstanceName() { - return childRyaInstanceName; - } - - /** - * @return the Rya table prefix of the child. - */ - public String getChildTablePrefix() { - return childTablePrefix; - } - - /** - * @return The URL of the Apache Tomcat server web page running on the child machine. - */ - public String getChildTomcatUrl() {;// - return childTomcatUrl; - } - - /** - * @return the Database Type of the child. - */ - public DBType getChildDBType() { - return childDBType; - } - - /** - * @return the port of the child. - */ - public int getChildPort() { - return childPort; - } - - /** - * @return the policy to use when merging data. - */ - public MergePolicy getMergePolicy() { - return mergePolicy; - } - - /** - * @return {@code true} to use the NTP server to handle time synchronization. - * {@code false} to not use the NTP server. - */ - public Boolean getUseNtpServer() { - return useNtpServer; - } - - /** - * @return The host name of the time server to use. - */ - public String getNtpServerHost() { - return ntpServerHost; - } - - /** - * Builder to help create {@link MergeConfiguration}s. - */ - public static class Builder { - private String parentHostname; - private String parentUsername; - private String parentPassword; - private String parentRyaInstanceName; - private String parentTablePrefix; - private String parentTomcatUrl; - private DBType parentDBType; - private Integer parentPort; - - private String childHostname; - private String childUsername; - private String childPassword; - private String childRyaInstanceName; - private String childTablePrefix; - private String childTomcatUrl; - private DBType childDBType; - private Integer childPort; - - private MergePolicy mergePolicy; - - private Boolean useNtpServer; - private String ntpServerHost; - - /** - * Creates a new Builder to create a {@link MergeConfiguration}. - */ - public Builder() { - } - - /** - * Creates a Builder based on an existing one. - * @param builder - The {@link Builder} - */ - public Builder(final Builder builder) { - parentDBType = builder.parentDBType; - parentHostname = builder.parentHostname; - parentPassword = builder.parentPassword; - parentPort = builder.parentPort; - parentRyaInstanceName = builder.parentRyaInstanceName; - parentTablePrefix = builder.parentTablePrefix; - parentTomcatUrl = builder.parentTomcatUrl; - parentUsername = builder.parentUsername; - - childDBType = builder.childDBType; - childHostname = builder.childHostname; - childPassword = builder.childPassword; - childPort = builder.childPort; - childRyaInstanceName = builder.childRyaInstanceName; - childTablePrefix = builder.childTablePrefix; - childTomcatUrl = builder.childTomcatUrl; - childUsername = builder.childUsername; - - mergePolicy = builder.mergePolicy; - - useNtpServer = builder.useNtpServer; - ntpServerHost = builder.ntpServerHost; - } - - /** - * @param hostname - the hostname of the parent. - * @return the updated {@link Builder}. - */ - public Builder setParentHostname(final String hostname) { - parentHostname = hostname; - return this; - } - - /** - * @param username - the username of the parent. - * @return the updated {@link Builder}. - */ - public Builder setParentUsername(final String username) { - parentUsername = username; - return this; - } - - /** - * @param password - the password of the parent. - * @return the updated {@link Builder}. - */ - public Builder setParentPassword(final String password) { - parentPassword = password; - return this; - } - - /** - * @param ryaInstanceName - the Rya Instance Name of the parent. - * @return the updated {@link Builder}. - */ - public Builder setParentRyaInstanceName(final String ryaInstanceName) { - parentRyaInstanceName = ryaInstanceName; - return this; - } - - /** - * @param tablePrefix - the Rya table prefix of the parent. - * @return the updated {@link Builder}. - */ - public Builder setParentTablePrefix(final String tablePrefix) { - parentTablePrefix = tablePrefix; - return this; - } - - /** - * @param tomcatUrl - The URL of the Apache Tomcat server web page - * running on the parent machine. - * @return the updated {@link Builder}. - */ - public Builder setParentTomcatUrl(final String tomcatUrl) { - parentTomcatUrl = tomcatUrl; - return this; - } - - /** - * @param dbType - the Database Type of the parent. - * @return the updated {@link Builder}. - */ - public Builder setParentDBType(final DBType dbType) { - parentDBType = dbType; - return this; - } - - /** - * @param port - the port of the parent. - * @return the updated {@link Builder}. - */ - public Builder setParentPort(final Integer port) { - parentPort = port; - return this; - } - - /** - * @param hostname - the hostname of the child. - * @return the updated {@link Builder}. - */ - public Builder setChildHostname(final String hostname) { - childHostname = hostname; - return this; - } - - /** - * @param username - the username of the child. - * @return the updated {@link Builder}. - */ - public Builder setChildUsername(final String username) { - childUsername = username; - return this; - } - - /** - * @param password - the password of the child. - * @return the updated {@link Builder}. - */ - public Builder setChildPassword(final String password) { - childPassword = password; - return this; - } - - /** - * @param ryaInstanceName - the Rya Instance Name of the child. - * @return the updated {@link Builder}. - */ - public Builder setChildRyaInstanceName(final String ryaInstanceName) { - childRyaInstanceName = ryaInstanceName; - return this; - } - - /** - * @param tablePrefix - the Rya table prefix of the child. - * @return the updated {@link Builder}. - */ - public Builder setChildTablePrefix(final String tablePrefix) { - childTablePrefix = tablePrefix; - return this; - } - - /** - * @param tomcatUrl -s The URL of the Apache Tomcat server web page - * running on the child machine. - * @return the updated {@link Builder}. - */ - public Builder setChildTomcatUrl(final String tomcatUrl) { - childTomcatUrl = tomcatUrl; - return this; - } - - /** - * @param dbType - the Database Type of the child. - * @return the updated {@link Builder}. - */ - public Builder setChildDBType(final DBType dbType) { - childDBType = dbType; - return this; - } - - /** - * @param port - the port of the child. - * @return the updated {@link Builder}. - */ - public Builder setChildPort(final Integer port) { - childPort = port; - return this; - } - - /** - * @param mergePolicy - the policy to use when merging data. - * @return the updated {@link Builder}. - */ - public Builder setMergePolicy(final MergePolicy mergePolicy) { - this.mergePolicy = mergePolicy; - return this; - } - - /** - * @param useNtpServer - {@code true} to use the NTP server to handle - * time synchronization. {@code false} to not use the NTP server. - * @return the updated {@link Builder}. - */ - public Builder setUseNtpServer(final Boolean useNtpServer) { - this.useNtpServer = useNtpServer; - return this; - } - - /** - * @param ntpServerHost - The host name of the time server to use. - * @return the updated {@link Builder}. - */ - public Builder setNtpServerHost(final String ntpServerHost) { - this.ntpServerHost = ntpServerHost; - return this; - } - - public MergeConfiguration build() throws MergeConfigurationException { - return new MergeConfiguration(this); - } - } -} \ No newline at end of file diff --git a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/MergeConfigurationDecorator.java b/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/MergeConfigurationDecorator.java deleted file mode 100644 index 461d5fbf2..000000000 --- a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/MergeConfigurationDecorator.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.rya.export.api.conf; - -import org.apache.rya.export.DBType; -import org.apache.rya.export.MergePolicy; - -/** - * Decorator for {@link MergeConfiguration}. - *

- * The {@link MergeConfiguration} should be decorated when a new datastore - * has specific configuration fields or a new statement merge policy - * requires configuration. - */ -public class MergeConfigurationDecorator extends MergeConfiguration { - - /** - * Creates a new {@link MergeConfigurationDecorator} - * @param builder - The configuratoin builder. - * @throws MergeConfigurationException - */ - public MergeConfigurationDecorator(final MergeConfiguration.Builder builder) throws MergeConfigurationException { - super(builder); - } - - @Override - public int hashCode() { - return super.hashCode(); - } - - @Override - public boolean equals(final Object obj) { - return super.equals(obj); - } - - @Override - public String getParentHostname() { - return super.getParentHostname(); - } - - @Override - public String getParentUsername() { - return super.getParentUsername(); - } - - @Override - public String getParentPassword() { - return super.getParentPassword(); - } - - @Override - public String getParentRyaInstanceName() { - return super.getParentRyaInstanceName(); - } - - @Override - public String getParentTablePrefix() { - return super.getParentTablePrefix(); - } - - @Override - public String getParentTomcatUrl() { - return super.getParentTomcatUrl(); - } - - @Override - public DBType getParentDBType() { - return super.getParentDBType(); - } - - @Override - public int getParentPort() { - return super.getParentPort(); - } - - @Override - public String getChildHostname() { - return super.getChildHostname(); - } - - @Override - public String getChildUsername() { - return super.getChildUsername(); - } - - @Override - public String getChildPassword() { - return super.getChildPassword(); - } - - @Override - public String getChildRyaInstanceName() { - return super.getChildRyaInstanceName(); - } - - @Override - public String getChildTablePrefix() { - return super.getChildTablePrefix(); - } - - @Override - public String getChildTomcatUrl() { - return super.getChildTomcatUrl(); - } - - @Override - public DBType getChildDBType() { - return super.getChildDBType(); - } - - @Override - public int getChildPort() { - return super.getChildPort(); - } - - @Override - public MergePolicy getMergePolicy() { - return super.getMergePolicy(); - } - - @Override - public Boolean getUseNtpServer() { - return super.getUseNtpServer(); - } - - @Override - public String getNtpServerHost() { - return super.getNtpServerHost(); - } - - @Override - public String toString() { - return super.toString(); - } -} \ No newline at end of file diff --git a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/MergeConfigurationException.java b/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/MergeConfigurationException.java deleted file mode 100644 index e7e8e8b5a..000000000 --- a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/MergeConfigurationException.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.rya.export.api.conf; - -/** - * An exception to be used when there is a problem configuring the Merge Tool. - */ -public class MergeConfigurationException extends Exception { - private static final long serialVersionUID = 1L; - - /** - * Creates a new {@link MergeConfigurationException} - * @param message - The error message. - */ - public MergeConfigurationException(final String message) { - super(message); - } - - /** - * Creates a new {@link MergeConfigurationException} - * @param message - The error message. - * @param source - The source of the error. - */ - public MergeConfigurationException(final String message, final Throwable source) { - super(message, source); - } -} diff --git a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/policy/TimestampPolicyConfigurationAdapter.java b/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/policy/TimestampPolicyConfigurationAdapter.java deleted file mode 100644 index c9e21a6e0..000000000 --- a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/policy/TimestampPolicyConfigurationAdapter.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.rya.export.api.conf.policy; - -import org.apache.rya.export.MergeToolConfiguration; -import org.apache.rya.export.TimestampMergePolicyConfiguration; -import org.apache.rya.export.api.conf.ConfigurationAdapter; -import org.apache.rya.export.api.conf.MergeConfiguration; -import org.apache.rya.export.api.conf.MergeConfigurationException; -import org.apache.rya.export.api.conf.policy.TimestampPolicyMergeConfiguration.TimestampPolicyBuilder; - -/** - * Helper for creating the immutable application configuration that uses - * Accumulo. - */ -public class TimestampPolicyConfigurationAdapter extends ConfigurationAdapter { - /** - * @param jConfig - The JAXB generated configuration. - * @return The {@link MergeConfiguration} used in the application - * @throws MergeConfigurationException - */ - @Override - public MergeConfiguration createConfig(final MergeToolConfiguration jConfig) throws MergeConfigurationException { - final TimestampMergePolicyConfiguration timeConfig = (TimestampMergePolicyConfiguration) jConfig; - final MergeConfiguration.Builder configBuilder = super.getBuilder(jConfig); - final TimestampPolicyBuilder builder = new TimestampPolicyBuilder(configBuilder); - builder.setToolStartTime(timeConfig.getToolStartTime()); - return builder.build(); - } -} diff --git a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/policy/TimestampPolicyMergeConfiguration.java b/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/policy/TimestampPolicyMergeConfiguration.java deleted file mode 100644 index 91bd4c2c6..000000000 --- a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/policy/TimestampPolicyMergeConfiguration.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.rya.export.api.conf.policy; - -import static java.util.Objects.requireNonNull; - -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; - -import org.apache.rya.export.api.conf.MergeConfiguration; -import org.apache.rya.export.api.conf.MergeConfigurationDecorator; -import org.apache.rya.export.api.conf.MergeConfigurationException; - -/** - * Configuration for merging Rya Statements using timestamp. - */ -public class TimestampPolicyMergeConfiguration extends MergeConfigurationDecorator { - private final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMddHHmmssSSSz"); - private final Date toolStartTime; - - /** - * Creates a new {@link TimestampPolicyMergeConfiguration} - * @param builder - Builder used to create the configuration. - * @throws MergeConfigurationException - The provided time is not formatted properly. - */ - public TimestampPolicyMergeConfiguration(final TimestampPolicyBuilder builder) throws MergeConfigurationException { - super(builder); - requireNonNull(builder.toolStartTime); - try { - toolStartTime = dateFormat.parse(builder.toolStartTime); - } catch (final ParseException e) { - throw new MergeConfigurationException("Cannot parse the configured start time.", e); - } - } - - /** - * @return The time of the data to be included in the copy/merge process. - */ - public Date getToolStartTime() { - return toolStartTime; - } - - /** - * Builder to help create {@link MergeConfiguration}s. - */ - public static class TimestampPolicyBuilder extends Builder { - private String toolStartTime; - - public TimestampPolicyBuilder(final Builder builder) { - super(builder); - } - - /** - * @param toolStartTime - The time of the data to be included in the - * copy/merge process. - * @return the updated {@link Builder}. - */ - public Builder setToolStartTime(final String toolStartTime) { - this.toolStartTime = toolStartTime; - return this; - } - } -} diff --git a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/metadata/MergeParentMetadata.java b/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/metadata/MergeParentMetadata.java index 262007e79..9487acd22 100644 --- a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/metadata/MergeParentMetadata.java +++ b/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/metadata/MergeParentMetadata.java @@ -32,7 +32,7 @@ public class MergeParentMetadata { private final String ryaInstanceName; private final Date timestamp; - private final Date filterTimestamp; + private final long filterTimestamp; private final Long parentTimeOffset; /** @@ -44,7 +44,7 @@ public class MergeParentMetadata { * @param parentTimeOffset - The parent time offset metadata key for the * table. */ - public MergeParentMetadata(final String ryaInstanceName, final Date timestamp, final Date filterTimestamp, final Long parentTimeOffset) { + public MergeParentMetadata(final String ryaInstanceName, final Date timestamp, final long filterTimestamp, final Long parentTimeOffset) { this.ryaInstanceName = checkNotNull(ryaInstanceName); this.timestamp = checkNotNull(timestamp); this.filterTimestamp = filterTimestamp; @@ -69,7 +69,7 @@ public Date getTimestamp() { * @return - The timestamp used by the copy tool to filter which data was * included when copying. */ - public Date getFilterTimestamp() { + public long getFilterTimestamp() { return filterTimestamp; } @@ -107,7 +107,7 @@ public int hashCode() { public static class Builder { private String name; private Date timestamp; - private Date filterTimestamp; + private long filterTimestamp; private Long parentTimeOffset; public Builder setRyaInstanceName(final String name) { @@ -120,7 +120,7 @@ public Builder setTimestamp(final Date timestamp) { return this; } - public Builder setFilterTimestmap(final Date filterTimestamp) { + public Builder setFilterTimestmap(final long filterTimestamp) { this.filterTimestamp = checkNotNull(filterTimestamp); return this; } diff --git a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/policy/TimestampPolicyStatementStore.java b/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/policy/TimestampPolicyStatementStore.java similarity index 61% rename from extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/policy/TimestampPolicyStatementStore.java rename to extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/policy/TimestampPolicyStatementStore.java index 79ac19d2a..adbd30fc3 100644 --- a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/conf/policy/TimestampPolicyStatementStore.java +++ b/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/policy/TimestampPolicyStatementStore.java @@ -16,38 +16,26 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.rya.export.api.conf.policy; +package org.apache.rya.export.api.policy; -import static java.util.Objects.requireNonNull; - -import java.util.Date; -import java.util.Iterator; - -import org.apache.rya.api.domain.RyaStatement; -import org.apache.rya.export.api.store.FetchStatementException; import org.apache.rya.export.api.store.RyaStatementStore; import org.apache.rya.export.api.store.RyaStatementStorePolicy; /** * Statement Store decorated to fetch statements based on a timestamp. */ -public abstract class TimestampPolicyStatementStore extends RyaStatementStorePolicy { - protected final Date timestamp; +public class TimestampPolicyStatementStore extends RyaStatementStorePolicy { + + protected final long timestamp; /** * Creates a new {@link TimestampPolicyStatementStore} * @param store - The {@link RyaStatementStore} to decorate - * @param timestamp - The timestamp to fetch statements based on. + * @param timestamp - The timestamp from which all parent statements will be merged into the child. */ - public TimestampPolicyStatementStore(final RyaStatementStore store, final Date timestamp) { + public TimestampPolicyStatementStore(final RyaStatementStore store, final long timestamp) { super(store); - this.timestamp = requireNonNull(timestamp); - } - /** - * The statements fetched will have been inserted into the statement store after - * the specified timestamp. - */ - @Override - public abstract Iterator fetchStatements() throws FetchStatementException; + this.timestamp = timestamp; + } } diff --git a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/store/RyaStatementStore.java b/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/store/RyaStatementStore.java index 68998daa2..d4768fb27 100644 --- a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/store/RyaStatementStore.java +++ b/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/store/RyaStatementStore.java @@ -38,6 +38,10 @@ * some storage system that is used when merging in data or exporting data. */ public interface RyaStatementStore { + /** + * @return - The name of the Rya Instance this store connects to. + */ + public String getRyaInstanceName(); /** * @return an {@link Iterator} containing all {@link RyaStatement}s found @@ -53,6 +57,12 @@ public interface RyaStatementStore { */ public void addStatement(final RyaStatement statement) throws AddStatementException; + /** + * @param statements - The {@link RyaStatement}s to add to this {@link RyaStatementStore}. + * @throws AddStatementException Thrown when adding a statement fails. + */ + public void addStatements(final Iterator statements) throws AddStatementException; + /** * @param statement - The {@link RyaStatement} to remove from this {@link RyaStatementStore}. * @throws RemoveStatementException - Thrown when the statement is not removed @@ -87,4 +97,9 @@ public interface RyaStatementStore { * @throws ParentMetadataExistsException */ public void setParentMetadata(MergeParentMetadata metadata) throws ParentMetadataExistsException; + + /** + * @return - The number of statements this store is expected to return during a merge. + */ + public long count(); } diff --git a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/store/RyaStatementStorePolicy.java b/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/store/RyaStatementStorePolicy.java index d7d2a4dad..cb532a96d 100644 --- a/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/store/RyaStatementStorePolicy.java +++ b/extras/rya.export/export.api/src/main/java/org/apache/rya/export/api/store/RyaStatementStorePolicy.java @@ -53,6 +53,11 @@ public void addStatement(final RyaStatement statement) throws AddStatementExcept store.addStatement(statement); } + @Override + public void addStatements(final Iterator statements) throws AddStatementException { + store.addStatements(statements); + } + @Override public void removeStatement(final RyaStatement statement) throws RemoveStatementException { store.removeStatement(statement); @@ -77,4 +82,14 @@ public Optional getParentMetadata() { public void setParentMetadata(final MergeParentMetadata metadata) throws ParentMetadataExistsException { store.setParentMetadata(metadata); } + + @Override + public String getRyaInstanceName() { + return store.getRyaInstanceName(); + } + + @Override + public long count() { + return store.count(); + } } diff --git a/extras/rya.export/export.client/conf/config.out.xml b/extras/rya.export/export.client/conf/config.out.xml new file mode 100644 index 000000000..e69de29bb diff --git a/extras/rya.export/export.client/conf/config.xml b/extras/rya.export/export.client/conf/config.xml index 879240cce..3f0a29581 100644 --- a/extras/rya.export/export.client/conf/config.xml +++ b/extras/rya.export/export.client/conf/config.xml @@ -16,20 +16,20 @@ KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. --> - 10.10.10.100 - accumuloUsername - accumuloPassword - accumuloInstance - rya_demo_export_ - http://10.10.10.100:8080 - accumulo - 1111 - 10.10.10.101 - rya_demo_child - rya_demo_export_ - http://10.10.10.101:8080 - mongo - 27017 - timestamp + + sia-2-query + + 127.0.0.1 + 27017 + + + + dest + + 127.0.0.1 + 27017 + + + http://127.0.0.1:8080 false \ No newline at end of file diff --git a/extras/rya.export/export.client/conf/config_accumulo-accumulo.xml b/extras/rya.export/export.client/conf/config_accumulo-accumulo.xml new file mode 100644 index 000000000..64ca36fb8 --- /dev/null +++ b/extras/rya.export/export.client/conf/config_accumulo-accumulo.xml @@ -0,0 +1,48 @@ + + + + + + merge_parent + 127.0.0.1 + 1111 + rya_ + + root + + U + mock + + + + + merge_child + 127.0.0.1 + 1111 + rya_ + + root + + U + mock + + + http://127.0.0.1:8080 + false + 123456 + \ No newline at end of file diff --git a/extras/rya.export/export.client/conf/config_accumulo-mongo.xml b/extras/rya.export/export.client/conf/config_accumulo-mongo.xml new file mode 100644 index 000000000..5f777d2c7 --- /dev/null +++ b/extras/rya.export/export.client/conf/config_accumulo-mongo.xml @@ -0,0 +1,43 @@ + + + + + + merge_parent + 127.0.0.1 + 1111 + rya_ + + root + + U + mock + + + + + merge_child + 127.0.0.1 + 27017 + rya + + + http://127.0.0.1:8080 + false + 123456 + \ No newline at end of file diff --git a/extras/rya.export/export.client/conf/config_mongo-accumulo.xml b/extras/rya.export/export.client/conf/config_mongo-accumulo.xml new file mode 100644 index 000000000..da263cbe8 --- /dev/null +++ b/extras/rya.export/export.client/conf/config_mongo-accumulo.xml @@ -0,0 +1,43 @@ + + + + + + merge_parent + 127.0.0.1 + 27017 + rya + + + + + merge_child + 127.0.0.1 + 1111 + rya_ + + root + + U + mock + + + http://127.0.0.1:8080 + false + 123456 + \ No newline at end of file diff --git a/extras/rya.export/export.client/conf/config_mongo-mongo.xml b/extras/rya.export/export.client/conf/config_mongo-mongo.xml new file mode 100644 index 000000000..f023e552b --- /dev/null +++ b/extras/rya.export/export.client/conf/config_mongo-mongo.xml @@ -0,0 +1,38 @@ + + + + + + merge_parent + 127.0.0.1 + 27017 + rya + + + + + merge_child + 127.0.0.1 + 27017 + rya + + + http://127.0.0.1:8080 + false + 123456 + \ No newline at end of file diff --git a/extras/rya.export/export.client/pom.xml b/extras/rya.export/export.client/pom.xml index c942702bf..2b6530dba 100644 --- a/extras/rya.export/export.client/pom.xml +++ b/extras/rya.export/export.client/pom.xml @@ -72,6 +72,12 @@ under the License. rdf4j-rio-turtle + + com.toedter + jcalendar + 1.4 + + junit @@ -82,28 +88,40 @@ under the License. - - maven-assembly-plugin + org.codehaus.mojo + jaxb2-maven-plugin + + + xjc + + xjc + + + + + org.apache.rya.export + + + + + com.mycila + license-maven-plugin - - - true - custom - WEB-INF/lib/$${artifact.groupIdPath}/$${artifact.artifactId}-$${artifact.version}$${dashClassifier?}.$${artifact.extension} - org.apache.rya.export.client.MergeDriverClient - - - - jar-with-dependencies - +

${project.basedir}/src/license/header.txt
- make-assembly - package + update-generated-source-headers + + ${project.build.directory}/generated-sources + + XML_STYLE + + + process-sources - single + format diff --git a/extras/rya.export/export.api/src/main/xsd/TimestampMergePolicyConfiguration.xsd b/extras/rya.export/export.client/src/license/header.txt similarity index 50% rename from extras/rya.export/export.api/src/main/xsd/TimestampMergePolicyConfiguration.xsd rename to extras/rya.export/export.client/src/license/header.txt index 181691a31..90705e02e 100644 --- a/extras/rya.export/export.api/src/main/xsd/TimestampMergePolicyConfiguration.xsd +++ b/extras/rya.export/export.client/src/license/header.txt @@ -1,5 +1,3 @@ - - - - - - - - - - - - - - - - \ No newline at end of file +under the License. \ No newline at end of file diff --git a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/MergeDriverClient.java b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/MergeDriverCLI.java similarity index 65% rename from extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/MergeDriverClient.java rename to extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/MergeDriverCLI.java index 2a902a7fc..bfbf0f81b 100644 --- a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/MergeDriverClient.java +++ b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/MergeDriverCLI.java @@ -18,9 +18,6 @@ */ package org.apache.rya.export.client; -import static org.apache.rya.export.DBType.ACCUMULO; -import static org.apache.rya.export.MergePolicy.TIMESTAMP; - import java.io.File; import java.io.IOException; import java.net.UnknownHostException; @@ -33,17 +30,15 @@ import org.apache.log4j.Logger; import org.apache.log4j.xml.DOMConfigurator; import org.apache.rya.api.path.PathUtils; -import org.apache.rya.export.accumulo.AccumuloRyaStatementStore; +import org.apache.rya.export.MergeToolConfiguration; import org.apache.rya.export.api.MergerException; -import org.apache.rya.export.api.conf.MergeConfiguration; -import org.apache.rya.export.api.conf.MergeConfigurationException; -import org.apache.rya.export.api.conf.policy.TimestampPolicyMergeConfiguration; +import org.apache.rya.export.api.policy.TimestampPolicyStatementStore; import org.apache.rya.export.api.store.RyaStatementStore; import org.apache.rya.export.client.conf.MergeConfigurationCLI; +import org.apache.rya.export.client.conf.MergeConfigurationException; import org.apache.rya.export.client.conf.TimeUtils; -import org.apache.rya.export.client.merge.MemoryTimeMerger; +import org.apache.rya.export.client.merge.MemoryMerger; import org.apache.rya.export.client.merge.StatementStoreFactory; -import org.apache.rya.export.client.merge.VisibilityStatementMerger; import org.apache.rya.rdftriplestore.inference.InferenceEngineException; import org.eclipse.rdf4j.query.MalformedQueryException; import org.eclipse.rdf4j.query.UpdateExecutionException; @@ -55,9 +50,9 @@ /** * Drives the MergeTool. */ -public class MergeDriverClient { - private static final Logger LOG = Logger.getLogger(MergeDriverClient.class); - private static MergeConfiguration configuration; +public class MergeDriverCLI { + private static final Logger LOG = Logger.getLogger(MergeDriverCLI.class); + private static MergeToolConfiguration configuration; public static void main(final String [] args) throws ParseException, MergeConfigurationException, UnknownHostException, MergerException, @@ -83,10 +78,10 @@ public static void main(final String [] args) throws ParseException, LOG.error("Configuration failed.", e); } - final boolean useTimeSync = configuration.getUseNtpServer(); + final boolean useTimeSync = configuration.isUseNtpServer(); Optional offset = Optional.absent(); if (useTimeSync) { - final String tomcat = configuration.getChildTomcatUrl(); + final String tomcat = configuration.getTomcatUrl(); final String ntpHost = configuration.getNtpServerHost(); try { offset = Optional.fromNullable(TimeUtils.getNtpServerAndMachineTimeDifference(ntpHost, tomcat)); @@ -97,41 +92,27 @@ public static void main(final String [] args) throws ParseException, final StatementStoreFactory storeFactory = new StatementStoreFactory(configuration); try { - final RyaStatementStore parentStore = storeFactory.getParentStatementStore(); - final RyaStatementStore childStore = storeFactory.getChildStatementStore(); + final RyaStatementStore parentStore = storeFactory.getStatementStore(configuration.getParent()); + final RyaStatementStore childStore = storeFactory.getStatementStore(configuration.getChild()); LOG.info("Starting Merge Tool"); - if(configuration.getParentDBType() == ACCUMULO && configuration.getChildDBType() == ACCUMULO) { - final AccumuloRyaStatementStore childAStore = (AccumuloRyaStatementStore) childStore; - final AccumuloRyaStatementStore parentAStore = (AccumuloRyaStatementStore) parentStore; - //do map reduce merging. - //TODO: Run Merger + final TimestampPolicyStatementStore parentTimestampStore = new TimestampPolicyStatementStore(parentStore, configuration.getStartTime()); + + final Long timeOffset; + if (offset.isPresent()) { + timeOffset = offset.get(); } else { - if(configuration.getMergePolicy() == TIMESTAMP) { - final TimestampPolicyMergeConfiguration timeConfig = (TimestampPolicyMergeConfiguration) configuration; - final Long timeOffset; - if (offset.isPresent()) { - timeOffset = offset.get(); - } else { - timeOffset = 0L; - } - final MemoryTimeMerger merger = new MemoryTimeMerger(parentStore, childStore, - new VisibilityStatementMerger(), timeConfig.getToolStartTime(), - configuration.getParentRyaInstanceName(), timeOffset); - merger.runJob(); - } + timeOffset = 0L; } + final MemoryMerger merger = new MemoryMerger(parentTimestampStore, childStore, + configuration.getParent().getRyaInstanceName(), timeOffset); + merger.runJob(); } catch (final Exception e) { LOG.error("Something went wrong creating a Rya Statement Store connection.", e); } - Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { - @Override - public void uncaughtException(final Thread thread, final Throwable throwable) { - LOG.error("Uncaught exception in " + thread.getName(), throwable); - } - }); + Thread.setDefaultUncaughtExceptionHandler((thread, throwable) -> LOG.error("Uncaught exception in " + thread.getName(), throwable)); LOG.info("Finished running Merge Tool"); System.exit(1); diff --git a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/RepSynchDriver.java b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/RepSynchDriver.java new file mode 100644 index 000000000..e8e81c078 --- /dev/null +++ b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/RepSynchDriver.java @@ -0,0 +1,117 @@ +package org.apache.rya.export.client; + +import java.awt.Dimension; +import java.awt.Toolkit; +import java.io.File; +import java.io.IOException; +import java.text.ParseException; + +import javax.swing.JFrame; +import javax.swing.UIManager; +import javax.swing.plaf.nimbus.NimbusLookAndFeel; + +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.BasicConfigurator; +import org.apache.log4j.Logger; +import org.apache.log4j.xml.DOMConfigurator; +import org.apache.rya.api.path.PathUtils; +import org.apache.rya.export.MergeToolConfiguration; +import org.apache.rya.export.api.store.RyaStatementStore; +import org.apache.rya.export.client.conf.MergeConfigurationCLI; +import org.apache.rya.export.client.conf.MergeConfigurationException; +import org.apache.rya.export.client.conf.TimeUtils; +import org.apache.rya.export.client.merge.StatementStoreFactory; +import org.apache.rya.export.client.view.RepSynchToolPane; + +import com.google.common.base.Optional; + +/** + * GUI based entry point into the rep/synch tool. + */ +public class RepSynchDriver { + private static final long serialVersionUID = 1L; + private static final Logger LOG = Logger.getLogger(RepSynchDriver.class); + private static MergeToolConfiguration configuration; + + public static void main(final String[] args) throws MergeConfigurationException, ParseException { + //configure logging + final String log4jConfiguration = System.getProperties().getProperty("log4j.configuration"); + if (StringUtils.isNotBlank(log4jConfiguration)) { + final String parsedConfiguration = PathUtils.clean(StringUtils.removeStart(log4jConfiguration, "file:")); + final File configFile = new File(parsedConfiguration); + if (configFile.exists()) { + DOMConfigurator.configure(parsedConfiguration); + } else { + BasicConfigurator.configure(); + } + } + + //setup rep/synch configs. + final MergeConfigurationCLI config = new MergeConfigurationCLI(args); + try { + configuration = config.createConfiguration(); + } catch (final MergeConfigurationException e) { + LOG.error("Configuration failed.", e); + } + + final boolean useTimeSync = configuration.isUseNtpServer(); + Optional offset = Optional.absent(); + if (useTimeSync) { + final String tomcat = configuration.getTomcatUrl(); + final String ntpHost = configuration.getNtpServerHost(); + try { + offset = Optional.fromNullable(TimeUtils.getNtpServerAndMachineTimeDifference(ntpHost, tomcat)); + } catch (final IOException e) { + LOG.error("Unable to get time difference between time server: " + ntpHost + " and the server: " + tomcat, e); + } + } + + //create statement stores + final StatementStoreFactory storeFactory = new StatementStoreFactory(configuration); + try { + final RyaStatementStore parentStore = storeFactory.getStatementStore(configuration.getParent()); + final RyaStatementStore childStore = storeFactory.getStatementStore(configuration.getChild()); + + final Long timeOffset; + if (offset.isPresent()) { + timeOffset = offset.get(); + } else { + timeOffset = 0L; + } + + LOG.info("Starting Merge Tool"); + + //Setup view + UIManager.setLookAndFeel(new NimbusLookAndFeel()); + + final RepSynchToolPane repSynchPane = new RepSynchToolPane(parentStore, childStore, configuration.getParent().getRyaInstanceName(), timeOffset); + final Dimension largeDim = new Dimension(600, 500); + final Dimension smallDim = new Dimension(600, 150); + final JFrame frame = new JFrame(); + frame.setContentPane(repSynchPane); + frame.setVisible(true); + frame.setSize(smallDim); + frame.setTitle("Rya Replication/Synchronization Tool"); + frame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE); + //frame.setIconImage(/*rya icon!*/); + repSynchPane.addResizeListener(logShown -> { + if(logShown) { + frame.setSize(largeDim); + frame.repaint(); + } else { + frame.setSize(smallDim); + frame.repaint(); + } + }); + + final Dimension screen = Toolkit.getDefaultToolkit().getScreenSize(); + frame.setLocation( + (int)Math.round(screen.getHeight() / 2 - 250), + (int)Math.round(screen.getHeight() / 2 - 150)); + + } catch (final Exception e) { + LOG.error("Something went wrong creating a Rya Statement Store connection.", e); + } + Thread.setDefaultUncaughtExceptionHandler((thread, throwable) -> LOG.error("Uncaught exception in " + thread.getName(), throwable)); + } +} diff --git a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/conf/MergeConfigHadoopAdapter.java b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/conf/MergeConfigHadoopAdapter.java index 3b722f16a..be4c5edc9 100644 --- a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/conf/MergeConfigHadoopAdapter.java +++ b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/conf/MergeConfigHadoopAdapter.java @@ -19,18 +19,18 @@ package org.apache.rya.export.client.conf; import org.apache.hadoop.conf.Configuration; -import org.apache.rya.export.api.conf.MergeConfiguration; +import org.apache.rya.export.Mongo; import org.apache.rya.mongodb.MongoDBRdfConfiguration; /** * Adapts the {@link MergeConfiguration} to the hadoop {@link Configuration}. */ public class MergeConfigHadoopAdapter { - public static MongoDBRdfConfiguration getMongoConfiguration(final MergeConfiguration config) { + public static MongoDBRdfConfiguration getMongoConfiguration(final Mongo mongo, final String ryaInstanceName) { final MongoDBRdfConfiguration configuration = new MongoDBRdfConfiguration(); - configuration.setMongoHostname(config.getChildHostname()); - configuration.set(MongoDBRdfConfiguration.MONGO_PORT, config.getChildPort() + ""); - configuration.set(MongoDBRdfConfiguration.MONGO_DB_NAME, config.getChildRyaInstanceName()); + configuration.setMongoHostname(mongo.getHostname()); + configuration.set(MongoDBRdfConfiguration.MONGO_PORT, mongo.getPort() + ""); + configuration.set(MongoDBRdfConfiguration.RYA_INSTANCE_NAME, ryaInstanceName); return configuration; } } diff --git a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/conf/MergeConfigurationCLI.java b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/conf/MergeConfigurationCLI.java index 8f39bd3cd..afbd39031 100644 --- a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/conf/MergeConfigurationCLI.java +++ b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/conf/MergeConfigurationCLI.java @@ -19,13 +19,11 @@ package org.apache.rya.export.client.conf; import static com.google.common.base.Preconditions.checkNotNull; -import static org.apache.rya.export.MergePolicy.TIMESTAMP; import java.io.File; import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; -import java.util.Date; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; @@ -41,17 +39,11 @@ import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.rya.api.utils.XmlFactoryConfiguration; -import org.apache.rya.export.AccumuloMergeToolConfiguration; -import org.apache.rya.export.DBType; +import org.apache.rya.export.Accumulo; +import org.apache.rya.export.Connection; import org.apache.rya.export.InstanceType; -import org.apache.rya.export.MergePolicy; import org.apache.rya.export.MergeToolConfiguration; -import org.apache.rya.export.TimestampMergePolicyConfiguration; -import org.apache.rya.export.api.conf.AccumuloMergeConfiguration; -import org.apache.rya.export.api.conf.ConfigurationAdapter; -import org.apache.rya.export.api.conf.MergeConfiguration; -import org.apache.rya.export.api.conf.MergeConfigurationException; -import org.apache.rya.export.api.conf.policy.TimestampPolicyMergeConfiguration; +import org.apache.rya.export.Mongo; import org.xml.sax.SAXException; import com.google.common.annotations.VisibleForTesting; @@ -61,35 +53,13 @@ */ public class MergeConfigurationCLI { private static final Option CONFIG_OPTION = new Option("c", true, "Defines the configuration file for the Merge Tool to use."); - private static final Option TIME_OPTION = new Option("t", true, "Defines the timestamp from which to filter RyaStatements when merging."); - private static final Option PARENT_HOST_OPTION = new Option("a", "pHost", true, "Defines the hostname of the parent db to connect to."); - private static final Option PARENT_USER_OPTION = new Option("b", "pUser", true, "Defines the username to connect to the parent DB."); - private static final Option PARENT_PSWD_OPTION = new Option("d", "pPswd", true, "Defines the password to connect to the parent DB."); - private static final Option PARENT_RYA_OPTION = new Option("e", "pRya", true, "Defines the rya instance name of the parent DB."); - private static final Option PARENT_PREFIX_OPTION = new Option("f", "pPrefix", true, "Defines the table prefix of the parent DB."); - private static final Option PARENT_TOMCAT_OPTION = new Option("g", "pTomcat", true, "Defines the location of Tomcat for the parent DB."); - private static final Option PARENT_DB_OPTION = new Option("h", "pDB", true, "Defines the type of database the parent is."); - private static final Option PARENT_PORT_OPTION = new Option("i", "pPort", true, "Defines the port of the parent DB to connect to."); - private static final Option PARENT_ACCUMULO_ZOOKEEPERS_OPTION = new Option("j", "paZookeepers", true, "Defines the location of the zookeepers."); - private static final Option PARENT_ACCUMULO_AUTHS_OPTION = new Option("k", "paAuths", true, "Defines the authorization level of the user."); - private static final Option PARENT_ACCUMULO_TYPE_OPTION = new Option("l", "paType", true, "Defines the type of accumulo to connect to."); - private static final Option CHILD_HOST_OPTION = new Option("m", "cHost", true, "Defines the hostname of the child db to connect to."); - private static final Option CHILD_USER_OPTION = new Option("n", "cUser", true, "Defines the username to connect to the child DB."); - private static final Option CHILD_PSWD_OPTION = new Option("o", "cPswd", true, "Defines the password to connect to the child DB."); - private static final Option CHILD_RYA_OPTION = new Option("p", "cRya", true, "Defines the rya instance name of the child DB."); - private static final Option CHILD_PREFIX_OPTION = new Option("q", "cPrefix", true, "Defines the table prefix of the child DB."); - private static final Option CHILD_TOMCAT_OPTION = new Option("r", "cTomcat", true, "Defines the location of Tomcat for the child DB."); - private static final Option CHILD_DB_OPTION = new Option("s", "cDB", true, "Defines the type of database the child is."); - private static final Option CHILD_PORT_OPTION = new Option("u", "cPort", true, "Defines the port of the child DB to connect to."); - private static final Option CHILD_ACCUMULO_ZOOKEEPERS_OPTION = new Option("v", "caZookeepers", true, "Defines the location of the zookeepers."); - private static final Option CHILD_ACCUMULO_AUTHS_OPTION = new Option("w", "caAuths", true, "Defines the authorization level of the user."); - private static final Option CHILD_ACCUMULO_TYPE_OPTION = new Option("x", "caType", true, "Defines the type of accumulo to connect to."); - private static final Option MERGE_OPTION = new Option("y", "merge", true, "Defines the type of merging that should occur."); - private static final Option NTP_OPTION = new Option("z", "useNTP", true, "Defines if NTP should be used to synch time."); + /** + * Formatted in "MMM ddd yyy HH:mm:ss" + */ public static final DateFormat DATE_FORMAT = new SimpleDateFormat("MMM ddd yyy HH:mm:ss"); private final CommandLine cmd; - private MergeConfiguration configuration; + private MergeToolConfiguration configuration; /** * * @param args @@ -113,38 +83,13 @@ public MergeConfigurationCLI(final String[] args) throws MergeConfigurationExcep @VisibleForTesting public static Options getOptions() { final Options cliOptions = new Options() - .addOption(TIME_OPTION) - .addOption(CONFIG_OPTION) - .addOption(PARENT_DB_OPTION) - .addOption(PARENT_HOST_OPTION) - .addOption(PARENT_PORT_OPTION) - .addOption(PARENT_PREFIX_OPTION) - .addOption(PARENT_PSWD_OPTION) - .addOption(PARENT_RYA_OPTION) - .addOption(PARENT_TOMCAT_OPTION) - .addOption(PARENT_USER_OPTION) - .addOption(PARENT_ACCUMULO_AUTHS_OPTION) - .addOption(PARENT_ACCUMULO_TYPE_OPTION) - .addOption(PARENT_ACCUMULO_ZOOKEEPERS_OPTION) - .addOption(CHILD_DB_OPTION) - .addOption(CHILD_HOST_OPTION) - .addOption(CHILD_PORT_OPTION) - .addOption(CHILD_PREFIX_OPTION) - .addOption(CHILD_PSWD_OPTION) - .addOption(CHILD_RYA_OPTION) - .addOption(CHILD_TOMCAT_OPTION) - .addOption(CHILD_USER_OPTION) - .addOption(CHILD_ACCUMULO_AUTHS_OPTION) - .addOption(CHILD_ACCUMULO_TYPE_OPTION) - .addOption(CHILD_ACCUMULO_ZOOKEEPERS_OPTION) - .addOption(MERGE_OPTION) - .addOption(NTP_OPTION); + .addOption(CONFIG_OPTION); return cliOptions; } public static MergeToolConfiguration createConfigurationFromFile(final File configFile) throws MergeConfigurationException { try { - final JAXBContext context = JAXBContext.newInstance(DBType.class, MergeToolConfiguration.class, AccumuloMergeToolConfiguration.class, TimestampMergePolicyConfiguration.class, MergePolicy.class, InstanceType.class); + final JAXBContext context = JAXBContext.newInstance(MergeToolConfiguration.class, Mongo.class, Accumulo.class, Connection.class, InstanceType.class); final Unmarshaller unmarshaller = context.createUnmarshaller(); final DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); XmlFactoryConfiguration.harden(dbf); @@ -154,26 +99,6 @@ public static MergeToolConfiguration createConfigurationFromFile(final File conf throw new MergeConfigurationException("Failed to create a config based on the provided configuration.", JAXBe); } } - /** - * Gets and parses the time from the configuration or command line. - * This is for API and is not used internally as of v3.2.11 - * @return a time or throws an exception when no time is configured because time is required. - * @throws MergeConfigurationException when date is badly formated or missing. - */ - public Date getRyaStatementMergeTime() throws MergeConfigurationException { - final Date time; - if(cmd.hasOption(TIME_OPTION.getOpt())) { - final String dateStr = cmd.getOptionValue(TIME_OPTION.getOpt()); - try { - time = DATE_FORMAT.parse(dateStr); - } catch (final java.text.ParseException e) { - throw new MergeConfigurationException("The provided timestamp was not formatted correctly.", e); - } - } else { - throw new MergeConfigurationException("The "+TIME_OPTION.getArgName()+" option was not specified. "+TIME_OPTION.getDescription()); - } - return time; - } /** * Attempts to create the {@link MergeConfiguration} based on the provided @@ -182,54 +107,18 @@ public Date getRyaStatementMergeTime() throws MergeConfigurationException { * @throws MergeConfigurationException - Thrown when the provided file is * not formatted properly for a {@link MergeConfiguration}. */ - public MergeConfiguration createConfiguration() throws MergeConfigurationException { + public MergeToolConfiguration createConfiguration() throws MergeConfigurationException { if(configuration == null) { //If the config option is present, ignore all other options. if(cmd.hasOption(CONFIG_OPTION.getOpt())) { final File xml = new File(cmd.getOptionValue(CONFIG_OPTION.getOpt())); - final ConfigurationAdapter adapter = new ConfigurationAdapter(); - configuration = adapter.createConfig(createConfigurationFromFile(xml)); + configuration = createConfigurationFromFile(xml); } else { - final DBType parentType = DBType.fromValue(cmd.getOptionValue(PARENT_DB_OPTION.getLongOpt())); - final DBType childType = DBType.fromValue(cmd.getOptionValue(CHILD_DB_OPTION.getLongOpt())); - final MergePolicy mergePolicy = MergePolicy.fromValue(cmd.getOptionValue(MERGE_OPTION.getLongOpt())); - MergeConfiguration.Builder builder = new MergeConfiguration.Builder() - .setParentHostname(cmd.getOptionValue(PARENT_HOST_OPTION.getLongOpt())) - .setParentUsername(cmd.getOptionValue(PARENT_USER_OPTION.getLongOpt())) - .setParentPassword(cmd.getOptionValue(PARENT_PSWD_OPTION.getLongOpt())) - .setParentRyaInstanceName(cmd.getOptionValue(PARENT_RYA_OPTION.getLongOpt())) - .setParentTablePrefix(cmd.getOptionValue(PARENT_PREFIX_OPTION.getLongOpt())) - .setParentTomcatUrl(cmd.getOptionValue(PARENT_TOMCAT_OPTION.getLongOpt())) - .setParentDBType(parentType) - .setParentPort(Integer.parseInt(cmd.getOptionValue(PARENT_PORT_OPTION.getLongOpt()))) - .setChildHostname(cmd.getOptionValue(CHILD_HOST_OPTION.getLongOpt())) - .setChildUsername(cmd.getOptionValue(CHILD_USER_OPTION.getLongOpt())) - .setChildPassword(cmd.getOptionValue(CHILD_PSWD_OPTION.getLongOpt())) - .setChildRyaInstanceName(cmd.getOptionValue(CHILD_RYA_OPTION.getLongOpt())) - .setChildTablePrefix(cmd.getOptionValue(CHILD_PREFIX_OPTION.getLongOpt())) - .setChildTomcatUrl(cmd.getOptionValue(CHILD_TOMCAT_OPTION.getLongOpt())) - .setChildDBType(childType) - .setChildPort(Integer.parseInt(cmd.getOptionValue(CHILD_PORT_OPTION.getLongOpt()))) - .setMergePolicy(mergePolicy); - if (mergePolicy == TIMESTAMP) { - builder = new TimestampPolicyMergeConfiguration.TimestampPolicyBuilder(builder) - .setToolStartTime(cmd.getOptionValue(TIME_OPTION.getLongOpt())); - } - if (parentType == DBType.ACCUMULO) { - builder = new AccumuloMergeConfiguration.AccumuloBuilder(builder) - .setParentZookeepers(cmd.getOptionValue(PARENT_ACCUMULO_ZOOKEEPERS_OPTION.getLongOpt())) - .setParentAuths(cmd.getOptionValue(PARENT_ACCUMULO_AUTHS_OPTION.getLongOpt())) - .setParentInstanceType(InstanceType.fromValue(cmd.getOptionValue(PARENT_ACCUMULO_TYPE_OPTION.getLongOpt()))); - } - if (childType == DBType.ACCUMULO) { - builder = new AccumuloMergeConfiguration.AccumuloBuilder(builder) - .setChildZookeepers(cmd.getOptionValue(CHILD_ACCUMULO_ZOOKEEPERS_OPTION.getLongOpt())) - .setChildAuths(cmd.getOptionValue(CHILD_ACCUMULO_AUTHS_OPTION.getLongOpt())) - .setChildInstanceType(InstanceType.fromValue(cmd.getOptionValue(CHILD_ACCUMULO_TYPE_OPTION.getLongOpt()))); - } - configuration = builder.build(); + throw new MergeConfigurationException("Must include a configuration file."); } } return configuration; } + + } diff --git a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/conf/MergeConfigurationException.java b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/conf/MergeConfigurationException.java new file mode 100644 index 000000000..f5f583982 --- /dev/null +++ b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/conf/MergeConfigurationException.java @@ -0,0 +1,30 @@ +package org.apache.rya.export.client.conf; + +/** + * Exception thrown when a problem occurs configuratin the merge tool. + */ +public class MergeConfigurationException extends Exception { + private static final long serialVersionUID = 1L; + + /** + * @param message - The error message. + */ + public MergeConfigurationException(final String message) { + super(message); + } + + /** + * @param cause - The cause of the exception + */ + public MergeConfigurationException(final Throwable cause) { + super(cause); + } + + /** + * @param message - The error message. + * @param cause - The cause of the exception + */ + public MergeConfigurationException(final String message, final Throwable cause) { + super(message, cause); + } +} diff --git a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/MemoryMerger.java b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/MemoryMerger.java new file mode 100644 index 000000000..4b25a94f0 --- /dev/null +++ b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/MemoryMerger.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.rya.export.client.merge; + +import static com.google.common.base.Preconditions.checkNotNull; +import static java.util.Objects.requireNonNull; + +import java.util.Date; +import java.util.Optional; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.apache.log4j.Logger; +import org.apache.rya.api.domain.RyaStatement; +import org.apache.rya.export.api.Merger; +import org.apache.rya.export.api.StatementMerger; +import org.apache.rya.export.api.metadata.MergeParentMetadata; +import org.apache.rya.export.api.metadata.ParentMetadataExistsException; +import org.apache.rya.export.api.store.AddStatementException; +import org.apache.rya.export.api.store.ContainsStatementException; +import org.apache.rya.export.api.store.FetchStatementException; +import org.apache.rya.export.api.store.RemoveStatementException; +import org.apache.rya.export.api.store.RyaStatementStore; +import org.apache.rya.export.client.merge.future.RepSynchRunnable; +import org.apache.rya.export.client.merge.future.RepSynchRunnableFactory; +import org.apache.rya.export.client.merge.future.RepSynchRunnableFactory.StatementListener; + +/** + * An in memory {@link Merger}. Merges {@link RyaStatement}s from a parent + * to a child. The statements merged will be any that have a timestamp after + * the provided time. If there are any conflicting statements, the provided + * {@link StatementMerger} will merge the statements and produce the desired + * {@link RyaStatement}. + */ +public class MemoryMerger implements Merger { + private static final Logger LOG = Logger.getLogger(MemoryMerger.class); + + private final RyaStatementStore sourceStore; + private final RyaStatementStore destStore; + private final String ryaInstanceName; + private final Long timeOffset; + + private final RepSynchRunnableFactory factory; + private final ExecutorService repSynchExecutor; + + private RepSynchRunnable repSynchTask; + private Future future; + + /** + * Creates a new {@link MemoryMerger} to merge the statements from the parent to a child. + * + * @param parentStore - The source store, where the statements are coming from. (not null) + * @param childStore - The destination store, where the statements are going. (not null) + * @param ryaInstanceName - The Rya instance to merge. (not null) + * @param timeOffset - The offset from a potential NTP server. (not null) + */ + public MemoryMerger(final RyaStatementStore parentStore, final RyaStatementStore childStore, + final String ryaInstanceName, final Long timeOffset) { + sourceStore = checkNotNull(parentStore); + destStore = checkNotNull(childStore); + this.ryaInstanceName = checkNotNull(ryaInstanceName); + this.timeOffset = checkNotNull(timeOffset); + + factory = RepSynchRunnableFactory.getInstance(); + repSynchExecutor = Executors.newSingleThreadExecutor(); + } + + @Override + public void runJob() { + final Optional metadata = sourceStore.getParentMetadata(); + + //check the source for a parent metadata + if(metadata.isPresent()) { + LOG.info("Importing statements..."); + final MergeParentMetadata parentMetadata = metadata.get(); + if(parentMetadata.getRyaInstanceName().equals(ryaInstanceName)) { + try { + importStatements(parentMetadata); + } catch (AddStatementException | ContainsStatementException | RemoveStatementException | FetchStatementException e) { + LOG.error("Failed to import statements.", e); + } + } + } else { + final Optional synchMetadata = destStore.getParentMetadata(); + //if the metadata exists, the destination store is going to be synched from the source store if the source is the original parent + if(synchMetadata.isPresent() && + synchMetadata.get().getRyaInstanceName().equals(sourceStore.getRyaInstanceName())) { + LOG.info("Synching child store..."); + synchExistingStore(); + + } else { + try { + LOG.info("Replicating store..."); + replicate(); + } catch (final ParentMetadataExistsException | FetchStatementException e) { + LOG.error("Failed to replicate store.", e); + } + } + } + } + + /** + * Exports all statements after the provided timestamp. + * @throws ParentMetadataExistsException - + * @throws FetchStatementException + */ + private void replicate() throws ParentMetadataExistsException, FetchStatementException { + LOG.info("Creating parent metadata in the child."); + //setup parent metadata repo in the child + final MergeParentMetadata metadata = new MergeParentMetadata.Builder() + .setRyaInstanceName(ryaInstanceName) + .setTimestamp(new Date()) + .setParentTimeOffset(timeOffset) + .build(); + destStore.setParentMetadata(metadata); + + repSynchTask = factory.getReplication(sourceStore, destStore); + future = repSynchExecutor.submit(repSynchTask); + try { + future.get(); + } catch (InterruptedException | ExecutionException e) { + LOG.error("Failed to replicate Rya instance."); + } + } + + private void importStatements(final MergeParentMetadata metadata) throws AddStatementException, ContainsStatementException, RemoveStatementException, FetchStatementException { + LOG.info("Importing statements."); + repSynchTask = factory.getImport(sourceStore, destStore, metadata, timeOffset); + future = repSynchExecutor.submit(repSynchTask); + try { + future.get(); + } catch (InterruptedException | ExecutionException e) { + LOG.error("Failed to replicate Rya instance."); + } + } + + /** + * Synchronizes the source store with the destination store. + */ + private void synchExistingStore() { + repSynchTask = factory.getSynchronize(sourceStore, destStore, timeOffset); + future = repSynchExecutor.submit(repSynchTask); + try { + future.get(); + } catch (InterruptedException | ExecutionException e) { + LOG.error("Failed to replicate Rya instance."); + } + } + + /** + * Cancels the current running rep/synch job. + */ + public void cancel() { + if(repSynchTask != null) { + repSynchTask.cancel(); + future.cancel(true); + } + } + + /** + * @param listener - The listener added to be notified when a statement has been rep/synched. (not null) + */ + public void addStatementListener(final StatementListener listener) { + requireNonNull(listener); + factory.addStatementListener(listener); + } + + /** + * @param listener - The listener to remove. (not null) + */ + public void removeStatementListener(final StatementListener listener) { + requireNonNull(listener); + factory.removeStatementListener(listener); + } +} diff --git a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/MemoryTimeMerger.java b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/MemoryTimeMerger.java deleted file mode 100644 index dc0a148f6..000000000 --- a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/MemoryTimeMerger.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.rya.export.client.merge; - -import static com.google.common.base.Preconditions.checkNotNull; - -import java.util.Date; -import java.util.Iterator; -import java.util.Optional; - -import org.apache.log4j.Logger; -import org.apache.rya.api.domain.RyaStatement; -import org.apache.rya.export.api.Merger; -import org.apache.rya.export.api.StatementMerger; -import org.apache.rya.export.api.metadata.MergeParentMetadata; -import org.apache.rya.export.api.metadata.ParentMetadataExistsException; -import org.apache.rya.export.api.store.AddStatementException; -import org.apache.rya.export.api.store.ContainsStatementException; -import org.apache.rya.export.api.store.FetchStatementException; -import org.apache.rya.export.api.store.RemoveStatementException; -import org.apache.rya.export.api.store.RyaStatementStore; - -/** - * An in memory {@link Merger}. Merges {@link RyaStatement}s from a parent - * to a child. The statements merged will be any that have a timestamp after - * the provided time. If there are any conflicting statements, the provided - * {@link StatementMerger} will merge the statements and produce the desired - * {@link RyaStatement}. - */ -public class MemoryTimeMerger implements Merger { - private static final Logger LOG = Logger.getLogger(MemoryTimeMerger.class); - - private final RyaStatementStore parentStore; - private final RyaStatementStore childStore; - private final StatementMerger statementMerger; - private final Date timestamp; - private final String ryaInstanceName; - private final Long timeOffset; - - /** - * Creates a new {@link MemoryTimeMerger} to merge the statements from the parent to a child. - * @param parentStore - * @param childStore - * @param childMetadata - * @param parentMetadata - * @param statementMerger - * @param timestamp - The timestamp from which all parent statements will be merged into the child. - */ - public MemoryTimeMerger(final RyaStatementStore parentStore, final RyaStatementStore childStore, - final StatementMerger statementMerger, final Date timestamp, final String ryaInstanceName, - final Long timeOffset) { - this.parentStore = checkNotNull(parentStore); - this.childStore = checkNotNull(childStore); - this.statementMerger = checkNotNull(statementMerger); - this.timestamp = checkNotNull(timestamp); - this.ryaInstanceName = checkNotNull(ryaInstanceName); - this.timeOffset = checkNotNull(timeOffset); - } - - @Override - public void runJob() { - final Optional metadata = parentStore.getParentMetadata(); - - //check the parent for a parent metadata repo - if(metadata.isPresent()) { - LOG.info("Merging statements..."); - final MergeParentMetadata parentMetadata = metadata.get(); - if(parentMetadata.getRyaInstanceName().equals(ryaInstanceName)) { - try { - importStatements(parentMetadata); - } catch (AddStatementException | ContainsStatementException | RemoveStatementException | FetchStatementException e) { - LOG.error("Failed to import statements.", e); - } - } - } else { - try { - LOG.info("Cloning statements..."); - export(); - } catch (final ParentMetadataExistsException | FetchStatementException e) { - LOG.error("Failed to export statements.", e); - } - } - } - - /** - * Exports all statements after the provided timestamp. - * @throws ParentMetadataExistsException - - * @throws FetchStatementException - */ - private void export() throws ParentMetadataExistsException, FetchStatementException { - LOG.info("Creating parent metadata in the child."); - //setup parent metadata repo in the child - final MergeParentMetadata metadata = new MergeParentMetadata.Builder() - .setRyaInstanceName(ryaInstanceName) - .setTimestamp(new Date()) - .setParentTimeOffset(timeOffset) - .setFilterTimestmap(timestamp) - .build(); - childStore.setParentMetadata(metadata); - - //fetch all statements after timestamp from the parent - final Iterator statements = parentStore.fetchStatements(); - LOG.info("Exporting statements."); - while(statements.hasNext()) { - System.out.print("."); - final RyaStatement statement = statements.next(); - try { - childStore.addStatement(statement); - } catch (final AddStatementException e) { - LOG.error("Failed to add statement: " + statement + " to the statement store.", e); - } - } - } - - private void importStatements(final MergeParentMetadata metadata) throws AddStatementException, ContainsStatementException, RemoveStatementException, FetchStatementException { - LOG.info("Importing statements."); - final Iterator parentStatements = parentStore.fetchStatements(); - final Iterator childStatements = childStore.fetchStatements(); - //statements are in order by timestamp. - - //Remove statements that were removed in the child. - //after the timestamp has passed, there is no need to keep checking the parent - while(childStatements.hasNext()) { - final RyaStatement statement = childStatements.next(); - if(statement.getTimestamp() > metadata.getTimestamp().getTime()) { - break; - } - if(!parentStore.containsStatement(statement)) { - System.out.println(statement.toString()); - childStore.removeStatement(statement); - } - } - - long curTime = -1L; - //Add all of the child statements that are not in the parent - while(parentStatements.hasNext()) { - final RyaStatement statement = parentStatements.next(); - curTime = statement.getTimestamp(); - if(!childStore.containsStatement(statement)) { - statement.setTimestamp(statement.getTimestamp() - timeOffset); - childStore.addStatement(statement); - } - } - } -} diff --git a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/StatementStoreFactory.java b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/StatementStoreFactory.java index 536ea6045..593671e7b 100644 --- a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/StatementStoreFactory.java +++ b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/StatementStoreFactory.java @@ -20,21 +20,19 @@ import static java.util.Objects.requireNonNull; -import java.util.Date; - import org.apache.rya.accumulo.AccumuloRyaDAO; import org.apache.rya.api.persist.RyaDAOException; -import org.apache.rya.export.DBType; -import org.apache.rya.export.InstanceType; -import org.apache.rya.export.MergePolicy; +import org.apache.rya.export.Accumulo; +import org.apache.rya.export.Connection; +import org.apache.rya.export.MergeToolConfiguration; +import org.apache.rya.export.Mongo; import org.apache.rya.export.accumulo.AccumuloRyaStatementStore; +import org.apache.rya.export.accumulo.conf.InstanceType; import org.apache.rya.export.accumulo.policy.TimestampPolicyAccumuloRyaStatementStore; import org.apache.rya.export.accumulo.util.AccumuloInstanceDriver; -import org.apache.rya.export.api.conf.AccumuloMergeConfiguration; -import org.apache.rya.export.api.conf.MergeConfiguration; -import org.apache.rya.export.api.conf.policy.TimestampPolicyMergeConfiguration; import org.apache.rya.export.api.store.RyaStatementStore; import org.apache.rya.export.client.conf.MergeConfigHadoopAdapter; +import org.apache.rya.export.client.conf.MergeConfigurationException; import org.apache.rya.export.mongo.MongoRyaStatementStore; import org.apache.rya.export.mongo.policy.TimestampPolicyMongoRyaStatementStore; import org.apache.rya.mongodb.MongoDBRyaDAO; @@ -46,9 +44,9 @@ * Factory for creating {@link RyaStatementStore}s based on the {@link MergeConfiguration}. */ public class StatementStoreFactory { - private final MergeConfiguration configuration; + private final MergeToolConfiguration configuration; - public StatementStoreFactory(final MergeConfiguration configuration) { + public StatementStoreFactory(final MergeToolConfiguration configuration) { this.configuration = requireNonNull(configuration); } @@ -57,85 +55,44 @@ public StatementStoreFactory(final MergeConfiguration configuration) { * @return The created {@link RyaStatementStore} that connects to the Parent rya. * @throws Exception - Something went wrong creating the {@link RyaStatementStore}. */ - public RyaStatementStore getParentStatementStore() throws Exception { - final DBType dbType = configuration.getParentDBType(); - final String ryaInstanceName = configuration.getParentRyaInstanceName(); - RyaStatementStore store = getBaseStatementStore(dbType, - configuration.getParentHostname(), - configuration.getParentPort(), - ryaInstanceName, - configuration.getParentTablePrefix(), - configuration, true); - store = getMergePolicyStatementStore(store, - configuration.getMergePolicy(), - ryaInstanceName, - dbType); - return store; - } - - public RyaStatementStore getChildStatementStore() throws Exception { - final RyaStatementStore store = getBaseStatementStore(configuration.getChildDBType(), - configuration.getChildHostname(), - configuration.getChildPort(), - configuration.getChildRyaInstanceName(), - configuration.getChildTablePrefix(), - configuration, false); - return store; - } + public RyaStatementStore getStatementStore(final Connection connection) throws Exception { + final long timestamp = configuration.getStartTime(); + final String ryaInstanceName = connection.getRyaInstanceName(); - /** - * @param isParent - * @param config - * These parameters are hacks until the Accumulo DAO only accepts a connector. - * Once that happens this will be much, much cleaner, and make the {@link AccumuloInstanceDriver} - * obsolete. - * @throws Exception - */ - private RyaStatementStore getBaseStatementStore(final DBType dbType, - final String hostname, final int port, final String ryaInstancename, - final String tablePrefix, final MergeConfiguration config, final boolean isParent) throws Exception { - RyaStatementStore store; - if(dbType == DBType.MONGO) { - store = getBaseMongoStore(hostname, port, ryaInstancename); + if(connection.getAccumulo() != null) { + final Accumulo accumulo = connection.getAccumulo(); + final AccumuloRyaStatementStore store = getAccumuloStore(accumulo, ryaInstanceName); + return new TimestampPolicyAccumuloRyaStatementStore(store, timestamp); + } else if(connection.getMongo() != null) { + final Mongo mongo = connection.getMongo(); + final MongoRyaStatementStore store = getMongoStore(mongo, connection.getRyaInstanceName()); + return new TimestampPolicyMongoRyaStatementStore(store, timestamp); } else { - final AccumuloMergeConfiguration aConfig = (AccumuloMergeConfiguration) config; - final InstanceType type = isParent ? aConfig.getParentInstanceType() : aConfig.getChildInstanceType(); - store = getBaseAccumuloStore(ryaInstancename, type, isParent, ryaInstancename, tablePrefix, tablePrefix, tablePrefix, tablePrefix); - } - return store; - } - - private RyaStatementStore getMergePolicyStatementStore(final RyaStatementStore store, final MergePolicy policy, final String ryaInstanceName, final DBType dbType) { - RyaStatementStore policyStore = null; - if(policy == MergePolicy.TIMESTAMP) { - final TimestampPolicyMergeConfiguration timeConfig = (TimestampPolicyMergeConfiguration) configuration; - final Date timestamp = timeConfig.getToolStartTime(); - if(dbType == DBType.MONGO) { - policyStore = new TimestampPolicyMongoRyaStatementStore((MongoRyaStatementStore) store, timestamp, ryaInstanceName); - } else { - policyStore = new TimestampPolicyAccumuloRyaStatementStore((AccumuloRyaStatementStore) store, timestamp); - } + throw new MergeConfigurationException("No parent database was specified."); } - return policyStore == null ? store : policyStore; } - private MongoRyaStatementStore getBaseMongoStore(final String hostname, final int port, final String ryaInstanceName) throws RyaDAOException { - final MongoClient client = new MongoClient(hostname, port); + private MongoRyaStatementStore getMongoStore(final Mongo mongo, final String ryaInstanceName) throws RyaDAOException { + final MongoClient client = new MongoClient(mongo.getHostname(), mongo.getPort()); final MongoDBRyaDAO dao = new MongoDBRyaDAO(); - dao.setConf(new StatefulMongoDBRdfConfiguration(MergeConfigHadoopAdapter.getMongoConfiguration(configuration), client)); + dao.setConf(new StatefulMongoDBRdfConfiguration(MergeConfigHadoopAdapter.getMongoConfiguration(mongo, ryaInstanceName), client)); dao.init(); return new MongoRyaStatementStore(client, ryaInstanceName, dao); } - private AccumuloRyaStatementStore getBaseAccumuloStore(final String ryaInstanceName, - final InstanceType type, final boolean isParent, - final String username, final String password, final String tablePrefix, - final String auths, final String zookeepers) throws Exception { + private AccumuloRyaStatementStore getAccumuloStore(final Accumulo accumulo, final String ryaInstanceName) throws Exception { final AccumuloInstanceDriver aInstance = new AccumuloInstanceDriver( - ryaInstanceName+"_driver", type, true, false, isParent, username, - password, ryaInstanceName, tablePrefix, auths, zookeepers); + ryaInstanceName+"_driver", + InstanceType.valueOf(accumulo.getInstanceType().toString()), + true, false, accumulo.equals(configuration.getParent().getAccumulo()), + accumulo.getUsername(), + accumulo.getPassword(), + ryaInstanceName, + accumulo.getTablePrefix(), + accumulo.getAuths(), + accumulo.getZookeepers()); aInstance.setUp(); final AccumuloRyaDAO dao = aInstance.getDao(); - return new AccumuloRyaStatementStore(dao, tablePrefix, ryaInstanceName); + return new AccumuloRyaStatementStore(dao, accumulo.getTablePrefix(), ryaInstanceName); } } diff --git a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/future/Import.java b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/future/Import.java new file mode 100644 index 000000000..03ab47df2 --- /dev/null +++ b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/future/Import.java @@ -0,0 +1,74 @@ +package org.apache.rya.export.client.merge.future; + +import static java.util.Objects.requireNonNull; + +import java.util.Iterator; +import java.util.List; + +import org.apache.rya.api.domain.RyaStatement; +import org.apache.rya.export.api.metadata.MergeParentMetadata; +import org.apache.rya.export.api.store.ContainsStatementException; +import org.apache.rya.export.api.store.FetchStatementException; +import org.apache.rya.export.api.store.RyaStatementStore; +import org.apache.rya.export.client.merge.future.RepSynchRunnableFactory.StatementListener; + +/** + * Synchronizes an existing Rya instance with it's parent. + */ +class Import extends RepSynchRunnable { + private final MergeParentMetadata metadata; + private final long timeOffset; + + /** + * Constructs a new {@link Import}. + * @param sourceStore - The source {@link RyaStatementStore} where statements are pulled from. (not null) + * @param destStore - The destination {@link RyaStatementStore} to synchronize. (not null) + * @param metadata - The {@link MergeParentMetadata} defining the destination store. (not null) + * @param timeOffset - The offset between the 2 {@link RyaStatementStore}s time. + * @param listeners - The {@link StatementListener}s notified as statements are added/removed from {@link RyaStatementStore}s. (not null) + */ + public Import(final RyaStatementStore sourceStore, final RyaStatementStore destStore, final MergeParentMetadata metadata, final long timeOffset, final List listeners) { + super(sourceStore, destStore, listeners); + this.timeOffset = timeOffset; + this.metadata = requireNonNull(metadata); + } + + + @Override + public void run() { + try { + //statements are in order by timestamp. + final Iterator parentStatements = sourceStore.fetchStatements(); + final Iterator childStatements = destStore.fetchStatements(); + + //Remove statements that were removed in the child. + //after the timestamp has passed, there is no need to keep checking the parent + while(childStatements.hasNext() && !isCanceled) { + final RyaStatement statement = childStatements.next(); + if(statement.getTimestamp() > metadata.getTimestamp().getTime()) { + break; + } + if(!sourceStore.containsStatement(statement)) { + remove(destStore, statement); + } else { + updateCount(); + } + } + + //Add all of the child statements that are not in the parent + while(parentStatements.hasNext() && !isCanceled) { + final RyaStatement statement = parentStatements.next(); + if(!destStore.containsStatement(statement)) { + statement.setTimestamp(statement.getTimestamp() - timeOffset); + add(destStore, statement); + } else { + updateCount(); + } + } + } catch (final FetchStatementException e) { + e.printStackTrace(); + } catch (final ContainsStatementException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/future/RepSynchRunnable.java b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/future/RepSynchRunnable.java new file mode 100644 index 000000000..e479f3ea1 --- /dev/null +++ b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/future/RepSynchRunnable.java @@ -0,0 +1,93 @@ +package org.apache.rya.export.client.merge.future; + +import static java.util.Objects.requireNonNull; + +import java.util.List; + +import org.apache.rya.api.domain.RyaStatement; +import org.apache.rya.export.api.store.AddStatementException; +import org.apache.rya.export.api.store.RemoveStatementException; +import org.apache.rya.export.api.store.RyaStatementStore; +import org.apache.rya.export.client.merge.future.RepSynchRunnableFactory.StatementListener; + +/** + * A {@link Runable} used by the rep/synch tool. + * All cancel/done functionality is the same across + * all rep/synch futures. + */ +public abstract class RepSynchRunnable implements Runnable { + protected boolean isCanceled; + protected final RyaStatementStore sourceStore; + protected final RyaStatementStore destStore; + + private final List listeners; + + /** + * Constructs a new {@link RepSynchRunnable}. + * @param sourceStore - The source {@link RyaStatementStore} where statements are pulled from. (not null) + * @param destStore - The destination {@link RyaStatementStore} where statements are put to. (not null) + * @param listeners - The {@link StatementListener}s notified as statements are added/removed from {@link RyaStatementStore}s. (not null) + */ + public RepSynchRunnable(final RyaStatementStore sourceStore, final RyaStatementStore destStore, final List listeners) { + this.sourceStore = requireNonNull(sourceStore); + this.destStore = requireNonNull(destStore); + this.listeners = requireNonNull(listeners); + + isCanceled = false; + } + + /** + * Cancels the current rep/synch task. + */ + public void cancel() { + isCanceled = true; + } + + @Override + public abstract void run(); + + /** + * Adds and notifies the listeners that a statement was added. + * + * @param store - The {@link RyaStatementStore} to add a statement to. (not null) + * @param stmnt - The {@link RyaStatement} to add. (not null) + */ + protected void add(final RyaStatementStore store, final RyaStatement stmnt) { + try { + store.addStatement(stmnt); + for(final StatementListener listen : listeners) { + listen.notifyAddStatement(store, stmnt); + } + } catch (final AddStatementException e) { + e.printStackTrace(); + } + } + + /** + * Removes and notifies the listeners that a statement was removed. + * + * @param store - The {@link RyaStatementStore} to remove a statement from. (not null) + * @param stmnt - The {@link RyaStatement} to remove. (not null) + */ + protected void remove(final RyaStatementStore store, final RyaStatement stmnt) { + requireNonNull(store); + requireNonNull(stmnt); + try { + store.removeStatement(stmnt); + for(final StatementListener listen : listeners) { + listen.notifyRemoveStatement(store, stmnt); + } + } catch (final RemoveStatementException e) { + e.printStackTrace(); + } + } + + /** + * Updates the count. A statement has been processed and may or may not have been added/removed + */ + protected void updateCount() { + for(final StatementListener listen : listeners) { + listen.updateCount(); + } + } +} \ No newline at end of file diff --git a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/future/RepSynchRunnableFactory.java b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/future/RepSynchRunnableFactory.java new file mode 100644 index 000000000..a3a74ecc8 --- /dev/null +++ b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/future/RepSynchRunnableFactory.java @@ -0,0 +1,114 @@ +package org.apache.rya.export.client.merge.future; + +import static java.util.Objects.requireNonNull; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.Future; + +import org.apache.rya.api.domain.RyaStatement; +import org.apache.rya.export.api.metadata.MergeParentMetadata; +import org.apache.rya.export.api.store.RyaStatementStore; + +/** + * Factory for creating various {@link Future}s used by the Rep/Synch tool. + */ +public class RepSynchRunnableFactory { + private final List listeners; + + /** + * Constructs a new {@link RepSynchRunnableFactory}. + */ + private RepSynchRunnableFactory() { + listeners = new ArrayList<>(); + } + + /** + * @return - an instance of {@link RepSynchRunnableFactory}. + */ + public static RepSynchRunnableFactory getInstance() { + return new RepSynchRunnableFactory(); + } + + /** + * Creates and returns a {@link Replication} runnable. + * @param sourceStore - The {@link RyaStatementStore} to replicate. (not null) + * @param destStore - The location to replicate to. (not null) + * @return The created {@link Replication}. + */ + public RepSynchRunnable getReplication(final RyaStatementStore sourceStore, final RyaStatementStore destStore) { + requireNonNull(sourceStore); + requireNonNull(destStore); + final Replication rep = new Replication(sourceStore, destStore, listeners); + return rep; + } + + /** + * Creates and returns an {@link Import} runnable. + * + * @param sourceStore - The {@link RyaStatementStore} the destination will be synchronized to. (not null) + * @param destStore - The {@link RyaStatementStore} to be synchronized. (not null) + * @param metadata - The {@link MergeParentMetadata} used to import {@link RyaStatement}s back into a Rya. (not null) + * @param timeOffset - The time difference between the different {@link RyaStatementStore}s. + * @return The created {@link Import}. + */ + public RepSynchRunnable getImport(final RyaStatementStore sourceStore, final RyaStatementStore destStore, final MergeParentMetadata metadata, final long timeOffset) { + requireNonNull(sourceStore); + requireNonNull(destStore); + requireNonNull(metadata); + final Import imp = new Import(sourceStore, destStore, metadata, timeOffset, listeners); + return imp; + } + + /** + * Creates and returns a {@link Synchronize} runnable. + * + * @param sourceStore - The {@link RyaStatementStore} the destination will be synchronized to. (not null) + * @param destStore - The {@link RyaStatementStore} to be synchronized. (not null) + * @param timeOffset - The time difference between the different {@link RyaStatementStore}s. + * @return The created {@link Synchronize}. + */ + public RepSynchRunnable getSynchronize(final RyaStatementStore sourceStore, final RyaStatementStore destStore, final long timeOffset) { + requireNonNull(sourceStore); + requireNonNull(destStore); + final Synchronize synch = new Synchronize(sourceStore, destStore, timeOffset, listeners); + return synch; + } + /** + * @param listener - The listener added to be notified when a statement has been rep/synched. (not null) + */ + public void addStatementListener(final StatementListener listener) { + requireNonNull(listener); + listeners.add(listener); + } + + /** + * @param listener - The listener to remove. (not null) + */ + public void removeStatementListener(final StatementListener listener) { + requireNonNull(listener); + listeners.remove(listener); + } + + /** + * Listener that notifies when a statement has been replicated. + */ + public interface StatementListener { + /** + * @param store - The store that a statement is being added to. (not null) + * @param statement - The statement that has been replicated. (not null) + */ + public void notifyAddStatement(final RyaStatementStore store, final RyaStatement statement); + + /** + * @param store - The store that a statement is being removed from. (not null) + * @param statement - The statmenet to remove. (not null) + */ + public void notifyRemoveStatement(final RyaStatementStore store, final RyaStatement statement); + + /** + * Updates the progress. + */ + public void updateCount(); + } +} diff --git a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/future/Replication.java b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/future/Replication.java new file mode 100644 index 000000000..9a2a6323a --- /dev/null +++ b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/future/Replication.java @@ -0,0 +1,44 @@ +package org.apache.rya.export.client.merge.future; + +import java.util.Iterator; +import java.util.List; + +import org.apache.log4j.Logger; +import org.apache.rya.api.domain.RyaStatement; +import org.apache.rya.export.api.store.FetchStatementException; +import org.apache.rya.export.api.store.RyaStatementStore; +import org.apache.rya.export.client.merge.future.RepSynchRunnableFactory.StatementListener; + +/** + * Replicates a rya instance. + */ +class Replication extends RepSynchRunnable { + private static final Logger LOG = Logger.getLogger(Replication.class); + + /** + * Constructs a new {@link Replication}. + * @param sourceStore - The source {@link RyaStatementStore} where statements are pulled from. (not null) + * @param destStore - The destination {@link RyaStatementStore} where statements are put to. (not null) + * @param listeners - The {@link StatementListener}s notified as statements are added/removed from {@link RyaStatementStore}s. (not null) + */ + public Replication(final RyaStatementStore sourceStore, final RyaStatementStore destStore, final List listeners) { + super(sourceStore, destStore, listeners); + } + + + @Override + public void run() { + //fetch all statements after timestamp from the parent + Iterator statements; + try { + statements = sourceStore.fetchStatements(); + LOG.info("Exporting statements."); + while(statements.hasNext() && !isCanceled) { + final RyaStatement statement = statements.next(); + super.add(destStore, statement); + } + } catch (final FetchStatementException e) { + e.printStackTrace(); + } + } +} \ No newline at end of file diff --git a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/future/Synchronize.java b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/future/Synchronize.java new file mode 100644 index 000000000..a6000f6fd --- /dev/null +++ b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/merge/future/Synchronize.java @@ -0,0 +1,67 @@ +package org.apache.rya.export.client.merge.future; + +import java.util.Iterator; +import java.util.List; + +import org.apache.log4j.Logger; +import org.apache.rya.api.domain.RyaStatement; +import org.apache.rya.export.api.store.ContainsStatementException; +import org.apache.rya.export.api.store.FetchStatementException; +import org.apache.rya.export.api.store.RyaStatementStore; +import org.apache.rya.export.client.merge.future.RepSynchRunnableFactory.StatementListener; + +/** + * Synchronizes an existing Rya instance with it's parent. + */ +class Synchronize extends RepSynchRunnable { + private static final Logger LOG = Logger.getLogger(Synchronize.class); + private final long timeOffset; + + /** + * Constructs a new {@link Synchronize}. + * @param sourceStore - The source {@link RyaStatementStore} where statements are pulled from. (not null) + * @param destStore - The destination {@link RyaStatementStore} to synchronize. (not null) + * @param timeOffset - The offset between the 2 {@link RyaStatementStore}s time. + * @param listeners - The {@link StatementListener}s notified as statements are added/removed from {@link RyaStatementStore}s. (not null) + */ + public Synchronize(final RyaStatementStore sourceStore, final RyaStatementStore destStore, final long timeOffset, final List listeners) { + super(sourceStore, destStore, listeners); + this.timeOffset = timeOffset; + } + + + @Override + public void run() { + LOG.info("Synching " + destStore.getRyaInstanceName() + " child store."); + try { + final Iterator sourceStatements = sourceStore.fetchStatements(); + final Iterator destinationStatements = destStore.fetchStatements(); + + //Remove statements that were removed from the parent. + while(destinationStatements.hasNext() && !isCanceled) { + final RyaStatement statement = destinationStatements.next(); + if(!sourceStore.containsStatement(statement)) { + remove(destStore, statement); + } else { + updateCount(); + } + } + + //Add all of the new statements in the parent store that aren't in the child + while(sourceStatements.hasNext() && !isCanceled) { + final RyaStatement statement = sourceStatements.next(); + + if(!destStore.containsStatement(statement)) { + statement.setTimestamp(statement.getTimestamp() - timeOffset); + add(destStore, statement); + } else { + updateCount(); + } + } + } catch (final FetchStatementException e) { + LOG.error("Failed to fetch statements from the Rya Store.", e); + } catch (final ContainsStatementException e) { + LOG.error("Unable to detect statement in the Rya Store.", e); + } + } +} \ No newline at end of file diff --git a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/view/ConfigPane.java b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/view/ConfigPane.java new file mode 100644 index 000000000..28b23359a --- /dev/null +++ b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/view/ConfigPane.java @@ -0,0 +1,156 @@ +package org.apache.rya.export.client.view; + +import static java.util.Calendar.HOUR; +import static java.util.Calendar.MINUTE; +import static java.util.Calendar.SECOND; +import static java.util.Objects.requireNonNull; + +import java.awt.GridBagConstraints; +import java.awt.GridBagLayout; +import java.awt.Insets; +import java.util.Calendar; +import java.util.Date; +import java.util.EventListener; + +import javax.swing.JButton; +import javax.swing.JLabel; +import javax.swing.JPanel; +import javax.swing.JSpinner; +import javax.swing.JSpinner.DateEditor; +import javax.swing.JTextField; +import javax.swing.SpinnerDateModel; +import javax.swing.event.DocumentEvent; +import javax.swing.event.DocumentListener; +import javax.swing.text.DateFormatter; + +import com.toedter.calendar.JDateChooser; + +/** + * A panel to pick a date/time combination to use when + * replicating/synchronizing a rya instance. + */ +public class ConfigPane extends JPanel { + private static final long serialVersionUID = 1L; + + private final JDateChooser calendarField; + private final JSpinner timeField; + private final SpinnerDateModel dateModel; + private final DateEditor timeEditor; + + private final JButton nextButton; + + /** + * Constructs a new {@link ConfigPane}. + */ + public ConfigPane() { + calendarField = new JDateChooser(); + + dateModel = new SpinnerDateModel(); + dateModel.setCalendarField(Calendar.SECOND); + timeField = new JSpinner(dateModel); + timeEditor = new DateEditor(timeField, "HH:mm:ss"); + + final DateFormatter formatter = (DateFormatter) timeEditor.getTextField().getFormatter(); + formatter.setAllowsInvalid(false); + formatter.setOverwriteMode(true); + + nextButton = new JButton("Run"); + nextButton.setEnabled(false); + nextButton.setToolTipText("Run the replication/synchronization"); + nextButton.addActionListener(e -> { + final RunMergeListener[] listeners = listenerList.getListeners(RunMergeListener.class); + for(final RunMergeListener listen : listeners) { + listen.notifyRun(); + } + }); + + //nextbutton enabled state + final JTextField cF = (JTextField) calendarField.getDateEditor().getUiComponent(); + cF.getDocument().addDocumentListener(new DocumentListener() { + @Override + public void removeUpdate(final DocumentEvent e) {} + @Override + public void changedUpdate(final DocumentEvent e) { } + @Override + public void insertUpdate(final DocumentEvent e) { + //since the time config field can't be checked, only the calendar field will be used to check state + nextButton.setEnabled(calendarField.getCalendar() != null); + } + }); + + initComponents(); + } + + /** + * @return - The configured Date/Time to use when replicating/synchronizing. + */ + public Date getDateTime() { + final Calendar date = calendarField.getCalendar(); + final Date chosenTime = dateModel.getDate(); + date.set(HOUR, chosenTime.getHours()); + date.set(MINUTE, chosenTime.getMinutes()); + date.set(SECOND, chosenTime.getSeconds()); + return date.getTime(); + } + + /** + * @param listen - Adds the provided {@link RunMergeListener} + * to be notified when the rep/synch should be run. (not null) + */ + public void addRunMergeListener(final RunMergeListener listen) { + requireNonNull(listen); + listenerList.add(RunMergeListener.class, listen); + } + + /** + * @param listen - removes the provided {@link RunMergeListener} + * from this {@link ConfigPane}. (not null) + */ + public void remove(final RunMergeListener listen) { + requireNonNull(listen); + listenerList.remove(RunMergeListener.class, listen); + } + + /** + * Initialize the starting view state. + */ + private void initComponents() { + setLayout(new GridBagLayout()); + final GridBagConstraints g = new GridBagConstraints(); + g.gridx = 0; + g.gridy = 0; + g.weightx = 0; + g.weighty = 1.0; + g.gridheight = 1; + g.gridwidth = 1; + g.insets = new Insets(0, 25, 0, 0); + final JLabel cloneTimeLabel = new JLabel("Clone Date/Time: "); + add(cloneTimeLabel, g); + + g.gridx++; + g.weightx = 1.0; + g.fill = GridBagConstraints.HORIZONTAL; + g.insets = new Insets(0, 0, 0, 25); + add(calendarField, g); + + g.gridx++; + add(timeEditor, g); + + g.gridy++; + g.weightx = 1.0; + g.fill = GridBagConstraints.NONE; + g.insets = new Insets(0, 0, 0, 0); + add(nextButton, g); + } + + /** + * Listener to be notified when the merge tool should be run against + * the configured fields in the {@link ConfigPane}. + */ + interface RunMergeListener extends EventListener { + /** + * Notifies the listener that the merge tool should run with the current configuration. + */ + public void notifyRun(); + } +} diff --git a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/view/LogPane.java b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/view/LogPane.java new file mode 100644 index 000000000..00965cf06 --- /dev/null +++ b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/view/LogPane.java @@ -0,0 +1,214 @@ +package org.apache.rya.export.client.view; + +import static java.util.Objects.requireNonNull; +import static javax.swing.ScrollPaneConstants.HORIZONTAL_SCROLLBAR_AS_NEEDED; +import static javax.swing.ScrollPaneConstants.VERTICAL_SCROLLBAR_AS_NEEDED; + +import java.awt.GridBagConstraints; +import java.awt.GridBagLayout; +import java.awt.Insets; +import java.util.EventListener; + +import javax.swing.JButton; +import javax.swing.JLabel; +import javax.swing.JPanel; +import javax.swing.JProgressBar; +import javax.swing.JScrollPane; +import javax.swing.JTextArea; + +/** + * View for logging the Rya Merge Tool as it runs. + */ +public class LogPane extends JPanel { + private static final long serialVersionUID = 1L; + + private final JProgressBar progressBar; + private final JLabel statementLabel; + private final JScrollPane scroll; + private final JTextArea logArea; + private final JButton showLogButton; + private final JButton cancelButton; + private final JButton doneButton; + + /** + * Constructs a new {@link LogPane}. + */ + public LogPane() { + progressBar = new JProgressBar(); + logArea = new JTextArea(); + scroll = new JScrollPane(logArea); + statementLabel = new JLabel("Replicated statements: "); + cancelButton = new JButton("Cancel"); + doneButton = new JButton("Done"); + cancelButton.setToolTipText("Cancels the current running replication/synchronization"); + + cancelButton.addActionListener(e -> { + cancelButton.setEnabled(false); + doneButton.setEnabled(true); + + final CancelListener[] listeners = listenerList.getListeners(CancelListener.class); + for(final CancelListener listen : listeners) { + listen.notifyCancel(); + } + }); + + showLogButton = new JButton("Show Statements"); + showLogButton.addActionListener(e -> { + scroll.setVisible(!scroll.isVisible()); + statementLabel.setVisible(scroll.isVisible()); + + final ResizeListener[] listeners = listenerList.getListeners(ResizeListener.class); + for(final ResizeListener listen : listeners) { + listen.notifyResize(scroll.isVisible()); + } + + showLogButton.setText(logArea.isVisible() ? "Hide Statements" : "Show Statements"); + }); + + progressBar.addChangeListener(e -> { + if(progressBar.getValue() == progressBar.getMaximum()) { + doneButton.setEnabled(true); + } + }); + + doneButton.addActionListener(e -> { + System.exit(1); + }); + + initComponents(); + } + + /** + * @param maxProgress - The total number of statements to process. + */ + public void setMaxProgress(final int maxProgress) { + if(maxProgress == -1) { + progressBar.setIndeterminate(true); + } else { + progressBar.setMaximum(maxProgress); + } + } + + public void updateProgress() { + if(!progressBar.isIndeterminate()) { + progressBar.setValue(progressBar.getValue() + 1); + } + } + + /** + * @param log - The log to display in the log area. (not null) + */ + public void addLog(final String log) { + requireNonNull(log); + logArea.append(log + "\n"); + + updateProgress(); + } + + /** + * @param listen + */ + public void addCancelListener(final CancelListener listen) { + requireNonNull(listen); + listenerList.add(CancelListener.class, listen); + } + + /** + * @param listen - Removes the provided CancelListener + * from this pane to no longer be notified to cancel the merge. (not null) + */ + public void removeCancelListener(final CancelListener listen) { + requireNonNull(listen); + listenerList.remove(CancelListener.class, listen); + } + + /** + * @param listen - The {@link ResizeListener} to add. (not null) + */ + public void addResizeListener(final ResizeListener listen) { + requireNonNull(listen); + listenerList.add(ResizeListener.class, listen); + } + + /** + * @param listen - Removes the provided ResizeListener. (not null) + */ + public void removeResizeListener(final ResizeListener listen) { + requireNonNull(listen); + listenerList.remove(ResizeListener.class, listen); + } + + /** + * Initialize the starting view state. + */ + private void initComponents() { + setLayout(new GridBagLayout()); + final GridBagConstraints g = new GridBagConstraints(); + g.gridx = 0; + g.gridy = 0; + g.weightx = 1.0; + g.weighty = 0; + g.gridheight = 1; + + g.fill = GridBagConstraints.HORIZONTAL; + g.gridwidth = GridBagConstraints.REMAINDER; + g.insets = new Insets(25, 25, 0, 25); + add(progressBar, g); + + g.gridy++; + g.gridwidth = 1; + g.fill = GridBagConstraints.NONE; + g.anchor = GridBagConstraints.WEST; + add(statementLabel, g); + statementLabel.setVisible(false); + + g.gridy++; + g.weighty = 1.0; + g.gridwidth = GridBagConstraints.REMAINDER; + g.fill = GridBagConstraints.BOTH; + g.insets = new Insets(0, 25, 0, 25); + add(scroll, g); + scroll.setHorizontalScrollBarPolicy(HORIZONTAL_SCROLLBAR_AS_NEEDED); + scroll.setVerticalScrollBarPolicy(VERTICAL_SCROLLBAR_AS_NEEDED); + scroll.setVisible(false); + logArea.setEditable(false); + + g.gridy++; + g.weighty = 0; + g.weightx = 0; + g.gridwidth = 1; + g.fill = GridBagConstraints.NONE; + g.anchor = GridBagConstraints.WEST; + add(cancelButton, g); + + g.gridx++; + g.anchor = GridBagConstraints.EAST; + add(showLogButton, g); + + g.gridy++; + g.insets = new Insets(0, 0, 25, 25); + add(doneButton, g); + doneButton.setEnabled(false); + } + + /** + * Listener to used to notify when the window should resize. + */ + public interface ResizeListener extends EventListener { + /** + * Notification that the window should be resized. + * @param logShown - Whether or not the statement log is shown. + */ + public void notifyResize(final boolean logShown); + } + + /** + * Listener to be used to notify when a merge/synch/clone should be canceled. + */ + interface CancelListener extends EventListener { + /** + * Notifies the listener that the current running merge should be canceled. + */ + public void notifyCancel(); + } +} diff --git a/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/view/RepSynchToolPane.java b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/view/RepSynchToolPane.java new file mode 100644 index 000000000..1ff428c19 --- /dev/null +++ b/extras/rya.export/export.client/src/main/java/org/apache/rya/export/client/view/RepSynchToolPane.java @@ -0,0 +1,150 @@ +package org.apache.rya.export.client.view; + +import static java.util.Objects.requireNonNull; + +import java.awt.GridBagConstraints; +import java.awt.GridBagLayout; +import java.util.Date; + +import javax.swing.JPanel; +import javax.swing.SwingUtilities; +import javax.swing.SwingWorker; + +import org.apache.rya.api.domain.RyaStatement; +import org.apache.rya.export.api.policy.TimestampPolicyStatementStore; +import org.apache.rya.export.api.store.RyaStatementStore; +import org.apache.rya.export.client.merge.MemoryMerger; +import org.apache.rya.export.client.merge.future.RepSynchRunnableFactory.StatementListener; +import org.apache.rya.export.client.view.LogPane.CancelListener; +import org.apache.rya.export.client.view.LogPane.ResizeListener; + +/** + * Displays the configuration and logging during a merge. + */ +public class RepSynchToolPane extends JPanel { + private static final long serialVersionUID = 1L; + + private final ConfigPane configPane; + private final LogPane logPane; + + private final RyaStatementStore sourceStore; + private final RyaStatementStore destStore; + private final String ryaInstanceName; + private final long timeOffset; + + /** + * Constructs a new {@link RepSynchToolPane}. + * + * @param sourceStore - The source store of the rep/synch. (not null) + * @param destStore - The destination store of the rep/synch. (not null) + * @param ryaInstanceName - The Rya instance to perfrom the rep/synch over. (not null) + * @param timeOffset - The server time offset. + */ + public RepSynchToolPane(final RyaStatementStore sourceStore, final RyaStatementStore destStore, final String ryaInstanceName, + final long timeOffset) { + this.sourceStore = requireNonNull(sourceStore); + this.destStore = requireNonNull(destStore); + this.ryaInstanceName = requireNonNull(ryaInstanceName); + this.timeOffset = timeOffset; + + configPane = new ConfigPane(); + logPane = new LogPane(); + + //when next is clicked, move the divider. + configPane.addRunMergeListener(() -> { + configPane.setVisible(false); + logPane.setVisible(true); + + runMerger(configPane.getDateTime()); + }); + + initComponents(); + } + + private void runMerger(final Date dateTime) { + final TimestampPolicyStatementStore sourceTimestampStore = new TimestampPolicyStatementStore(sourceStore, dateTime.getTime()); + final MemoryMerger merger = new MemoryMerger(sourceTimestampStore, destStore, ryaInstanceName, timeOffset); + + + final CancelListener cancelListen = () -> merger.cancel(); + logPane.addCancelListener(cancelListen); + //find the number of statements to process. + long max = sourceTimestampStore.count(); + if(max != -1) { + final long destCount = destStore.count(); + if(destCount != -1) { + max += destCount; + } + } + + logPane.setMaxProgress((int)max); + + final StatementListener listen = new StatementListener() { + @Override + public void notifyAddStatement(final RyaStatementStore store, final RyaStatement statement) { + SwingUtilities.invokeLater(() -> logPane.addLog( + String.format("Adding: %s, %s, %s", + statement.getSubject().getData(), + statement.getPredicate().getData(), + statement.getObject().getData()))); + } + + @Override + public void notifyRemoveStatement(final RyaStatementStore store, final RyaStatement statement) { + SwingUtilities.invokeLater(() -> logPane.addLog( + String.format("Removing: %s, %s, %s", + statement.getSubject().getData(), + statement.getPredicate().getData(), + statement.getObject().getData()))); + } + + @Override + public void updateCount() { + logPane.updateProgress(); + } + }; + merger.addStatementListener(listen); + + final SwingWorker repSynchWorker = new SwingWorker() { + @Override + protected Void doInBackground() throws Exception { + merger.runJob(); + logPane.removeCancelListener(cancelListen); + return null; + } + }; + repSynchWorker.execute(); + } + + private void initComponents() { + setLayout(new GridBagLayout()); + final GridBagConstraints g = new GridBagConstraints(); + g.gridx = 0; + g.gridy = 0; + g.weightx = 1.0; + g.weighty = 1.0; + g.gridheight = 1; + g.gridwidth = 1; + g.fill = GridBagConstraints.BOTH; + add(configPane, g); + + add(logPane, g); + logPane.setVisible(false); + } + + /** + * @param listen - The {@link ResizeListener} to add. (not null) + */ + public void addResizeListener(final ResizeListener listen) { + requireNonNull(listen); + logPane.addResizeListener(listen); + } + + /** + * @param listen - Removes the provided ResizeListener. (not null) + */ + public void removeResizeListener(final ResizeListener listen) { + requireNonNull(listen); + logPane.removeResizeListener(listen); + } +} diff --git a/extras/rya.export/export.api/src/main/xsd/MergeConfiguration.xsd b/extras/rya.export/export.client/src/main/xsd/MergeConfiguration.xsd similarity index 52% rename from extras/rya.export/export.api/src/main/xsd/MergeConfiguration.xsd rename to extras/rya.export/export.client/src/main/xsd/MergeConfiguration.xsd index 7b84baf69..52baade61 100644 --- a/extras/rya.export/export.api/src/main/xsd/MergeConfiguration.xsd +++ b/extras/rya.export/export.client/src/main/xsd/MergeConfiguration.xsd @@ -35,44 +35,23 @@ under the License. - - - - - - - - - - - - - - - - - - - + + + + - - - - - - - - - - - - - + + + + + + + @@ -86,4 +65,37 @@ under the License. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/extras/rya.export/export.client/src/test/java/org/apache/rya/export/client/conf/MergeConfigurationCLITest.java b/extras/rya.export/export.client/src/test/java/org/apache/rya/export/client/conf/MergeConfigurationCLITest.java deleted file mode 100644 index 75ad02129..000000000 --- a/extras/rya.export/export.client/src/test/java/org/apache/rya/export/client/conf/MergeConfigurationCLITest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.rya.export.client.conf; - -import static org.junit.Assert.assertEquals; - -import java.io.File; - -import javax.xml.bind.JAXBException; - -import org.apache.rya.export.DBType; -import org.apache.rya.export.MergePolicy; -import org.apache.rya.export.MergeToolConfiguration; -import org.apache.rya.export.api.conf.MergeConfigurationException; -import org.junit.Test; - -public class MergeConfigurationCLITest { - @Test - public void testCreate1ConfigurationFromFile() throws MergeConfigurationException, JAXBException { - - final MergeToolConfiguration conf = MergeConfigurationCLI.createConfigurationFromFile(new File("conf/config.xml")); - assertEquals("10.10.10.100", conf.getParentHostname()); - assertEquals("accumuloUsername", conf.getParentUsername()); - assertEquals("accumuloPassword", conf.getParentPassword()); - assertEquals("accumuloInstance", conf.getParentRyaInstanceName()); - assertEquals("rya_demo_export_", conf.getParentTablePrefix()); - assertEquals("http://10.10.10.100:8080", conf.getParentTomcatUrl()); - assertEquals(DBType.ACCUMULO, conf.getParentDBType()); - assertEquals(1111, conf.getParentPort()); - assertEquals("10.10.10.101", conf.getChildHostname()); - assertEquals("rya_demo_child", conf.getChildRyaInstanceName()); - assertEquals("rya_demo_export_", conf.getChildTablePrefix()); - assertEquals("http://10.10.10.101:8080", conf.getChildTomcatUrl()); - assertEquals(DBType.MONGO, conf.getChildDBType()); - assertEquals(27017, conf.getChildPort()); - assertEquals(MergePolicy.TIMESTAMP, conf.getMergePolicy()); - assertEquals(Boolean.FALSE, conf.isUseNtpServer()); - assertEquals(null, conf.getNtpServerHost()); - } -} \ No newline at end of file diff --git a/extras/rya.export/export.integration/src/test/java/org/apache/rya/indexing/export/ITBase.java b/extras/rya.export/export.integration/src/test/java/org/apache/rya/indexing/export/ITBase.java index 4b7d3f9ad..3f49e6554 100644 --- a/extras/rya.export/export.integration/src/test/java/org/apache/rya/indexing/export/ITBase.java +++ b/extras/rya.export/export.integration/src/test/java/org/apache/rya/indexing/export/ITBase.java @@ -31,11 +31,10 @@ import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.AccumuloSecurityException; import org.apache.log4j.Logger; -import org.apache.rya.api.RdfCloudTripleStoreConfiguration; +import org.apache.rya.api.domain.RyaIRI; import org.apache.rya.api.domain.RyaStatement; import org.apache.rya.api.domain.RyaStatement.RyaStatementBuilder; import org.apache.rya.api.domain.RyaType; -import org.apache.rya.api.domain.RyaIRI; import org.apache.rya.api.persist.RyaDAOException; import org.apache.rya.api.resolver.RyaToRdfConversions; import org.apache.rya.indexing.accumulo.ConfigUtils; @@ -252,21 +251,14 @@ protected static MongoDBRdfConfiguration getConf(final String ryaInstanceName, conf.setBoolean(ConfigUtils.USE_MONGO, true); conf.setBoolean(ConfigUtils.USE_MOCK_INSTANCE, false); - conf.setTablePrefix(RYA_TABLE_PREFIX); - conf.setDisplayQueryPlan(true); conf.set(ConfigUtils.CLOUDBASE_USER, USER); conf.set(ConfigUtils.CLOUDBASE_PASSWORD, PASSWORD); - conf.set(MongoDBRdfConfiguration.MONGO_DB_NAME, "test"); - conf.set(MongoDBRdfConfiguration.MONGO_COLLECTION_PREFIX, "rya_"); - - conf.set(RdfCloudTripleStoreConfiguration.CONF_TBL_PREFIX, "rya_"); - conf.setMongoPort(""+port); conf.setMongoHostname(hostname); - conf.setMongoDBName(ryaInstanceName); + conf.setRyaInstanceName(ryaInstanceName); return conf; } diff --git a/extras/rya.export/export.integration/src/test/java/org/apache/rya/indexing/export/StoreToStoreIT.java b/extras/rya.export/export.integration/src/test/java/org/apache/rya/indexing/export/StoreToStoreIT.java index 9b6c0c08d..42232eed8 100644 --- a/extras/rya.export/export.integration/src/test/java/org/apache/rya/indexing/export/StoreToStoreIT.java +++ b/extras/rya.export/export.integration/src/test/java/org/apache/rya/indexing/export/StoreToStoreIT.java @@ -20,6 +20,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.util.ArrayList; import java.util.Collection; @@ -28,20 +29,23 @@ import java.util.List; import org.apache.rya.api.domain.RyaStatement; -import org.apache.rya.export.InstanceType; +import org.apache.rya.api.domain.RyaType; import org.apache.rya.export.accumulo.AccumuloRyaStatementStore; +import org.apache.rya.export.accumulo.conf.InstanceType; import org.apache.rya.export.accumulo.policy.TimestampPolicyAccumuloRyaStatementStore; import org.apache.rya.export.accumulo.util.AccumuloInstanceDriver; import org.apache.rya.export.api.metadata.ParentMetadataDoesNotExistException; import org.apache.rya.export.api.store.AddStatementException; import org.apache.rya.export.api.store.FetchStatementException; import org.apache.rya.export.api.store.RyaStatementStore; -import org.apache.rya.export.client.merge.MemoryTimeMerger; -import org.apache.rya.export.client.merge.VisibilityStatementMerger; +import org.apache.rya.export.client.merge.MemoryMerger; import org.apache.rya.export.mongo.MongoRyaStatementStore; import org.apache.rya.export.mongo.policy.TimestampPolicyMongoRyaStatementStore; import org.apache.rya.mongodb.MongoDBRyaDAO; import org.apache.rya.mongodb.StatefulMongoDBRdfConfiguration; +import org.eclipse.rdf4j.model.IRI; +import org.eclipse.rdf4j.model.ValueFactory; +import org.eclipse.rdf4j.model.impl.ValueFactoryImpl; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -53,6 +57,9 @@ @RunWith(Parameterized.class) public class StoreToStoreIT extends ITBase { + private static final ValueFactory VF = ValueFactoryImpl.getInstance(); + + private static final IRI ANY = VF.createIRI("http://www.w3.org/2001/XMLSchema#anyURI"); private static final String RYA_INSTANCE = "ryaInstance"; private static final InstanceType type = InstanceType.MOCK; private static final String tablePrefix = "accumuloTest"; @@ -62,30 +69,32 @@ public class StoreToStoreIT extends ITBase { private final RyaStatementStore childStore; private final static List clients = new ArrayList<>(); private final static List drivers = new ArrayList<>(); - private static Date currentDate; + private static long currentDate; - private static TimestampPolicyMongoRyaStatementStore getParentMongo() throws Exception { + private static RyaStatementStore getParentMongo() throws Exception { final MongoClient mongo = getNewMongoResources(RYA_INSTANCE); final MongoDBRyaDAO dao = new MongoDBRyaDAO(); dao.setConf(new StatefulMongoDBRdfConfiguration(ITBase.getConf(mongo), mongo)); dao.init(); + final MongoRyaStatementStore store = new MongoRyaStatementStore(mongo, RYA_INSTANCE, dao); - final TimestampPolicyMongoRyaStatementStore timeStore = new TimestampPolicyMongoRyaStatementStore(store, currentDate, RYA_INSTANCE); + final TimestampPolicyMongoRyaStatementStore timeStore = new TimestampPolicyMongoRyaStatementStore(store, currentDate); clients.add(mongo); return timeStore; } - private static MongoRyaStatementStore getChildMongo() throws Exception { + private static RyaStatementStore getChildMongo() throws Exception { final MongoClient mongo = getNewMongoResources(RYA_INSTANCE); final MongoDBRyaDAO dao = new MongoDBRyaDAO(); dao.setConf(new StatefulMongoDBRdfConfiguration(ITBase.getConf(mongo), mongo)); dao.init(); + final MongoRyaStatementStore store = new MongoRyaStatementStore(mongo, RYA_INSTANCE, dao); clients.add(mongo); return store; } - private static TimestampPolicyAccumuloRyaStatementStore getParentAccumulo() throws Exception { + private static RyaStatementStore getParentAccumulo() throws Exception { final AccumuloInstanceDriver driver = new AccumuloInstanceDriver(RYA_INSTANCE, type, true, false, true, "TEST1", PASSWORD, RYA_INSTANCE, tablePrefix, auths, ""); driver.setUp(); final AccumuloRyaStatementStore store = new AccumuloRyaStatementStore(driver.getDao(), tablePrefix, RYA_INSTANCE); @@ -93,7 +102,7 @@ private static TimestampPolicyAccumuloRyaStatementStore getParentAccumulo() thro return new TimestampPolicyAccumuloRyaStatementStore(store, currentDate); } - private static AccumuloRyaStatementStore getChildAccumulo() throws Exception { + private static RyaStatementStore getChildAccumulo() throws Exception { final AccumuloInstanceDriver driver = new AccumuloInstanceDriver(RYA_INSTANCE, type, true, false, false, "TEST2", PASSWORD, RYA_INSTANCE+"_child", tablePrefix, auths, ""); driver.setUp(); drivers.add(driver); @@ -115,6 +124,7 @@ public void cleanupTables() throws Exception { for(final AccumuloInstanceDriver driver : drivers) { driver.tearDown(); } + for(final MongoClient client : clients) { client.dropDatabase(RYA_INSTANCE); } @@ -125,6 +135,7 @@ public static void shutdown() throws Exception { for(final AccumuloInstanceDriver driver : drivers) { driver.tearDown(); } + for(final MongoClient client : clients) { client.close(); } @@ -132,7 +143,7 @@ public static void shutdown() throws Exception { @Parameterized.Parameters public static Collection instancesToTest() throws Exception { - currentDate = new Date(); + currentDate = new Date().getTime(); final Collection stores = new ArrayList<>(); stores.add(new Object[]{getParentMongo(), getChildMongo()}); stores.add(new Object[]{getParentMongo(), getChildAccumulo()}); @@ -141,18 +152,16 @@ public static Collection instancesToTest() throws Exception { return stores; } - public StoreToStoreIT(final RyaStatementStore parentStore, - final RyaStatementStore childStore) { + public StoreToStoreIT(final RyaStatementStore parentStore, final RyaStatementStore childStore) { this.parentStore = parentStore; this.childStore = childStore; } @Test public void cloneTest() throws AddStatementException, FetchStatementException, ParentMetadataDoesNotExistException { - loadMockStatements(parentStore, 50, new Date(currentDate.getTime() + 10000L)); + loadMockStatements(parentStore, 50, new Date(currentDate + 10000L)); - final MemoryTimeMerger merger = new MemoryTimeMerger(parentStore, childStore, - new VisibilityStatementMerger(), currentDate, RYA_INSTANCE, 0L); + final MemoryMerger merger = new MemoryMerger(parentStore, childStore, RYA_INSTANCE, 0L); merger.runJob(); assertEquals(50, count(childStore)); } @@ -162,19 +171,17 @@ public void no_statementsTest() throws AddStatementException, FetchStatementExce loadMockStatements(parentStore, 50, new Date(0L)); assertEquals(0, count(childStore)); - final MemoryTimeMerger merger = new MemoryTimeMerger(parentStore, childStore, - new VisibilityStatementMerger(), currentDate, RYA_INSTANCE, 0L); + final MemoryMerger merger = new MemoryMerger(parentStore, childStore, RYA_INSTANCE, 0L); merger.runJob(); assertEquals(0, count(childStore)); } @Test public void childToParent_ChildAddTest() throws AddStatementException, FetchStatementException { - loadMockStatements(parentStore, 50, new Date(currentDate.getTime() + 100L)); + loadMockStatements(parentStore, 50, new Date(currentDate + 100L)); //setup child - final MemoryTimeMerger merger = new MemoryTimeMerger(parentStore, childStore, - new VisibilityStatementMerger(), currentDate, RYA_INSTANCE, 0L); + final MemoryMerger merger = new MemoryMerger(parentStore, childStore, RYA_INSTANCE, 0L); merger.runJob(); //add a few statements to child @@ -183,19 +190,17 @@ public void childToParent_ChildAddTest() throws AddStatementException, FetchStat childStore.addStatement(stmnt1); childStore.addStatement(stmnt2); - final MemoryTimeMerger otherMerger = new MemoryTimeMerger(childStore, parentStore, - new VisibilityStatementMerger(), currentDate, RYA_INSTANCE, 0L); + final MemoryMerger otherMerger = new MemoryMerger(childStore, parentStore, RYA_INSTANCE, 0L); otherMerger.runJob(); assertEquals(52, count(parentStore)); } @Test public void childToParent_ChildReAddsDeletedStatementTest() throws Exception { - loadMockStatements(parentStore, 50, new Date(currentDate.getTime() + 10000L)); + loadMockStatements(parentStore, 50, new Date(currentDate + 10000L)); //setup child - final MemoryTimeMerger merger = new MemoryTimeMerger(parentStore, childStore, - new VisibilityStatementMerger(), currentDate, RYA_INSTANCE, 0L); + final MemoryMerger merger = new MemoryMerger(parentStore, childStore, RYA_INSTANCE, 0L); merger.runJob(); //remove a statement from the parent @@ -205,8 +210,7 @@ public void childToParent_ChildReAddsDeletedStatementTest() throws Exception { assertFalse(parentStore.containsStatement(stmnt1)); - final MemoryTimeMerger otherMerger = new MemoryTimeMerger(childStore, parentStore, - new VisibilityStatementMerger(), currentDate, RYA_INSTANCE, 0L); + final MemoryMerger otherMerger = new MemoryMerger(childStore, parentStore, RYA_INSTANCE, 0L); otherMerger.runJob(); //merging will have added the statement back @@ -215,11 +219,10 @@ public void childToParent_ChildReAddsDeletedStatementTest() throws Exception { @Test public void childToParent_BothAddTest() throws Exception { - loadMockStatements(parentStore, 50, new Date(currentDate.getTime() + 10000L)); + loadMockStatements(parentStore, 50, new Date(currentDate + 10000L)); assertEquals(0, count(childStore)); - final MemoryTimeMerger merger = new MemoryTimeMerger(parentStore, childStore, - new VisibilityStatementMerger(), currentDate, RYA_INSTANCE, 0L); + final MemoryMerger merger = new MemoryMerger(parentStore, childStore, RYA_INSTANCE, 0L); merger.runJob(); @@ -230,17 +233,137 @@ public void childToParent_BothAddTest() throws Exception { final RyaStatement stmnt1 = makeRyaStatement("http://subject", "http://predicate", "http://add"); final RyaStatement stmnt2 = makeRyaStatement("http://subject", "http://predicate", "http://add2"); stmnt1.setTimestamp(new Date().getTime() + 10L); - stmnt2.setTimestamp(currentDate.getTime() + 1000L); + stmnt2.setTimestamp(currentDate + 1000L); parentStore.addStatement(stmnt1); childStore.addStatement(stmnt2); - final MemoryTimeMerger otherMerger = new MemoryTimeMerger(childStore, parentStore, - new VisibilityStatementMerger(), currentDate, RYA_INSTANCE, 0L); + final MemoryMerger otherMerger = new MemoryMerger(childStore, parentStore, RYA_INSTANCE, 0L); otherMerger.runJob(); //both should still be there assertEquals(52, count(parentStore)); } + @Test + public void parentToChildSynch_childDeleted() throws Exception { + loadMockStatements(parentStore, 50, new Date(currentDate + 10000L)); + + // fill child store + MemoryMerger merger = new MemoryMerger(parentStore, childStore, RYA_INSTANCE, 0L); + merger.runJob(); + + // delete statement from child + final RyaStatement statement = makeRyaStatement("http://subject", "http://predicate", "http://25"); + childStore.removeStatement(statement); + + // re-synch child from parent + merger = new MemoryMerger(parentStore, childStore, RYA_INSTANCE, 0L); + merger.runJob(); + + // deleted statement should be back in child + final List stmnts = new ArrayList<>(); + boolean found = false; + final Iterator stmntsIter = childStore.fetchStatements(); + + while(stmntsIter.hasNext()) { + final RyaStatement stmnt = stmntsIter.next(); + if(stmnt.getObject().equals(new RyaType(ANY, "http://25"))) { + //since the timestamp changed, can't assert over contains + found = true; + } + stmnts.add(stmnt); + }; + + assertEquals(50, stmnts.size()); + assertTrue(found); + } + + @Test + public void parentToChildSynch_parentDeleted() throws Exception { + loadMockStatements(parentStore, 50, new Date(currentDate + 10000L)); + + // fill child store + MemoryMerger merger = new MemoryMerger(parentStore, childStore, RYA_INSTANCE, 0L); + merger.runJob(); + + // delete statement from parent + final RyaStatement statement = makeRyaStatement("http://subject", "http://predicate", "http://25"); + parentStore.removeStatement(statement); + + // re-synch child from parent + merger = new MemoryMerger(parentStore, childStore, RYA_INSTANCE, 0L); + merger.runJob(); + + // deleted statement should be gone from child + final List stmnts = new ArrayList<>(); + childStore.fetchStatements().forEachRemaining(stmnt -> { + stmnts.add(stmnt); + }); + + assertFalse(stmnts.contains(statement)); + assertEquals(49, stmnts.size()); + } + + @Test + public void parentToChildSynch_childAdded() throws Exception { + loadMockStatements(parentStore, 50, new Date(currentDate + 10000L)); + + // fill child store + MemoryMerger merger = new MemoryMerger(parentStore, childStore, RYA_INSTANCE, 0L); + merger.runJob(); + + // add statement to child + final RyaStatement statement = makeRyaStatement("http://subject", "http://predicate", "http://100"); + childStore.addStatement(statement); + + // re-synch child from parent + merger = new MemoryMerger(parentStore, childStore, RYA_INSTANCE, 0L); + merger.runJob(); + + // added statement should be gone from child + final List stmnts = new ArrayList<>(); + childStore.fetchStatements().forEachRemaining(stmnt -> { + stmnts.add(stmnt); + }); + + assertFalse(stmnts.contains(statement)); + assertEquals(50, stmnts.size()); + } + + @Test + public void parentToChildSynch_parentAdded() throws Exception { + loadMockStatements(parentStore, 50, new Date(currentDate + 10000L)); + + // fill child store + MemoryMerger merger = new MemoryMerger(parentStore, childStore, RYA_INSTANCE, 0L); + merger.runJob(); + + // add statement to parent + final RyaStatement statement = makeRyaStatement("http://subject", "http://predicate", "http://100"); + statement.setTimestamp(currentDate + 20000L); + parentStore.addStatement(statement); + + // re-synch child from parent + merger = new MemoryMerger(parentStore, childStore, RYA_INSTANCE, 0L); + merger.runJob(); + + // added statement should be in child + final List stmnts = new ArrayList<>(); + boolean found = false; + final Iterator stmntsIter = childStore.fetchStatements(); + + while(stmntsIter.hasNext()) { + final RyaStatement stmnt = stmntsIter.next(); + if(stmnt.getObject().equals(new RyaType(ANY, "http://100"))) { + //since the timestamp changed, can't assert over contains + found = true; + } + stmnts.add(stmnt); + }; + + assertEquals(51, stmnts.size()); + assertTrue(found); + } + private void loadMockStatements(final RyaStatementStore store, final int count, final Date timestamp) throws AddStatementException { for(int ii = 0; ii < count; ii++) { final RyaStatement statement = makeRyaStatement("http://subject", "http://predicate", "http://"+ii); diff --git a/extras/rya.export/export.mongo/src/main/java/org/apache/rya/export/mongo/MongoRyaStatementStore.java b/extras/rya.export/export.mongo/src/main/java/org/apache/rya/export/mongo/MongoRyaStatementStore.java index d77c0d6fb..cd96f1143 100644 --- a/extras/rya.export/export.mongo/src/main/java/org/apache/rya/export/mongo/MongoRyaStatementStore.java +++ b/extras/rya.export/export.mongo/src/main/java/org/apache/rya/export/mongo/MongoRyaStatementStore.java @@ -21,9 +21,7 @@ import static com.google.common.base.Preconditions.checkNotNull; import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.TIMESTAMP; -import java.util.ArrayList; import java.util.Iterator; -import java.util.List; import java.util.Optional; import org.apache.rya.api.domain.RyaStatement; @@ -54,7 +52,7 @@ public class MongoRyaStatementStore implements RyaStatementStore { private static final Logger logger = LoggerFactory.getLogger(MongoRyaStatementStore.class); - public static final String TRIPLES_COLLECTION = "rya__triples"; + public static final String TRIPLES_COLLECTION = "rya_triples"; public static final String METADATA_COLLECTION = "parent_metadata"; protected final SimpleMongoDBStorageStrategy adapter; protected final DB db; @@ -77,17 +75,28 @@ public MongoRyaStatementStore(final MongoClient client, final String ryaInstance db = this.client.getDB(ryaInstanceName); adapter = new SimpleMongoDBStorageStrategy(); parentMetadataRepo = new MongoParentMetadataRepository(client, ryaInstance); + db.getCollection(TRIPLES_COLLECTION).createIndex(new BasicDBObject(TIMESTAMP, 1)); } @Override public Iterator fetchStatements() { final Cursor cur = db.getCollection(TRIPLES_COLLECTION).find().sort(new BasicDBObject(TIMESTAMP, 1)); - final List statements = new ArrayList<>(); - while(cur.hasNext()) { - final RyaStatement statement = adapter.deserializeDBObject(cur.next()); - statements.add(statement); - } - return statements.iterator(); + return new Iterator() { + @Override + public boolean hasNext() { + return cur.hasNext(); + } + + @Override + public RyaStatement next() { + return adapter.deserializeDBObject(cur.next()); + } + }; + } + + @Override + public long count() { + return (int) db.getCollection(TRIPLES_COLLECTION).count(); } @Override @@ -99,6 +108,16 @@ public void addStatement(final RyaStatement statement) throws AddStatementExcept } } + @Override + public void addStatements(final Iterator statements) throws AddStatementException { + try { + dao.add(statements); + dao.flush(); + } catch (final RyaDAOException e) { + throw new AddStatementException("Unable to add statements.", e); + } + } + @Override public void removeStatement(final RyaStatement statement) throws RemoveStatementException { try { @@ -146,4 +165,9 @@ public Optional getParentMetadata() { public void setParentMetadata(final MergeParentMetadata metadata) throws ParentMetadataExistsException { parentMetadataRepo.set(metadata); } + + @Override + public String getRyaInstanceName() { + return ryaInstanceName; + } } \ No newline at end of file diff --git a/extras/rya.export/export.mongo/src/main/java/org/apache/rya/export/mongo/parent/ParentMetadataRepositoryAdapter.java b/extras/rya.export/export.mongo/src/main/java/org/apache/rya/export/mongo/parent/ParentMetadataRepositoryAdapter.java index fea962ddc..0e4f05a9f 100644 --- a/extras/rya.export/export.mongo/src/main/java/org/apache/rya/export/mongo/parent/ParentMetadataRepositoryAdapter.java +++ b/extras/rya.export/export.mongo/src/main/java/org/apache/rya/export/mongo/parent/ParentMetadataRepositoryAdapter.java @@ -57,7 +57,7 @@ public DBObject serialize(final MergeParentMetadata metadata) { public MergeParentMetadata deserialize(final DBObject dbo) { final Date timestamp = (Date) dbo.get(TIMESTAMP_KEY); final String ryaInstance = (String) dbo.get(RYANAME_KEY); - final Date filterTimestamp = (Date) dbo.get(FILTER_TIMESTAMP_KEY); + final long filterTimestamp = (long) dbo.get(FILTER_TIMESTAMP_KEY); final Long offset = (Long) dbo.get(PARENT_TIME_OFFSET_KEY); return new MergeParentMetadata.Builder() .setRyaInstanceName(ryaInstance) diff --git a/extras/rya.export/export.mongo/src/main/java/org/apache/rya/export/mongo/policy/TimestampPolicyMongoRyaStatementStore.java b/extras/rya.export/export.mongo/src/main/java/org/apache/rya/export/mongo/policy/TimestampPolicyMongoRyaStatementStore.java index 2bb923e37..e71311426 100644 --- a/extras/rya.export/export.mongo/src/main/java/org/apache/rya/export/mongo/policy/TimestampPolicyMongoRyaStatementStore.java +++ b/extras/rya.export/export.mongo/src/main/java/org/apache/rya/export/mongo/policy/TimestampPolicyMongoRyaStatementStore.java @@ -22,13 +22,10 @@ import static org.apache.rya.export.mongo.MongoRyaStatementStore.TRIPLES_COLLECTION; import static org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy.TIMESTAMP; -import java.util.ArrayList; -import java.util.Date; import java.util.Iterator; -import java.util.List; import org.apache.rya.api.domain.RyaStatement; -import org.apache.rya.export.api.conf.policy.TimestampPolicyStatementStore; +import org.apache.rya.export.api.policy.TimestampPolicyStatementStore; import org.apache.rya.export.api.store.FetchStatementException; import org.apache.rya.export.api.store.RyaStatementStore; import org.apache.rya.export.mongo.MongoRyaStatementStore; @@ -37,7 +34,7 @@ import com.mongodb.BasicDBObject; import com.mongodb.BasicDBObjectBuilder; import com.mongodb.Cursor; -import com.mongodb.DB; +import com.mongodb.DBCollection; import com.mongodb.DBObject; /** @@ -46,7 +43,7 @@ */ public class TimestampPolicyMongoRyaStatementStore extends TimestampPolicyStatementStore { private final SimpleMongoDBStorageStrategy adapter; - private final DB db; + private final DBCollection coll; /** * Creates a new {@link TimestampPolicyMongoRyaStatementStore} @@ -54,25 +51,44 @@ public class TimestampPolicyMongoRyaStatementStore extends TimestampPolicyStatem * @param timestamp - The Date to filter statements on. * @param ryaInstanceName - The rya instance to merge statements to/from. */ - public TimestampPolicyMongoRyaStatementStore(final MongoRyaStatementStore store, final Date timestamp, final String ryaInstanceName) { + public TimestampPolicyMongoRyaStatementStore(final MongoRyaStatementStore store, final long timestamp) { super(store, timestamp); adapter = new SimpleMongoDBStorageStrategy(); - db = store.getClient().getDB(ryaInstanceName); + coll = store.getClient().getDB(store.getRyaInstanceName()).getCollection(TRIPLES_COLLECTION); } @Override public Iterator fetchStatements() throws FetchStatementException { - final DBObject timeObj = new BasicDBObjectBuilder() + final DBObject timeObj = getQuery(); + final Cursor cur = coll.find(timeObj).sort(new BasicDBObject(TIMESTAMP, 1)); + return new Iterator() { + @Override + public boolean hasNext() { + return cur.hasNext(); + } + + @Override + public RyaStatement next() { + return adapter.deserializeDBObject(cur.next()); + } + }; + } + + @Override + public long count() { + return coll.count(getQuery()); + } + + private DBObject getQuery() { + return new BasicDBObjectBuilder() .add(SimpleMongoDBStorageStrategy.TIMESTAMP, new BasicDBObjectBuilder() - .add("$gte", timestamp.getTime()).get()) + .add("$gte", timestamp).get()) .get(); - final Cursor cur = db.getCollection(TRIPLES_COLLECTION).find(timeObj).sort(new BasicDBObject(TIMESTAMP, 1)); - final List statements = new ArrayList<>(); - while(cur.hasNext()) { - final RyaStatement statement = adapter.deserializeDBObject(cur.next()); - statements.add(statement); - } - return statements.iterator(); + } + + @Override + public String getRyaInstanceName() { + return store.getRyaInstanceName(); } } diff --git a/extras/rya.export/export.mongo/src/test/java/org/apache/rya/export/mongo/parent/ParentMetadataRepositoryAdapterTest.java b/extras/rya.export/export.mongo/src/test/java/org/apache/rya/export/mongo/parent/ParentMetadataRepositoryAdapterTest.java index d0e412d55..4f01857e4 100644 --- a/extras/rya.export/export.mongo/src/test/java/org/apache/rya/export/mongo/parent/ParentMetadataRepositoryAdapterTest.java +++ b/extras/rya.export/export.mongo/src/test/java/org/apache/rya/export/mongo/parent/ParentMetadataRepositoryAdapterTest.java @@ -35,7 +35,7 @@ public class ParentMetadataRepositoryAdapterTest { private final static String TEST_INSTANCE = "test_instance"; private final static Date TEST_TIMESTAMP = new Date(8675309L); - private final static Date TEST_FILTER_TIMESTAMP = new Date(1234567L); + private final static long TEST_FILTER_TIMESTAMP = 1234567L; private final static long TEST_TIME_OFFSET = 123L; private final ParentMetadataRepositoryAdapter adapter = new ParentMetadataRepositoryAdapter(); diff --git a/extras/shell/src/test/java/org/apache/rya/shell/MongoRyaShellIT.java b/extras/shell/src/test/java/org/apache/rya/shell/MongoRyaShellIT.java index ab0fe01ad..a3c3559bd 100644 --- a/extras/shell/src/test/java/org/apache/rya/shell/MongoRyaShellIT.java +++ b/extras/shell/src/test/java/org/apache/rya/shell/MongoRyaShellIT.java @@ -190,29 +190,4 @@ public void disconnect() throws IOException { final CommandResult disconnectResult = shell.executeCommand( RyaConnectionCommands.DISCONNECT_COMMAND_NAME_CMD ); assertTrue( disconnectResult.isSuccess() ); } - - // TODO the rest of them? - - private static final ConsolePrinter systemPrinter = new ConsolePrinter() { - - @Override - public void print(final CharSequence cs) throws IOException { - System.out.print(cs); - } - - @Override - public void println(final CharSequence cs) throws IOException { - System.out.println(cs); - } - - @Override - public void println() throws IOException { - System.out.println(); - } - - @Override - public void flush() throws IOException { - System.out.flush(); - } - }; -} \ No newline at end of file +} diff --git a/extras/shell/src/test/java/org/apache/rya/shell/RyaShellMongoITBase.java b/extras/shell/src/test/java/org/apache/rya/shell/RyaShellMongoITBase.java index 6a0129e91..060a032ea 100644 --- a/extras/shell/src/test/java/org/apache/rya/shell/RyaShellMongoITBase.java +++ b/extras/shell/src/test/java/org/apache/rya/shell/RyaShellMongoITBase.java @@ -68,4 +68,10 @@ public Bootstrap getTestBootstrap() { public JLineShellComponent getTestShell() { return shell; } + + @Override + public String getMongoHostname() { + // TODO Auto-generated method stub + return null; + } } \ No newline at end of file