Skip to content

Commit

Permalink
90% Introduced resque worker to create all indexes (#114)
Browse files Browse the repository at this point in the history 
* New resque worker created

- added constants for resque job

ensureIndexes now creates all required indexes

IndexUtils->ensureIndexes has been updated
- this method now ensures ALL tripod required indexes for Tables
- this method now ensures ALL tripod required indexes for Views
- this method now ensures ALL tripod required indexes for Search Docs

Removed hardcoded index creation from delegates

on every insert/update of a composite we were ensuring a set of hardcoded
indexes, in addition to ensuring ALL indexes specified in the compose spec i.e.
in the View/Table/SearchDoc Specification.  All of this code has been removed,
in favor of a more coherent approach via background jobs facilitated by the
IndexUtils->ensureIndexes method.

removed redundant tests from MongoTripodTablesTest

- these tests contained expectations around calls to the Tables->ensureIndex
  method which now, no longer exists.

Also added tests for IndexUtils which previously had no test coverage.
This class does the actual work now around creating indexes

* adds createJob method to EnsureIndexes job + tests

* p/r feedback

* fixed typo

* renamed helper method to createMockJob

* addressed all p/r feedback

* ensureindexes cli script now schedules jobs

* added docblock

* added debug to ensureindexes job
  • Loading branch information
@kiyanwang @rgubby
kiyanwang authored and rgubby committed Feb 13, 2017
1 parent 97c88f6 commit d3f95f3
Show file tree
Hide file tree
Showing 14 changed files with 1,146 additions and 236 deletions.
3 changes: 1 addition & 2 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,10 @@
composer.phar
output
vendor

tags
atlassian-ide-plugin.xml

composer.lock

test/performance/rest-interface/.vagrant/
test/performance/rest-interface/config/tripod-config-*.json

12 changes: 6 additions & 6 deletions scripts/mongo/ensureIndexes.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,9 @@

require_once dirname(dirname(dirname(__FILE__))).'/src/tripod.inc.php';


if ($argc!=2&&$argc!=3&&$argc!=4)
{
echo "usage: php ensureIndexes.php tripodConfig.json [storeName] [forceReindex (default is false)]\n";
echo "usage: php ensureIndexes.php tripodConfig.json [storeName] [forceReindex (default is false)] [background (default is true)]\n";
die();
}
array_shift($argv);
Expand All @@ -15,14 +14,15 @@

$storeName = (isset($argv[1])) ? $argv[1] : null;
$forceReindex = (isset($argv[2])&&($argv[2]=="true")) ? true : false;
$background = (isset($argv[3])&&($argv[3]=="false")) ? false : true;

\Tripod\Mongo\Config::getInstance()->setMongoCursorTimeout(-1);

$ei = new \Tripod\Mongo\IndexUtils();
$ei = new \Tripod\Mongo\Jobs\EnsureIndexes();

$t = new \Tripod\Timer();
$t->start();
print("About to start indexing on $storeName...\n");
$ei->ensureIndexes($forceReindex,$storeName);
print("About to start scheduling indexing jobs for $storeName...\n");
$ei->createJob($storeName, $forceReindex, $background);
$t->stop();
print "Indexing complete, took {$t->result()} seconds\n";
print "Finished scheduling ensure indexes jobs, took {$t->result()} seconds\n";
13 changes: 8 additions & 5 deletions src/mongo/Config.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -388,7 +388,6 @@ protected function loadConfig(Array $config)
}
}


}

/**
Expand Down Expand Up @@ -1013,7 +1012,6 @@ protected function getPredicatesFromFilterCondition($filter)
return $predicates;
}


/**
* Returns an array of associated predicates in a table or search document specification
* Note: will not return viewSpec predicates
Expand Down Expand Up @@ -1066,7 +1064,6 @@ public function checkModifierFunctions(array $array, $parent, $parentKey = null)
$this->checkModifierFunctions($v, $parent[$k], $k);
}


} else if(is_string($k))
{
// Check key
Expand Down Expand Up @@ -1497,7 +1494,6 @@ public function getViewSpecifications($storeName)
return (isset($this->viewSpecs[$storeName]) ? $this->viewSpecs[$storeName] : array());
}


/**
* This method returns a unique list of every rdf type configured in a specifications ['type'] restriction
* @param string $storeName
Expand Down Expand Up @@ -2088,6 +2084,14 @@ public static function getApplyQueueName()
return self::getQueueName(TRIPOD_APPLY_QUEUE,"apply");
}

/**
* @return string
*/
public static function getEnsureIndexesQueueName()
{
return self::getQueueName(TRIPOD_ENSURE_INDEXES_QUEUE, "ensureindexes");
}

/**
* @param string $envVar
* @param string $type
Expand Down Expand Up @@ -2160,4 +2164,3 @@ public static function getLogger()
return self::$logger;
}
}

9 changes: 6 additions & 3 deletions src/mongo/MongoTripodConstants.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
define("SEARCH_PROVIDER_MONGO", '\Tripod\Mongo\MongoSearchProvider');
define("SEARCH_PROVIDER_ELASTIC_SEARCH", 'es');


// Tripod document properties
define('_ID_KEY', '_id');
define('_ID_RESOURCE','r');
Expand Down Expand Up @@ -63,7 +62,6 @@
define('_UPDATED_TS_INDEX','_updatedTsIdx');
define('_CREATED_TS_INDEX','_createdTsIdx');


// other events, used mainly for logging
define('MONGO_IMPACT_INDEX_HIT','MONGO_IMPACT_INDEX_HIT');
define('MONGO_CREATE_VIEW','MONGO_CREATE_VIEW');
Expand All @@ -86,6 +84,11 @@
define('MONGO_QUEUE_APPLY_OPERATION_SUCCESS','MONGO_QUEUE_APPLY_OPERATION_SUCCESS');
define('MONGO_QUEUE_APPLY_OPERATION_FAIL','MONGO_QUEUE_APPLY_OPERATION_FAIL');

define('MONGO_QUEUE_ENSURE_INDEXES','MONGO_QUEUE_ENSURE_INDEXES');
define('MONGO_QUEUE_ENSURE_INDEXES_JOB','MONGO_QUEUE_ENSURE_INDEXES_JOB');
define('MONGO_QUEUE_ENSURE_INDEXES_SUCCESS','MONGO_QUEUE_ENSURE_INDEXES_SUCCESS');
define('MONGO_QUEUE_ENSURE_INDEXES_FAIL','MONGO_QUEUE_ENSURE_INDEXES_FAIL');

define('MONGO_CREATE_SEARCH_DOC','MONGO_CREATE_SEARCH_DOC');
define('MONGO_CONNECTION_ERROR','MONGO_CONNECTION_ERROR');

Expand All @@ -108,7 +111,7 @@
define("MONGO_TRIPOD_RESQUE_SERVER","MONGO_TRIPOD_RESQUE_SERVER"); // deprecated, will be removed in future releases
define("TRIPOD_DISCOVER_QUEUE","TRIPOD_DISCOVER_QUEUE");
define("TRIPOD_APPLY_QUEUE","TRIPOD_APPLY_QUEUE");
define("TRIPOD_ENSURE_INDEXES_QUEUE","TRIPOD_ENSURE_INDEXES_QUEUE");

// View functions
define("INCLUDE_RDF_SEQUENCE","_seq_");

86 changes: 33 additions & 53 deletions src/mongo/delegates/SearchDocuments.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ public function generateSearchDocumentBasedOnSpecId($specId, $resource, $context
$proceedWithGeneration = false;

foreach ($searchSpec['filter'] as $indexRules)
{
{
// run a query to work out
if (!empty($indexRules['condition']))
{
Expand Down Expand Up @@ -117,27 +117,7 @@ public function generateSearchDocumentBasedOnSpecId($specId, $resource, $context
$_id['type'] = $specId;
$generatedDocument['_id'] = $_id;

Config::getInstance()->getCollectionForSearchDocument(
$this->storeName,
$specId)
->createIndex(
array('_id.type'=>1),
array(
'background'=>1
)
);

Config::getInstance()->getCollectionForSearchDocument(
$this->storeName,
$specId)
->createIndex(
array('_impactIndex'=>1),
array(
'background'=>1
)
);

if(isset($searchSpec['fields'])){
if(isset($searchSpec['fields'])){
$this->addFields($sourceDocument, $searchSpec['fields'], $generatedDocument);
}
if(isset($searchSpec['indices'])){
Expand Down Expand Up @@ -309,7 +289,7 @@ protected function addFields(Array $source, Array $fieldsOrIndices, Array &$targ
*/
protected function getSearchDocumentSpecification($specId)
{
return Config::getInstance()->getSearchDocumentSpecification($this->storeName, $specId);
return Config::getInstance()->getSearchDocumentSpecification($this->storeName, $specId);
}

/**
Expand All @@ -333,35 +313,35 @@ private function addValuesToTarget($values, $field, &$target)
$limit = $field['limit'];
} else {
$limit = count($values);
}
if(count($values) > 0){
for ($i=0; $i<$limit; $i++) {
$v = $values[$i];
if (empty($objName)) {
if (!isset($target[$name])) {
$target[$name] = $v;
} else if (is_array($target[$name])) {
$target[$name][] = $v;
} else {
$existingVal = $target[$name];
$target[$name] = array();
$target[$name][] = $existingVal;
$target[$name][] = $v;
}
} else {
if (!isset($target[$objName][$name])) {
$target[$objName][$name] = $v;
} else if (is_array($target[$objName][$name])) {
$target[$objName][$name][] = $v;
} else {
$existingVal = $target[$objName][$name];
$target[$objName][$name] = array();
$target[$objName][$name][] = $existingVal;
$target[$objName][$name][] = $v;
}
}
}
}

if(count($values) > 0){
for ($i=0; $i<$limit; $i++) {
$v = $values[$i];
if (empty($objName)) {
if (!isset($target[$name])) {
$target[$name] = $v;
} else if (is_array($target[$name])) {
$target[$name][] = $v;
} else {
$existingVal = $target[$name];
$target[$name] = array();
$target[$name][] = $existingVal;
$target[$name][] = $v;
}
} else {
if (!isset($target[$objName][$name])) {
$target[$objName][$name] = $v;
} else if (is_array($target[$objName][$name])) {
$target[$objName][$name][] = $v;
} else {
$existingVal = $target[$objName][$name];
$target[$objName][$name] = array();
$target[$objName][$name][] = $existingVal;
$target[$objName][$name][] = $v;
}
}
}
}
}

Expand All @@ -372,4 +352,4 @@ public function getSearchCollectionName()
{
return SEARCH_INDEX_COLLECTION;
}
}
}
60 changes: 0 additions & 60 deletions src/mongo/delegates/Tables.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -502,51 +502,6 @@ public function generateTableRows($tableType,$resource=null,$context=null,$queue
return null;
}

// ensure that the ID field, view type, and the impactIndex indexes are correctly set up
$collection->createIndex(
array(
'_id.r'=>1,
'_id.c'=>1,
'_id.type'=>1
),
array(
'background'=>1
)
);

$collection->createIndex(
array(
'_id.type'=>1
),
array(
'background'=>1
)
);

$collection->createIndex(
array(
'value.'._IMPACT_INDEX=>1
),
array(
'background'=>1
)
);

// ensure any custom view indexes
foreach (Config::getInstance()->getTableSpecifications($this->storeName) as $tSpec)
{
if (isset($tSpec['ensureIndexes']) && $tSpec['to_data_source'] == $tableSpec['to_data_source']) // only ensure table_rows indexes for the data source that matches the table spec we're generating
{
foreach ($tSpec['ensureIndexes'] as $ensureIndex)
{
$this->ensureIndex(
$collection,
$ensureIndex
);
}
}
}

// default the context
$contextAlias = $this->getContextAlias($context);

Expand Down Expand Up @@ -1458,21 +1413,6 @@ protected function checkIfTypeShouldTriggerOperation($rdfType, array $validTypes
return false;
}

/**
* Ensure $indexToEnsure on the given mongo $collection
* @param Collection $collection
* @param array $indexToEnsure
*/
protected function ensureIndex(Collection $collection,array $indexToEnsure)
{
$collection->createIndex(
$indexToEnsure,
array(
'background'=>1
)
);

}
/**
* Apply a regex to the RDF property value defined in $value
* @param $regex
Expand Down
45 changes: 0 additions & 45 deletions src/mongo/delegates/Views.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -410,50 +410,6 @@ public function generateView($viewId,$resource=null,$context=null,$queueName=nul
throw new \Tripod\Exceptions\ViewException('Could not find any joins in view specification - usecase better served with select()');
}

// ensure that the ID field, view type, and the impactIndex indexes are correctly set up
$collection->createIndex(
array(
'_id.r'=>1,
'_id.c'=>1,
'_id.type'=>1
),
array(
'background'=>1
)
);

$collection->createIndex(
array(
'_id.type'=>1
),
array(
'background'=>1
)
);

$collection->createIndex(
array(
'value.'._IMPACT_INDEX=>1
),
array(
'background'=>1
)
);

// ensure any custom view indexes
if (isset($viewSpec['ensureIndexes']))
{
foreach ($viewSpec['ensureIndexes'] as $ensureIndex)
{
$collection->createIndex(
$ensureIndex,
array(
'background'=>1
)
);
}
}

$types = array(); // this is used to filter the CBD table to speed up the view creation
if (is_array($viewSpec["type"]))
{
Expand Down Expand Up @@ -681,7 +637,6 @@ protected function matchesFilter($linkMatchType, $linkMatchValue, $filterType, $
return false;
}


/**
* Returns a document with properties extracted from $source, according to $viewSpec. Useful for partial representations
* of CBDs in a view
Expand Down
Loading

0 comments on commit d3f95f3

Please sign in to comment.