diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..adbc1517 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +# This file is for unifying the coding style for different editors and IDEs +# editorconfig.org + +# PHP PSR-2 Coding Standards +# http://www.php-fig.org/psr/psr-2/ + +root = true + +[*.php] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 4 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3ee7cbe0..3f246e37 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ .idea composer.phar +composer.lock output vendor tags diff --git a/.travis.yml b/.travis.yml index 6dc6d668..3a44cd74 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,10 @@ php: - 5.4 services: - redis +- docker before_install: -- sudo apt-get install -y mongodb-org=2.6.9 mongodb-org-server=2.6.9 mongodb-org-shell=2.6.9 mongodb-org-mongos=2.6.9 mongodb-org-tools=2.6.9 +- docker pull rossfsinger/mongo-2.6.12 +- docker run -d -p 127.0.0.1:27017:27017 -v ~/data:/data/db rossfsinger/mongo-2.6.12:latest - echo "extension = mongodb.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini - sleep 15 install: diff --git a/composer.json b/composer.json index ba2e11a8..dab691ac 100644 --- a/composer.json +++ b/composer.json @@ -18,10 +18,11 @@ } ], "require": { + "php" : ">=5.4", "semsol/arc2": "v2.2.4", "chrisboulton/php-resque": "dev-master#98fde571db008a8b48e73022599d1d1c07d4a7b5", "monolog/monolog" : "~1.13", - "mongodb/mongodb": "^1.0.0" + "mongodb/mongodb": "1.0.4" }, "require-dev": { "phpunit/phpunit": "4.1.*" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..3cf8dd7f --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,4 @@ +mongo: + image: mongo/2.6.12:latest + ports: + - "27017:27017" \ No newline at end of file diff --git a/src/mongo/Config.class.php b/src/mongo/Config.class.php index fb0975ce..0de41cbb 100644 --- a/src/mongo/Config.class.php +++ b/src/mongo/Config.class.php @@ -2054,6 +2054,19 @@ public function getCollectionForManualRollbackAudit($storeName, $readPreference ); } + /** + * @param string $storeName + * @param string $readPreference + * @return Collection + */ + public function getCollectionForJobGroups($storeName, $readPreference = ReadPreference::RP_PRIMARY_PREFERRED) + { + return $this->getMongoCollection( + $this->getDatabase($storeName, $this->dbConfig[$storeName]['data_source'], $readPreference), + OPERATION_GROUPS_COLLECTION + ); + } + /** * @param $readPreference * @return Database diff --git a/src/mongo/JobGroup.php b/src/mongo/JobGroup.php new file mode 100644 index 00000000..6b5d223d --- /dev/null +++ b/src/mongo/JobGroup.php @@ -0,0 +1,86 @@ +storeName = $storeName; + if (!$groupId) { + $groupId = new ObjectId(); + } elseif (!$groupId instanceof ObjectId) { + $groupId = new ObjectId($groupId); + } + $this->id = $groupId; + } + + /** + * Update the number of jobs + * + * @param integer $count Number of jobs in group + * @return void + */ + public function setJobCount($count) + { + $this->getMongoCollection()->updateOne( + ['_id' => $this->getId()], + ['$set' => ['count' => $count]], + ['upsert' => true] + ); + } + + /** + * Update the number of jobs by $inc. To decrement, use a negative integer + * + * @param integer $inc Number to increment or decrement by + * @return integer Updated job count + */ + public function incrementJobCount($inc = 1) + { + $updateResult = $this->getMongoCollection()->findOneAndUpdate( + ['_id' => $this->getId()], + ['$inc' => ['count' => $inc]], + ['upsert' => true, 'returnDocument' => \MongoDB\Operation\FindOneAndUpdate::RETURN_DOCUMENT_AFTER] + ); + if (\is_array($updateResult)) { + return $updateResult['count']; + } elseif (isset($updateResult->count)) { + return $updateResult->count; + } + } + + /** + * @return ObjectId + */ + public function getId() + { + return $this->id; + } + + /** + * For mocking + * + * @return \MongoDB\Collection + */ + protected function getMongoCollection() + { + if (!isset($this->collection)) { + $config = Config::getInstance(); + + $this->collection = $config->getCollectionForJobGroups($this->storeName); + } + return $this->collection; + } +} diff --git a/src/mongo/MongoTripodConstants.php b/src/mongo/MongoTripodConstants.php index 4725e5aa..81909fd0 100644 --- a/src/mongo/MongoTripodConstants.php +++ b/src/mongo/MongoTripodConstants.php @@ -6,6 +6,7 @@ define('VIEWS_COLLECTION', 'views'); define('LOCKS_COLLECTION', 'locks'); define('AUDIT_MANUAL_ROLLBACKS_COLLECTION','audit_manual_rollbacks'); +define('OPERATION_GROUPS_COLLECTION', 'job_groups'); // search define('SEARCH_INDEX_COLLECTION', 'search'); diff --git a/src/mongo/delegates/SearchDocuments.class.php b/src/mongo/delegates/SearchDocuments.class.php index 013790b4..302a4e91 100644 --- a/src/mongo/delegates/SearchDocuments.class.php +++ b/src/mongo/delegates/SearchDocuments.class.php @@ -111,7 +111,7 @@ public function generateSearchDocumentBasedOnSpecId($specId, $resource, $context $this->debugLog("Processing {$specId}"); // build the document - $generatedDocument = array(); + $generatedDocument = [\_CREATED_TS => DateUtil::getMongoDate()]; $this->addIdToImpactIndex($_id, $generatedDocument); $_id['type'] = $specId; diff --git a/src/mongo/delegates/SearchIndexer.class.php b/src/mongo/delegates/SearchIndexer.class.php index 8d9eef92..eeb3231c 100644 --- a/src/mongo/delegates/SearchIndexer.class.php +++ b/src/mongo/delegates/SearchIndexer.class.php @@ -10,6 +10,8 @@ use Tripod\Mongo\Config; use Tripod\Mongo\ImpactedSubject; use Tripod\Mongo\Labeller; +use Tripod\Mongo\Jobs\ApplyOperation; +use Tripod\Mongo\JobGroup; use \MongoDB\Driver\ReadPreference; use \MongoDB\Collection; @@ -166,6 +168,7 @@ public function generateAndIndexSearchDocuments($resourceUri, $context, $podName * @param string|null $resourceUri * @param string|null $context * @param string|null $queueName + * @return array|null Will return an array with a count and group id, if $queueName is sent and $resourceUri is null */ public function generateSearchDocuments($searchDocumentType, $resourceUri=null, $context=null, $queueName=null) { @@ -174,9 +177,9 @@ public function generateSearchDocuments($searchDocumentType, $resourceUri=null, // default the context $contextAlias = $this->getContextAlias($context); $spec = \Tripod\Mongo\Config::getInstance()->getSearchDocumentSpecification($this->getStoreName(), $searchDocumentType); - + if($resourceUri) - { + { $this->generateAndIndexSearchDocuments($resourceUri, $contextAlias, $spec['from'], $searchDocumentType); return; } @@ -205,13 +208,20 @@ public function generateSearchDocuments($searchDocumentType, $resourceUri=null, $filter["_id"] = array(_ID_RESOURCE=>$this->labeller->uri_to_alias($resource),_ID_CONTEXT=>$contextAlias); } + $count = $this->config->getCollectionForCBD($this->getStoreName(), $from)->count($filter); $docs = $this->config->getCollectionForCBD($this->getStoreName(), $from)->find($filter, array( 'maxTimeMS' => $this->config->getMongoCursorTimeout() )); - foreach ($docs as $doc) - { - if($queueName && !$resourceUri) - { + + $jobOptions = []; + if ($queueName && !$resourceUri) { + $jobOptions['statsConfig'] = $this->getStatsConfig(); + $jobGroup = new JobGroup($this->storeName); + $jobOptions[ApplyOperation::TRACKING_KEY] = $jobGroup->getId()->__toString(); + $jobGroup->setJobCount($count); + } + foreach ($docs as $doc) { + if ($queueName && !$resourceUri) { $subject = new ImpactedSubject( $doc['_id'], OP_SEARCH, @@ -219,17 +229,9 @@ public function generateSearchDocuments($searchDocumentType, $resourceUri=null, $from, array($searchDocumentType) ); - $jobOptions = array(); - - if($this->stat || !empty($this->statsConfig)) - { - $jobOptions['statsConfig'] = $this->getStatsConfig(); - } $this->getApplyOperation()->createJob(array($subject), $queueName, $jobOptions); - } - else - { + } else { $this->generateAndIndexSearchDocuments( $doc[_ID_KEY][_ID_RESOURCE], $doc[_ID_KEY][_ID_CONTEXT], @@ -246,6 +248,12 @@ public function generateSearchDocuments($searchDocumentType, $resourceUri=null, 'filter'=>$filter, 'from'=>$from)); $this->getStat()->timer(MONGO_CREATE_SEARCH_DOC.".$searchDocumentType",$t->result()); + + $stat = ['count' => $count]; + if (isset($jobOptions[ApplyOperation::TRACKING_KEY])) { + $stat[ApplyOperation::TRACKING_KEY] = $jobOptions[ApplyOperation::TRACKING_KEY]; + } + return $stat; } /** @@ -266,7 +274,7 @@ public function deleteSearchDocumentsByTypeId($typeId) { return $this->getSearchProvider()->deleteSearchDocumentsByTypeId($typeId); } - + /** * @return \Tripod\ISearchProvider @@ -300,4 +308,4 @@ protected function deDupe(Array $input) } return $output; } -} \ No newline at end of file +} diff --git a/src/mongo/delegates/Tables.class.php b/src/mongo/delegates/Tables.class.php index 8df1da74..a20992fe 100644 --- a/src/mongo/delegates/Tables.class.php +++ b/src/mongo/delegates/Tables.class.php @@ -5,9 +5,11 @@ require_once TRIPOD_DIR . 'mongo/MongoTripodConstants.php'; require_once TRIPOD_DIR . 'mongo/base/DriverBase.class.php'; -use Tripod\Mongo\Config; -use Tripod\Mongo\ImpactedSubject; -use Tripod\Mongo\Labeller; +use \Tripod\Mongo\Jobs\ApplyOperation; +use \Tripod\Mongo\Config; +use \Tripod\Mongo\ImpactedSubject; +use \Tripod\Mongo\Labeller; +use \Tripod\Mongo\JobGroup; use \MongoDB\Driver\ReadPreference; use \MongoDB\Collection; @@ -358,24 +360,35 @@ protected function deleteTableRowsForResource($resource, $context=null, $specTyp /** * This method will delete all table rows where the _id.type matches the specified $tableId - * @param string $tableId + * @param string $tableId Table spec ID + * @param \MongoDB\BSON\UTCDateTime|null $timestamp Optional timestamp to delete all table rows that are older than + * @return integer The number of table rows deleted */ - public function deleteTableRowsByTableId($tableId) { + public function deleteTableRowsByTableId($tableId, $timestamp = null) { $t = new \Tripod\Timer(); $t->start(); $tableSpec = Config::getInstance()->getTableSpecification($this->storeName, $tableId); - if ($tableSpec==null) - { + if ($tableSpec == null) { $this->debugLog("Could not find a table specification for $tableId"); return; } - $query = array("_id.type"=>$tableId); - $this->config->getCollectionForTable($this->storeName, $tableId) + $query = ['_id.type' => $tableId]; + if ($timestamp) { + if (!($timestamp instanceof \MongoDB\BSON\UTCDateTime)) { + $timestamp = \Tripod\Mongo\DateUtil::getMongoDate($timestamp); + } + $query['$or'] = [ + [\_CREATED_TS => ['$lt' => $timestamp]], + [\_CREATED_TS => ['$exists' => false]] + ]; + } + $deleteResult = $this->getCollectionForTableSpec($tableId) ->deleteMany($query); $t->stop(); $this->timingLog(MONGO_DELETE_TABLE_ROWS, array('duration'=>$t->result(), 'query'=>$query)); + return $deleteResult->getDeletedCount(); } /** @@ -486,9 +499,9 @@ public function generateTableRowsForType($rdfType,$subject=null,$context=null, $ * @param string|null $resource * @param string|null $context * @param string|null $queueName Queue for background bulk generation - * @return null //@todo: this should be a bool + * @return array */ - public function generateTableRows($tableType,$resource=null,$context=null,$queueName=null) + public function generateTableRows($tableType, $resource = null, $context = null, $queueName = null) { $t = new \Tripod\Timer(); $t->start(); @@ -496,8 +509,7 @@ public function generateTableRows($tableType,$resource=null,$context=null,$queue $tableSpec = Config::getInstance()->getTableSpecification($this->storeName, $tableType); $collection = $this->config->getCollectionForTable($this->storeName, $tableType); - if ($tableSpec==null) - { + if ($tableSpec==null) { $this->debugLog("Could not find a table specification for $tableType"); return null; } @@ -509,33 +521,35 @@ public function generateTableRows($tableType,$resource=null,$context=null,$queue $from = (isset($tableSpec["from"])) ? $tableSpec["from"] : $this->podName; $types = array(); - if (is_array($tableSpec["type"])) - { - foreach ($tableSpec["type"] as $type) - { + if (is_array($tableSpec["type"])) { + foreach ($tableSpec["type"] as $type) { $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($type)); $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($type)); } - } - else - { + } else { $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($tableSpec["type"])); $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($tableSpec["type"])); } $filter = array('$or'=> $types); - if (isset($resource)) - { + if (isset($resource)) { $filter["_id"] = array(_ID_RESOURCE=>$this->labeller->uri_to_alias($resource),_ID_CONTEXT=>$contextAlias); } - + // @todo Change this to a command when we upgrade MongoDB to 1.1+ + $count = $this->config->getCollectionForCBD($this->storeName, $from)->count($filter); $docs = $this->config->getCollectionForCBD($this->storeName, $from)->find($filter, array( 'maxTimeMS' => 1000000 )); - foreach ($docs as $doc) - { - if($queueName && !$resource) - { + $jobOptions = []; + if ($queueName && !$resource && ($this->stat || !empty($this->statsConfig))) { + $jobOptions['statsConfig'] = $this->getStatsConfig(); + $jobGroup = new JobGroup($this->storeName); + $jobOptions[ApplyOperation::TRACKING_KEY] = $jobGroup->getId()->__toString(); + $jobGroup->setJobCount($count); + } + + foreach ($docs as $doc) { + if ($queueName && !$resource) { $subject = new ImpactedSubject( $doc['_id'], OP_TABLES, @@ -544,34 +558,29 @@ public function generateTableRows($tableType,$resource=null,$context=null,$queue array($tableType) ); - $jobOptions = array(); - - if($this->stat || !empty($this->statsConfig)) - { - $jobOptions['statsConfig'] = $this->getStatsConfig(); - } - $this->getApplyOperation()->createJob(array($subject), $queueName, $jobOptions); - } - else - { + } else { // set up ID - $generatedRow = array("_id"=>array(_ID_RESOURCE=>$doc["_id"][_ID_RESOURCE],_ID_CONTEXT=>$doc["_id"][_ID_CONTEXT],_ID_TYPE=>$tableSpec['_id'])); - - $value = array('_id'=>$doc['_id']); // everything must go in the value object todo: this is a hang over from map reduce days, engineer out once we have stability on new PHP method for M/R + $generatedRow = [ + '_id' => [ + _ID_RESOURCE => $doc['_id'][_ID_RESOURCE], + _ID_CONTEXT => $doc['_id'][_ID_CONTEXT], + _ID_TYPE=>$tableSpec['_id'] + ], + \_CREATED_TS => \Tripod\Mongo\DateUtil::getMongoDate() + ]; + // everything must go in the value object todo: this is a hang over from map reduce days, engineer out once we have stability on new PHP method for M/R + $value = ['_id' => $doc['_id']]; $this->addIdToImpactIndex($doc['_id'], $value); // need to add the doc to the impact index to be consistent with views/search etc. this is needed for discovering impacted operations $this->addFields($doc,$tableSpec,$value); - if (isset($tableSpec['joins'])) - { + if (isset($tableSpec['joins'])) { $this->doJoins($doc,$tableSpec['joins'],$value,$from,$contextAlias); } - if (isset($tableSpec['counts'])) - { + if (isset($tableSpec['counts'])) { $this->doCounts($doc,$tableSpec['counts'],$value); } - if (isset($tableSpec['computed_fields'])) - { + if (isset($tableSpec['computed_fields'])) { $this->doComputedFields($tableSpec, $value); } @@ -589,6 +598,12 @@ public function generateTableRows($tableType,$resource=null,$context=null,$queue 'filter'=>$filter, 'from'=>$from)); $this->getStat()->timer(MONGO_CREATE_TABLE.".$tableType",$t->result()); + + $stat = ['count' => $count]; + if (isset($jobOptions[ApplyOperation::TRACKING_KEY])) { + $stat[ApplyOperation::TRACKING_KEY] = $jobOptions[ApplyOperation::TRACKING_KEY]; + } + return $stat; } /** @@ -1437,4 +1452,28 @@ private function applyRegexToValue($regex, $value) throw new \Tripod\Exceptions\Exception("Was expecting either VALUE_URI or VALUE_LITERAL when applying regex to value - possible data corruption with: ".var_export($value,true)); } } + + /** + * Count the number of documents in the spec that match $filters + * + * @param string $tableSpec Table spec ID + * @param array $filters Query filters to get count on + * @return integer + */ + public function count($tableSpec, array $filters = []) + { + $filters['_id.type'] = $tableSpec; + return $this->getCollectionForTableSpec($tableSpec)->count($filters); + } + + /** + * For mocking + * + * @param string $tableSpecId Table spec ID + * @return \MongoDB\Collection + */ + protected function getCollectionForTableSpec($tableSpecId) + { + return $this->getConfigInstance()->getCollectionForTable($this->storeName, $tableSpecId); + } } diff --git a/src/mongo/delegates/Views.class.php b/src/mongo/delegates/Views.class.php index 0067d327..ce810417 100644 --- a/src/mongo/delegates/Views.class.php +++ b/src/mongo/delegates/Views.class.php @@ -4,11 +4,13 @@ require_once TRIPOD_DIR . 'mongo/base/DriverBase.class.php'; -use Tripod\Mongo\Config; -use Tripod\Mongo\ImpactedSubject; -use Tripod\Mongo\Labeller; +use \Tripod\Mongo\Jobs\ApplyOperation; +use \Tripod\Mongo\Config; +use \Tripod\Mongo\ImpactedSubject; +use \Tripod\Mongo\Labeller; use \MongoDB\Driver\ReadPreference; use \MongoDB\Collection; +use Tripod\Mongo\JobGroup; /** * Class Views @@ -365,18 +367,30 @@ public function generateViewsForResourcesOfType($rdfType,$resource=null,$context /** * This method will delete all views where the _id.type of the viewmatches the specified $viewId - * @param $viewId + * @param string $viewId View spec ID + * @param \MongoDB\BSON\UTCDateTime|null $timestamp Optional timestamp to delete all views that are older than + * @return integer The number of views deleted */ - public function deleteViewsByViewId($viewId){ + public function deleteViewsByViewId($viewId, $timestamp = null) + { $viewSpec = Config::getInstance()->getViewSpecification($this->storeName, $viewId); - if ($viewSpec==null) - { + if ($viewSpec == null) { $this->debugLog("Could not find a view specification with viewId '$viewId'"); return; } - - $this->config->getCollectionForView($this->storeName, $viewId) - ->deleteMany(array("_id.type"=>$viewId)); + $query = ['_id.type' => $viewId]; + if ($timestamp) { + if (!($timestamp instanceof \MongoDB\BSON\UTCDateTime)) { + $timestamp = \Tripod\Mongo\DateUtil::getMongoDate($timestamp); + } + $query['$or'] = [ + [\_CREATED_TS => ['$lt' => $timestamp]], + [\_CREATED_TS => ['$exists' => false]] + ]; + } + $deleteResult = $this->getCollectionForViewSpec($viewId) + ->deleteMany($query); + return $deleteResult->getDeletedCount(); } /** @@ -392,108 +406,108 @@ public function generateView($viewId,$resource=null,$context=null,$queueName=nul { $contextAlias = $this->getContextAlias($context); $viewSpec = Config::getInstance()->getViewSpecification($this->storeName, $viewId); - if ($viewSpec==null) - { + if ($viewSpec == null) { $this->debugLog("Could not find a view specification for $resource with viewId '$viewId'"); return null; } - else - { - $t = new \Tripod\Timer(); - $t->start(); + $t = new \Tripod\Timer(); + $t->start(); - $from = $this->getFromCollectionForViewSpec($viewSpec); - $collection = $this->config->getCollectionForView($this->storeName, $viewId); + $from = $this->getFromCollectionForViewSpec($viewSpec); + $collection = $this->config->getCollectionForView($this->storeName, $viewId); - if (!isset($viewSpec['joins'])) - { - throw new \Tripod\Exceptions\ViewException('Could not find any joins in view specification - usecase better served with select()'); - } + if (!isset($viewSpec['joins'])) { + throw new \Tripod\Exceptions\ViewException('Could not find any joins in view specification - usecase better served with select()'); + } - $types = array(); // this is used to filter the CBD table to speed up the view creation - if (is_array($viewSpec["type"])) - { - foreach ($viewSpec["type"] as $type) - { - $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($type)); - $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($type)); - } - } - else - { - $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($viewSpec["type"])); - $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($viewSpec["type"])); - } - $filter = array('$or'=> $types); - if (isset($resource)) - { - $resourceAlias = $this->labeller->uri_to_alias($resource); - $filter["_id"] = array(_ID_RESOURCE=>$resourceAlias,_ID_CONTEXT=>$contextAlias); + $types = array(); // this is used to filter the CBD table to speed up the view creation + if (is_array($viewSpec["type"])) { + foreach ($viewSpec["type"] as $type) { + $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($type)); + $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($type)); } + } else { + $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($viewSpec["type"])); + $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($viewSpec["type"])); + } + $filter = array('$or'=> $types); + if (isset($resource)) { + $resourceAlias = $this->labeller->uri_to_alias($resource); + $filter["_id"] = array(_ID_RESOURCE=>$resourceAlias,_ID_CONTEXT=>$contextAlias); + } - $docs = $this->config->getCollectionForCBD($this->storeName, $from)->find($filter, array( - 'maxTimeMS' => \Tripod\Mongo\Config::getInstance()->getMongoCursorTimeout() - )); - - foreach ($docs as $doc) - { - if($queueName && !$resource) - { - $subject = new ImpactedSubject( - $doc['_id'], - OP_VIEWS, - $this->storeName, - $from, - array($viewId) - ); + // @todo Change this to a command when we upgrade MongoDB to 1.1+ + $count = $this->config->getCollectionForCBD($this->storeName, $from)->count($filter); + $docs = $this->config->getCollectionForCBD($this->storeName, $from)->find($filter, array( + 'maxTimeMS' => \Tripod\Mongo\Config::getInstance()->getMongoCursorTimeout() + )); - $jobOptions = array(); - if($this->stat || !empty($this->statsConfig)) - { - $jobOptions['statsConfig'] = $this->getStatsConfig(); - } - $this->getApplyOperation()->createJob(array($subject), $queueName, $jobOptions); - } - else - { - // set up ID - $id = array("_id"=>array(_ID_RESOURCE=>$doc["_id"][_ID_RESOURCE],_ID_CONTEXT=>$doc["_id"][_ID_CONTEXT],_ID_TYPE=>$viewSpec['_id'])); - $generatedView = $id; - $value = array(); // everything must go in the value object todo: this is a hang over from map reduce days, engineer out once we have stability on new PHP method for M/R - $value[_GRAPHS] = array(); + $jobOptions = []; + if ($queueName && !$resource) { + $jobOptions['statsConfig'] = $this->getStatsConfig(); + $jobGroup = new JobGroup($this->storeName); + $jobOptions[ApplyOperation::TRACKING_KEY] = $jobGroup->getId()->__toString(); + $jobGroup->setJobCount($count); + } + foreach ($docs as $doc) { + if ($queueName && !$resource) { + $subject = new ImpactedSubject( + $doc['_id'], + OP_VIEWS, + $this->storeName, + $from, + array($viewId) + ); - $buildImpactIndex=true; - if (isset($viewSpec['ttl'])) - { - $buildImpactIndex=false; - $value[_EXPIRES] = \Tripod\Mongo\DateUtil::getMongoDate($this->getExpirySecFromNow($viewSpec['ttl']) * 1000); - } - else - { - $value[_IMPACT_INDEX] = array($doc['_id']); - } + $this->getApplyOperation()->createJob(array($subject), $queueName, $jobOptions); + } else { + // Set up view meta information + $generatedView = [ + '_id' => [ + _ID_RESOURCE => $doc['_id'][_ID_RESOURCE], + _ID_CONTEXT => $doc['_id'][_ID_CONTEXT], + _ID_TYPE=>$viewSpec['_id'] + ], + \_CREATED_TS => \Tripod\Mongo\DateUtil::getMongoDate() + ]; + $value = array(); // everything must go in the value object todo: this is a hang over from map reduce days, engineer out once we have stability on new PHP method for M/R + + $value[_GRAPHS] = array(); + + $buildImpactIndex=true; + if (isset($viewSpec['ttl'])) { + $buildImpactIndex=false; + $value[_EXPIRES] = \Tripod\Mongo\DateUtil::getMongoDate($this->getExpirySecFromNow($viewSpec['ttl']) * 1000); + } else { + $value[_IMPACT_INDEX] = array($doc['_id']); + } - $this->doJoins($doc,$viewSpec['joins'],$value,$from,$contextAlias,$buildImpactIndex); + $this->doJoins($doc,$viewSpec['joins'],$value,$from,$contextAlias,$buildImpactIndex); - // add top level properties - $value[_GRAPHS][] = $this->extractProperties($doc,$viewSpec,$from); + // add top level properties + $value[_GRAPHS][] = $this->extractProperties($doc,$viewSpec,$from); - $generatedView['value'] = $value; + $generatedView['value'] = $value; - $collection->replaceOne($id, $generatedView, ['upsert' => true]); - } + $collection->replaceOne(['_id' => $generatedView['_id']], $generatedView, ['upsert' => true]); } + } - $t->stop(); - $this->timingLog(MONGO_CREATE_VIEW, array( - 'view'=>$viewSpec['type'], - 'duration'=>$t->result(), - 'filter'=>$filter, - 'from'=>$from)); - $this->getStat()->timer(MONGO_CREATE_VIEW.".$viewId",$t->result()); + $t->stop(); + $this->timingLog(MONGO_CREATE_VIEW, array( + 'view'=>$viewSpec['type'], + 'duration'=>$t->result(), + 'filter'=>$filter, + 'from'=>$from)); + $this->getStat()->timer(MONGO_CREATE_VIEW.".$viewId",$t->result()); + + $stat = ['count' => $count]; + if (isset($jobOptions[ApplyOperation::TRACKING_KEY])) { + $stat[ApplyOperation::TRACKING_KEY] = $jobOptions[ApplyOperation::TRACKING_KEY]; } + return $stat; } /** @@ -812,4 +826,16 @@ protected function getCollectionForViewSpec($viewSpecId) return $this->getConfigInstance()->getCollectionForView($this->storeName, $viewSpecId); } + /** + * Count the number of documents in the spec that match $filters + * + * @param string $viewSpec View spec ID + * @param array $filters Query filters to get count on + * @return integer + */ + public function count($viewSpec, array $filters = []) + { + $filters['_id.type'] = $viewSpec; + return $this->getCollectionForViewSpec($viewSpec)->count($filters); + } } diff --git a/src/mongo/jobs/ApplyOperation.class.php b/src/mongo/jobs/ApplyOperation.class.php index eee50390..318cfec1 100644 --- a/src/mongo/jobs/ApplyOperation.class.php +++ b/src/mongo/jobs/ApplyOperation.class.php @@ -2,6 +2,10 @@ namespace Tripod\Mongo\Jobs; +use Tripod\Mongo\JobGroup; +use Tripod\Mongo\Driver; + + /** * Class ApplyOperation * @package Tripod\Mongo\Jobs @@ -9,14 +13,15 @@ class ApplyOperation extends JobBase { const SUBJECTS_KEY = 'subjects'; + const TRACKING_KEY = 'batchId'; + /** * Run the ApplyOperation job * @throws \Exception */ public function perform() { - try - { + try { $this->debugLog("[JOBID " . $this->job->payload['id'] . "] ApplyOperation::perform() start"); $timer = new \Tripod\Timer(); @@ -35,8 +40,7 @@ public function perform() $this->getStat()->increment(MONGO_QUEUE_APPLY_OPERATION_JOB . '.' . SUBJECT_COUNT, count($this->args[self::SUBJECTS_KEY])); - foreach($this->args[self::SUBJECTS_KEY] as $subject) - { + foreach ($this->args[self::SUBJECTS_KEY] as $subject) { $opTimer = new \Tripod\Timer(); $opTimer->start(); @@ -46,17 +50,49 @@ public function perform() $opTimer->stop(); // stat time taken to perform operation for the given subject $this->getStat()->timer(MONGO_QUEUE_APPLY_OPERATION.'.'.$subject['operation'], $opTimer->result()); + + /** + * ApplyOperation jobs can either apply to a single resource (e.g. 'create composite for the given + * resource uri) or for a specification id (i.e. regenerate all of the composites defined by the + * specification). For the latter, we need to keep track of how many jobs have run so we can clean + * up any stale composite documents when completed. The TRACKING_KEY value will be the JobGroup id. + */ + if (isset($this->args[self::TRACKING_KEY])) { + $jobGroup = $this->getJobGroup($subject['storeName'], $this->args[self::TRACKING_KEY]); + $jobCount = $jobGroup->incrementJobCount(-1); + if ($jobCount <= 0) { + // @todo Replace this with ObjectId->getTimestamp() if we upgrade Mongo driver to 1.2 + $timestamp = new \MongoDB\BSON\UTCDateTime(hexdec(substr($jobGroup->getId(), 0, 8)) * 1000); + $tripod = $this->getTripod($subject['storeName'], $subject['podName']); + $count = 0; + foreach ($subject['specTypes'] as $specId) { + switch ($subject['operation']) { + case \OP_VIEWS: + $count += $tripod->getComposite(\OP_VIEWS)->deleteViewsByViewId($specId, $timestamp); + break; + case \OP_TABLES: + $count += $tripod->getComposite(\OP_TABLES)->deleteTableRowsByTableId($specId, $timestamp); + break; + case \OP_SEARCH: + $searchProvider = $this->getSearchProvider($tripod); + $count += $searchProvider->deleteSearchDocumentsByTypeId($specId, $timestamp); + break; + } + } + $this->infoLog( + '[JobGroupId ' . $jobGroup->getId()->__toString() . '] composite cleanup for ' . + $subject['operation'] . ' removed ' . $count . ' stale composite documents' + ); + } + } } $timer->stop(); // stat time taken to process job, from time it was picked up $this->getStat()->timer(MONGO_QUEUE_APPLY_OPERATION_SUCCESS,$timer->result()); - $this->debugLog("[JOBID " . $this->job->payload['id'] . "] ApplyOperation::perform() done in {$timer->result()}ms"); - } - catch (\Exception $e) - { + } catch (\Exception $e) { $this->getStat()->increment(MONGO_QUEUE_APPLY_OPERATION_FAIL); $this->errorLog("Caught exception in ".get_class($this).": ".$e->getMessage()); throw $e; @@ -100,7 +136,7 @@ protected function createImpactedSubject(array $args) $args["storeName"], $args["podName"], $args["specTypes"] - ); + ); } /** @@ -111,4 +147,27 @@ protected function getMandatoryArgs() { return array(self::TRIPOD_CONFIG_KEY,self::SUBJECTS_KEY); } + + /** + * For mocking + * + * @param string $storeName Tripod store (database) name + * @param string|\MongoDB\BSON\ObjectId $trackingKey JobGroup ID + * @return JobGroup + */ + protected function getJobGroup($storeName, $trackingKey) + { + return new JobGroup($storeName, $trackingKey); + } + + /** + * For mocking + * + * @param Driver $tripod + * @return \Tripod\Mongo\MongoSearchProvider + */ + protected function getSearchProvider(Driver $tripod) + { + return new \Tripod\Mongo\MongoSearchProvider($tripod); + } } diff --git a/src/mongo/providers/MongoSearchProvider.class.php b/src/mongo/providers/MongoSearchProvider.class.php index d5895755..d4ffef47 100644 --- a/src/mongo/providers/MongoSearchProvider.class.php +++ b/src/mongo/providers/MongoSearchProvider.class.php @@ -336,20 +336,30 @@ public function getSearchCollectionName() * Removes all documents from search index based on the specified type id. * Here search type id represents to id from, mongo tripod config, that is converted to _id.type in SEARCH_INDEX_COLLECTION * If type id is not specified this method will throw an exception. - * @param string $typeId search type id - * @return bool|array response returned by mongo + * @param string $typeId Search type id + * @param \MongoDB\BSON\UTCDateTime|null $timestamp Optional timestamp to delete all search docs that are older than + * @return integer The number of search documents deleted * @throws \Tripod\Exceptions\Exception if there was an error performing the operation */ - public function deleteSearchDocumentsByTypeId($typeId) + public function deleteSearchDocumentsByTypeId($typeId, $timestamp = null) { $searchSpec = $this->getSearchDocumentSpecification($typeId); - if ($searchSpec == null) - { + if ($searchSpec == null) { throw new \Tripod\Exceptions\SearchException("Could not find a search specification for $typeId"); } - - return $this->config->getCollectionForSearchDocument($this->storeName, $typeId) - ->deleteMany(array("_id.type" => $typeId)); + $query = ['_id.type' => $typeId]; + if ($timestamp) { + if (!($timestamp instanceof \MongoDB\BSON\UTCDateTime)) { + $timestamp = new \MongoDB\BSON\UTCDateTime($timestamp); + } + $query['$or'] = [ + [\_CREATED_TS => ['$lt' => $timestamp]], + [\_CREATED_TS => ['$exists' => false]] + ]; + } + $deleteResponse = $this->getCollectionForSearchSpec($typeId) + ->deleteMany($query); + return $deleteResponse->getDeletedCount(); } /** @@ -361,4 +371,28 @@ protected function getSearchDocumentSpecification($typeId) { return Config::getInstance()->getSearchDocumentSpecification($this->storeName, $typeId); } + + /** + * Count the number of documents in the spec that match $filters + * + * @param string $searchSpec Search spec ID + * @param array $filters Query filters to get count on + * @return integer + */ + public function count($searchSpec, array $filters = []) + { + $filters['_id.type'] = $searchSpec; + return $this->getCollectionForSearchSpec($searchSpec)->count($filters); + } + + /** + * For mocking + * + * @param string $searchSpecId Search spec ID + * @return \MongoDB\Collection + */ + protected function getCollectionForSearchSpec($searchSpecId) + { + return $this->config->getCollectionForSearchDocument($this->storeName, $searchSpecId); + } } diff --git a/src/mongo/util/IndexUtils.class.php b/src/mongo/util/IndexUtils.class.php index 523cfec2..8e9999f2 100644 --- a/src/mongo/util/IndexUtils.class.php +++ b/src/mongo/util/IndexUtils.class.php @@ -68,11 +68,12 @@ public function ensureIndexes($reindex=false,$storeName=null,$background=true) $collection = $config->getCollectionForView($storeName, $viewId); if($collection) { - $indexes = array( - array(_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1, _ID_KEY.'.'._ID_TYPE => 1), - array(_ID_KEY.'.'._ID_TYPE => 1), - array('value.'._IMPACT_INDEX => 1) - ); + $indexes = [ + [_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1, _ID_KEY.'.'._ID_TYPE => 1], + [_ID_KEY.'.'._ID_TYPE => 1], + ['value.'._IMPACT_INDEX => 1], + [\_CREATED_TS => 1] + ]; if(isset($spec['ensureIndexes'])) { $indexes = array_merge($indexes, $spec['ensureIndexes']); @@ -99,11 +100,12 @@ public function ensureIndexes($reindex=false,$storeName=null,$background=true) $collection = $config->getCollectionForTable($storeName, $tableId); if($collection) { - $indexes = array( - array(_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1, _ID_KEY.'.'._ID_TYPE => 1), - array(_ID_KEY.'.'._ID_TYPE => 1), - array('value.'._IMPACT_INDEX => 1) - ); + $indexes = [ + [_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1, _ID_KEY.'.'._ID_TYPE => 1], + [_ID_KEY.'.'._ID_TYPE => 1], + ['value.'._IMPACT_INDEX => 1], + [\_CREATED_TS => 1] + ]; if(isset($spec['ensureIndexes'])) { $indexes = array_merge($indexes, $spec['ensureIndexes']); @@ -130,11 +132,12 @@ public function ensureIndexes($reindex=false,$storeName=null,$background=true) $collection = $config->getCollectionForSearchDocument($storeName, $searchId); if($collection) { - $indexes = array( - array(_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1), - array(_ID_KEY.'.'._ID_TYPE => 1), - array(_IMPACT_INDEX => 1) - ); + $indexes = [ + [_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1], + [_ID_KEY.'.'._ID_TYPE => 1], + [_IMPACT_INDEX => 1], + [\_CREATED_TS => 1] + ]; if($reindex) { diff --git a/src/tripod.inc.php b/src/tripod.inc.php index adb7654b..6326c486 100644 --- a/src/tripod.inc.php +++ b/src/tripod.inc.php @@ -31,6 +31,7 @@ require_once TRIPOD_DIR.'classes/ChangeSet.class.php'; require_once TRIPOD_DIR.'classes/Labeller.class.php'; require_once TRIPOD_DIR . '/mongo/Driver.class.php'; +require_once TRIPOD_DIR . '/mongo/JobGroup.php'; require_once TRIPOD_DIR.'/mongo/base/JobBase.class.php'; require_once TRIPOD_DIR . '/mongo/jobs/DiscoverImpactedSubjects.class.php'; diff --git a/test/unit/mongo/ApplyOperationTest.php b/test/unit/mongo/ApplyOperationTest.php index 007746b3..a5741a8b 100644 --- a/test/unit/mongo/ApplyOperationTest.php +++ b/test/unit/mongo/ApplyOperationTest.php @@ -1,5 +1,7 @@ perform(); } + public function testApplyViewOperationDecrementsJobGroupForBatchOperations() + { + $this->setArgs(); + $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation') + ->setMethods(['createImpactedSubject', 'getStat', 'getJobGroup', 'getTripod']) + ->getMock(); + + $applyOperation->args = $this->args; + $applyOperation->job->payload['id'] = uniqid(); + $jobTrackerId = new \MongoDB\BSON\ObjectId(); + $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString(); + + $jobGroup = $this->getMockBuilder('\Tripod\Mongo\JobGroup') + ->setMethods(['incrementJobCount']) + ->setConstructorArgs(['tripod_php_testing', $jobTrackerId]) + ->getMock(); + + $jobGroup->expects($this->once()) + ->method('incrementJobCount') + ->with(-1) + ->will($this->returnValue(2)); + + $statMock = $this->getMockStat( + $this->args['statsConfig']['config']['host'], + $this->args['statsConfig']['config']['port'], + $this->args['statsConfig']['config']['prefix'], + ['timer','increment'] + ); + + $subject = $this->getMockBuilder('\Tripod\Mongo\ImpactedSubject') + ->setMethods(['getTripod']) + ->setConstructorArgs( + [ + [ + _ID_RESOURCE=>'http://example.com/resources/foo', + _ID_CONTEXT=>'http://talisaspire.com' + ], + OP_VIEWS, + 'tripod_php_testing', + 'CBD_testing' + ] + )->getMock(); + + $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver') + ->setMethods(['getComposite']) + ->setConstructorArgs(['CBD_testing', 'tripod_php_testing']) + ->getMock(); + + $views = $this->getMockBuilder('\Tripod\Mongo\Composites\Views') + ->setMethods(['update', 'deleteViewsByViewId']) + ->setConstructorArgs( + [ + 'tripod_php_testing', + \Tripod\Mongo\Config::getInstance()->getCollectionForCBD('tripod_php_testing', 'CBD_testing'), + 'http://talisapire.com/' + ] + )->getMock(); + + $applyOperation->expects($this->once()) + ->method('createImpactedSubject') + ->will($this->returnValue($subject)); + + $applyOperation->expects($this->exactly(3)) + ->method('getStat') + ->will($this->returnValue($statMock)); + + $applyOperation->expects($this->once()) + ->method('getJobGroup') + ->with('tripod_php_testing', $jobTrackerId->__toString()) + ->will($this->returnValue($jobGroup)); + + $applyOperation->expects($this->never()) + ->method('getTripod'); + + $statMock->expects($this->once()) + ->method('increment') + ->with(MONGO_QUEUE_APPLY_OPERATION_JOB . '.' . SUBJECT_COUNT, 1); + $statMock->expects($this->exactly(2)) + ->method('timer') + ->withConsecutive( + [MONGO_QUEUE_APPLY_OPERATION.'.'.OP_VIEWS, $this->anything()], + [MONGO_QUEUE_APPLY_OPERATION_SUCCESS, $this->anything()] + ); + + $subject->expects($this->once()) + ->method('getTripod') + ->will($this->returnValue($tripod)); + + $tripod->expects($this->once()) + ->method('getComposite') + ->with(OP_VIEWS) + ->will($this->returnValue($views)); + + $views->expects($this->once()) + ->method('update') + ->with($subject); + + $views->expects($this->never())->method('deleteViewsByViewId'); + $applyOperation->perform(); + } + + public function testApplyViewOperationCleanupIfAllGroupJobsComplete() + { + $this->setArgs(OP_VIEWS, ['v_foo_bar']); + $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation') + ->setMethods(['createImpactedSubject', 'getStat', 'getJobGroup', 'getTripod']) + ->getMock(); + + $applyOperation->args = $this->args; + $applyOperation->job->payload['id'] = uniqid(); + $jobTrackerId = new \MongoDB\BSON\ObjectId(); + $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString(); + $timestamp = new \MongoDB\BSON\UTCDateTime(hexdec(substr($jobTrackerId, 0, 8)) * 1000); + + $jobGroup = $this->getMockBuilder('\Tripod\Mongo\JobGroup') + ->setMethods(['incrementJobCount']) + ->setConstructorArgs(['tripod_php_testing', $jobTrackerId]) + ->getMock(); + + $jobGroup->expects($this->once()) + ->method('incrementJobCount') + ->with(-1) + ->will($this->returnValue(0)); + + $statMock = $this->getMockStat( + $this->args['statsConfig']['config']['host'], + $this->args['statsConfig']['config']['port'], + $this->args['statsConfig']['config']['prefix'], + array('timer','increment') + ); + + $subject = $this->getMockBuilder('\Tripod\Mongo\ImpactedSubject') + ->setMethods(['getTripod']) + ->setConstructorArgs( + [ + [ + _ID_RESOURCE=>'http://example.com/resources/foo', + _ID_CONTEXT=>'http://talisaspire.com' + ], + OP_VIEWS, + 'tripod_php_testing', + 'CBD_testing' + ] + )->getMock(); + + $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver') + ->setMethods(['getComposite']) + ->setConstructorArgs(['CBD_testing', 'tripod_php_testing']) + ->getMock(); + + $views = $this->getMockBuilder('\Tripod\Mongo\Composites\Views') + ->setMethods(['update', 'deleteViewsByViewId']) + ->setConstructorArgs( + [ + 'tripod_php_testing', + \Tripod\Mongo\Config::getInstance()->getCollectionForCBD('tripod_php_testing', 'CBD_testing'), + 'http://talisapire.com/' + ] + )->getMock(); + + $applyOperation->expects($this->once()) + ->method('createImpactedSubject') + ->will($this->returnValue($subject)); + + $applyOperation->expects($this->exactly(3)) + ->method('getStat') + ->will($this->returnValue($statMock)); + + $applyOperation->expects($this->once()) + ->method('getJobGroup') + ->with('tripod_php_testing', $jobTrackerId->__toString()) + ->will($this->returnValue($jobGroup)); + + $applyOperation->expects($this->once()) + ->method('getTripod') + ->with('tripod_php_testing', 'CBD_testing') + ->will($this->returnValue($tripod)); + + $statMock->expects($this->once()) + ->method('increment') + ->with(MONGO_QUEUE_APPLY_OPERATION_JOB . '.' . SUBJECT_COUNT, 1); + $statMock->expects($this->exactly(2)) + ->method('timer') + ->withConsecutive( + [MONGO_QUEUE_APPLY_OPERATION.'.'.OP_VIEWS, $this->anything()], + [MONGO_QUEUE_APPLY_OPERATION_SUCCESS, $this->anything()] + ); + + $subject->expects($this->once()) + ->method('getTripod') + ->will($this->returnValue($tripod)); + + $tripod->expects($this->exactly(2)) + ->method('getComposite') + ->with(OP_VIEWS) + ->will($this->returnValue($views)); + + $views->expects($this->once()) + ->method('update') + ->with($subject); + + $views->expects($this->once()) + ->method('deleteViewsByViewId') + ->with('v_foo_bar', $timestamp) + ->will($this->returnValue(3)); + + $applyOperation->perform(); + } + public function testApplyTableOperation() { $this->setArgs(); @@ -200,9 +411,222 @@ public function testApplyTableOperation() $applyOperation->perform(); } + public function testApplyTableOperationDecrementsJobGroupForBatchOperations() + { + $this->setArgs(OP_TABLES, ['t_resource']); + $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation') + ->setMethods(['createImpactedSubject', 'getStat', 'getJobGroup', 'getTripod']) + ->getMock(); + + $statMock = $this->getMockStat( + $this->args['statsConfig']['config']['host'], + $this->args['statsConfig']['config']['port'], + $this->args['statsConfig']['config']['prefix'], + ['timer', 'increment'] + ); + + $applyOperation->args = $this->args; + $applyOperation->job->payload['id'] = uniqid(); + + $jobTrackerId = new \MongoDB\BSON\ObjectId(); + $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString(); + + $jobGroup = $this->getMockBuilder('\Tripod\Mongo\JobGroup') + ->setMethods(['incrementJobCount']) + ->setConstructorArgs(['tripod_php_testing', $jobTrackerId]) + ->getMock(); + + $jobGroup->expects($this->once()) + ->method('incrementJobCount') + ->with(-1) + ->will($this->returnValue(2)); + + $subject = $this->getMockBuilder('\Tripod\Mongo\ImpactedSubject') + ->setMethods(['getTripod']) + ->setConstructorArgs( + [ + [ + _ID_RESOURCE=>'http://example.com/resources/foo', + _ID_CONTEXT=>'http://talisaspire.com' + ], + OP_TABLES, + 'tripod_php_testing', + 'CBD_testing' + ] + )->getMock(); + + $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver') + ->setMethods(['getComposite']) + ->setConstructorArgs(['CBD_testing', 'tripod_php_testing']) + ->getMock(); + + $tables = $this->getMockBuilder('\Tripod\Mongo\Composites\Tables') + ->setMethods(['update', 'deleteTableRowsByTableId']) + ->setConstructorArgs( + [ + 'tripod_php_testing', + \Tripod\Mongo\Config::getInstance()->getCollectionForCBD('tripod_php_testing', 'CBD_testing'), + 'http://talisapire.com/' + ] + )->getMock(); + + $applyOperation->expects($this->once()) + ->method('createImpactedSubject') + ->will($this->returnValue($subject)); + + $applyOperation->expects($this->exactly(3)) + ->method('getStat') + ->will($this->returnValue($statMock)); + + $applyOperation->expects($this->once()) + ->method('getJobGroup') + ->with('tripod_php_testing', $jobTrackerId->__toString()) + ->will($this->returnValue($jobGroup)); + + $applyOperation->expects($this->never()) + ->method('getTripod'); + + $statMock->expects($this->once()) + ->method('increment') + ->with(MONGO_QUEUE_APPLY_OPERATION_JOB . '.' . SUBJECT_COUNT, 1); + $statMock->expects($this->exactly(2)) + ->method('timer') + ->withConsecutive( + array(MONGO_QUEUE_APPLY_OPERATION.'.'.OP_TABLES, $this->anything()), + array(MONGO_QUEUE_APPLY_OPERATION_SUCCESS, $this->anything()) + ); + + + $subject->expects($this->once()) + ->method('getTripod') + ->will($this->returnValue($tripod)); + + $tripod->expects($this->once()) + ->method('getComposite') + ->with(OP_TABLES) + ->will($this->returnValue($tables)); + + $tables->expects($this->once()) + ->method('update') + ->with($subject); + $tables->expects($this->never()) + ->method('deleteTableRowsByTableId'); + + $applyOperation->perform(); + } + + public function testApplyTableOperationCleanupIfAllGroupJobsComplete() + { + $this->setArgs(OP_TABLES, ['t_resource']); + $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation') + ->setMethods(['createImpactedSubject', 'getStat', 'getJobGroup', 'getTripod']) + ->getMock(); + + $statMock = $this->getMockStat( + $this->args['statsConfig']['config']['host'], + $this->args['statsConfig']['config']['port'], + $this->args['statsConfig']['config']['prefix'], + ['timer','increment'] + ); + + $applyOperation->args = $this->args; + $applyOperation->job->payload['id'] = uniqid(); + + $jobTrackerId = new \MongoDB\BSON\ObjectId(); + $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString(); + $timestamp = new \MongoDB\BSON\UTCDateTime(hexdec(substr($jobTrackerId, 0, 8)) * 1000); + + $jobGroup = $this->getMockBuilder('\Tripod\Mongo\JobGroup') + ->setMethods(['incrementJobCount']) + ->setConstructorArgs(['tripod_php_testing', $jobTrackerId]) + ->getMock(); + + $jobGroup->expects($this->once()) + ->method('incrementJobCount') + ->with(-1) + ->will($this->returnValue(0)); + + $subject = $this->getMockBuilder('\Tripod\Mongo\ImpactedSubject') + ->setMethods(['getTripod']) + ->setConstructorArgs( + [ + [ + _ID_RESOURCE=>'http://example.com/resources/foo', + _ID_CONTEXT=>'http://talisaspire.com' + ], + OP_TABLES, + 'tripod_php_testing', + 'CBD_testing' + ] + )->getMock(); + + $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver') + ->setMethods(['getComposite']) + ->setConstructorArgs(['CBD_testing', 'tripod_php_testing']) + ->getMock(); + + $tables = $this->getMockBuilder('\Tripod\Mongo\Composites\Tables') + ->setMethods(['update', 'deleteTableRowsByTableId']) + ->setConstructorArgs( + [ + 'tripod_php_testing', + \Tripod\Mongo\Config::getInstance()->getCollectionForCBD('tripod_php_testing', 'CBD_testing'), + 'http://talisapire.com/' + ] + )->getMock(); + + $applyOperation->expects($this->once()) + ->method('createImpactedSubject') + ->will($this->returnValue($subject)); + + $applyOperation->expects($this->exactly(3)) + ->method('getStat') + ->will($this->returnValue($statMock)); + + $applyOperation->expects($this->once()) + ->method('getJobGroup') + ->with('tripod_php_testing', $jobTrackerId->__toString()) + ->will($this->returnValue($jobGroup)); + + $applyOperation->expects($this->once()) + ->method('getTripod') + ->with('tripod_php_testing', 'CBD_testing') + ->will($this->returnValue($tripod)); + + $statMock->expects($this->once()) + ->method('increment') + ->with(MONGO_QUEUE_APPLY_OPERATION_JOB . '.' . SUBJECT_COUNT, 1); + $statMock->expects($this->exactly(2)) + ->method('timer') + ->withConsecutive( + array(MONGO_QUEUE_APPLY_OPERATION.'.'.OP_TABLES, $this->anything()), + array(MONGO_QUEUE_APPLY_OPERATION_SUCCESS, $this->anything()) + ); + + + $subject->expects($this->once()) + ->method('getTripod') + ->will($this->returnValue($tripod)); + + $tripod->expects($this->exactly(2)) + ->method('getComposite') + ->with(OP_TABLES) + ->will($this->returnValue($tables)); + + $tables->expects($this->once()) + ->method('update') + ->with($subject); + $tables->expects($this->once()) + ->method('deleteTableRowsByTableId') + ->with('t_resource', $timestamp) + ->will($this->returnValue(4)); + + $applyOperation->perform(); + } + public function testApplySearchOperation() { - $this->setArgs(); + $this->setArgs(OP_SEARCH, ['i_search_resource']); $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation') ->setMethods(array('createImpactedSubject', 'getStat')) ->getMock(); @@ -214,18 +638,6 @@ public function testApplySearchOperation() array('timer','increment') ); - $impactedSubject = new \Tripod\Mongo\ImpactedSubject( - array( - _ID_RESOURCE=>'http://example.com/resources/foo', - _ID_CONTEXT=>'http://talisaspire.com/' - ), - OP_SEARCH, - 'tripod_php_testing', - 'CBD_testing', - array('t_resource') - ); - $this->args['subjects'] = array($impactedSubject->toArray()); - $applyOperation->args = $this->args; $applyOperation->job->payload['id'] = uniqid(); @@ -288,6 +700,219 @@ public function testApplySearchOperation() $applyOperation->perform(); } + public function testApplySearchOperationDecrementsJobGroupForBatchOperations() + { + $this->setArgs(OP_SEARCH, ['i_search_resource']); + $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation') + ->setMethods(['createImpactedSubject', 'getStat', 'getJobGroup', 'getTripod']) + ->getMock(); + + $statMock = $this->getMockStat( + $this->args['statsConfig']['config']['host'], + $this->args['statsConfig']['config']['port'], + $this->args['statsConfig']['config']['prefix'], + ['timer', 'increment'] + ); + + $applyOperation->args = $this->args; + $applyOperation->job->payload['id'] = uniqid(); + + $jobTrackerId = new \MongoDB\BSON\ObjectId(); + $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString(); + + $jobGroup = $this->getMockBuilder('\Tripod\Mongo\JobGroup') + ->setMethods(['incrementJobCount']) + ->setConstructorArgs(['tripod_php_testing', $jobTrackerId]) + ->getMock(); + + $jobGroup->expects($this->once()) + ->method('incrementJobCount') + ->with(-1) + ->will($this->returnValue(2)); + + $subject = $this->getMockBuilder('\Tripod\Mongo\ImpactedSubject') + ->setMethods(['getTripod']) + ->setConstructorArgs( + [ + [ + _ID_RESOURCE=>'http://example.com/resources/foo', + _ID_CONTEXT=>'http://talisaspire.com' + ], + OP_SEARCH, + 'tripod_php_testing', + 'CBD_testing' + ] + )->getMock(); + + $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver') + ->setMethods(['getComposite']) + ->setConstructorArgs(['CBD_testing', 'tripod_php_testing']) + ->getMock(); + + $search = $this->getMockBuilder('\Tripod\Mongo\Composites\SearchIndexer') + ->setMethods(['update']) + ->setConstructorArgs([$tripod]) + ->getMock(); + + $applyOperation->expects($this->once()) + ->method('createImpactedSubject') + ->will($this->returnValue($subject)); + + $applyOperation->expects($this->exactly(3)) + ->method('getStat') + ->will($this->returnValue($statMock)); + + $applyOperation->expects($this->once()) + ->method('getJobGroup') + ->with('tripod_php_testing', $jobTrackerId->__toString()) + ->will($this->returnValue($jobGroup)); + + $applyOperation->expects($this->never()) + ->method('getTripod'); + + $statMock->expects($this->once()) + ->method('increment') + ->with(MONGO_QUEUE_APPLY_OPERATION_JOB . '.' . SUBJECT_COUNT, 1); + $statMock->expects($this->exactly(2)) + ->method('timer') + ->withConsecutive( + array(MONGO_QUEUE_APPLY_OPERATION.'.'.OP_SEARCH, $this->anything()), + array(MONGO_QUEUE_APPLY_OPERATION_SUCCESS, $this->anything()) + ); + + + $subject->expects($this->once()) + ->method('getTripod') + ->will($this->returnValue($tripod)); + + $tripod->expects($this->once()) + ->method('getComposite') + ->with(OP_SEARCH) + ->will($this->returnValue($search)); + + $search->expects($this->once()) + ->method('update') + ->with($subject); + + $applyOperation->perform(); + } + + public function testApplySearchOperationCleanupIfAllGroupJobsComplete() + { + $this->setArgs(OP_SEARCH, ['i_search_resource']); + $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation') + ->setMethods(['createImpactedSubject', 'getStat', 'getJobGroup', 'getTripod', 'getSearchProvider']) + ->getMock(); + + $statMock = $this->getMockStat( + $this->args['statsConfig']['config']['host'], + $this->args['statsConfig']['config']['port'], + $this->args['statsConfig']['config']['prefix'], + ['timer', 'increment'] + ); + + $applyOperation->args = $this->args; + $applyOperation->job->payload['id'] = uniqid(); + + $jobTrackerId = new \MongoDB\BSON\ObjectId(); + $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString(); + $timestamp = new \MongoDB\BSON\UTCDateTime(hexdec(substr($jobTrackerId, 0, 8)) * 1000); + + $jobGroup = $this->getMockBuilder('\Tripod\Mongo\JobGroup') + ->setMethods(['incrementJobCount']) + ->setConstructorArgs(['tripod_php_testing', $jobTrackerId]) + ->getMock(); + + $jobGroup->expects($this->once()) + ->method('incrementJobCount') + ->with(-1) + ->will($this->returnValue(0)); + + $subject = $this->getMockBuilder('\Tripod\Mongo\ImpactedSubject') + ->setMethods(['getTripod']) + ->setConstructorArgs( + [ + [ + _ID_RESOURCE=>'http://example.com/resources/foo', + _ID_CONTEXT=>'http://talisaspire.com' + ], + OP_SEARCH, + 'tripod_php_testing', + 'CBD_testing' + ] + )->getMock(); + + $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver') + ->setMethods(['getComposite']) + ->setConstructorArgs(['CBD_testing', 'tripod_php_testing']) + ->getMock(); + + $searchProvider = $this->getMockBuilder('\Tripod\Mongo\MongoSearchProvider') + ->setMethods(['deleteSearchDocumentsByTypeId']) + ->setConstructorArgs([$tripod]) + ->getMock(); + + $search = $this->getMockBuilder('\Tripod\Mongo\Composites\SearchIndexer') + ->setMethods(['update']) + ->setConstructorArgs([$tripod]) + ->getMock(); + + $applyOperation->expects($this->once()) + ->method('createImpactedSubject') + ->will($this->returnValue($subject)); + + $applyOperation->expects($this->exactly(3)) + ->method('getStat') + ->will($this->returnValue($statMock)); + + $applyOperation->expects($this->once()) + ->method('getJobGroup') + ->with('tripod_php_testing', $jobTrackerId->__toString()) + ->will($this->returnValue($jobGroup)); + + $applyOperation->expects($this->once()) + ->method('getTripod') + ->with('tripod_php_testing', 'CBD_testing') + ->will($this->returnValue($tripod)); + + $applyOperation->expects($this->once()) + ->method('getSearchProvider') + ->with($tripod) + ->will($this->returnValue($searchProvider)); + + $statMock->expects($this->once()) + ->method('increment') + ->with(MONGO_QUEUE_APPLY_OPERATION_JOB . '.' . SUBJECT_COUNT, 1); + + $statMock->expects($this->exactly(2)) + ->method('timer') + ->withConsecutive( + array(MONGO_QUEUE_APPLY_OPERATION.'.'.OP_SEARCH, $this->anything()), + array(MONGO_QUEUE_APPLY_OPERATION_SUCCESS, $this->anything()) + ); + + + $subject->expects($this->once()) + ->method('getTripod') + ->will($this->returnValue($tripod)); + + $tripod->expects($this->once()) + ->method('getComposite') + ->with(OP_SEARCH) + ->will($this->returnValue($search)); + + $search->expects($this->once()) + ->method('update') + ->with($subject); + + $searchProvider->expects($this->once()) + ->method('deleteSearchDocumentsByTypeId') + ->with('i_search_resource', $timestamp) + ->will($this->returnValue(8)); + + $applyOperation->perform(); + } + public function testCreateJobDefaultQueue() { $impactedSubject = new \Tripod\Mongo\ImpactedSubject( @@ -429,22 +1054,23 @@ public function testCreateJobSpecifyQueue() /** * Sets job arguments */ - protected function setArgs() + protected function setArgs($operation = OP_VIEWS, array $specTypes = []) { $subject = new \Tripod\Mongo\ImpactedSubject( - array( + [ _ID_RESOURCE=>'http://example.com/resources/foo', _ID_CONTEXT=>'http://talisaspire.com/' - ), - OP_VIEWS, + ], + $operation, 'tripod_php_testing', - 'CBD_testing' + 'CBD_testing', + $specTypes ); $this->args = array( - 'tripodConfig'=>\Tripod\Mongo\Config::getConfig(), - 'subjects'=>array($subject->toArray()), - 'statsConfig'=>$this->getStatsDConfig() + 'tripodConfig' => \Tripod\Mongo\Config::getConfig(), + 'subjects'=> [$subject->toArray()], + 'statsConfig' => $this->getStatsDConfig() ); } } diff --git a/test/unit/mongo/IndexUtilsTest.php b/test/unit/mongo/IndexUtilsTest.php index bbce2432..b26499a0 100644 --- a/test/unit/mongo/IndexUtilsTest.php +++ b/test/unit/mongo/IndexUtilsTest.php @@ -410,13 +410,14 @@ protected function oneCustomAndThreeInternalTripodViewIndexesShouldBeCreated($mo // params that we know. // a) one custom index is created based on the view specification // b) three internal indexes are always created - $mockCollection->expects($this->exactly(4)) + $mockCollection->expects($this->exactly(5)) ->method('createIndex') ->withConsecutive( array(array(_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1, _ID_KEY.'.'._ID_TYPE => 1), array('background'=>$background)), array(array(_ID_KEY.'.'._ID_TYPE => 1), array('background' => $background)), array(array('value.'._IMPACT_INDEX => 1), array('background' => $background)), - array(array('rdf:type.u' => 1), array('background' => $background)) + [['_cts' => 1], ['background' => $background]], + [['rdf:type.u' => 1, '_cts' => 1], ['background' => $background]] ); } @@ -430,13 +431,14 @@ protected function oneCustomAndThreeInternalTripodTableIndexesShouldBeCreated($m // params that we know. // a) one custom index is created based on the view specification // b) three internal indexes are always created - $mockCollection->expects($this->exactly(4)) + $mockCollection->expects($this->exactly(5)) ->method('createIndex') ->withConsecutive( array(array(_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1, _ID_KEY.'.'._ID_TYPE => 1), array('background'=>$background)), array(array(_ID_KEY.'.'._ID_TYPE => 1), array('background' => $background)), array(array('value.'._IMPACT_INDEX => 1), array('background' => $background)), - array(array('rdf:type.u' => 1), array('background'=>$background)) + [['_cts' => 1], ['background' => $background]], + [['rdf:type.u' => 1], ['background' => $background]] ); } @@ -449,12 +451,13 @@ protected function threeInternalTripodSearchDocIndexesShouldBeCreated($mockColle // create index is called 3 times, each time with a different set of // params that we know. // for search docs only internal indexes are created - $mockCollection->expects($this->exactly(3)) + $mockCollection->expects($this->exactly(4)) ->method('createIndex') ->withConsecutive( array(array(_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1), array('background' => $background)), array(array(_ID_KEY.'.'._ID_TYPE => 1), array('background' => $background)), - array(array(_IMPACT_INDEX => 1), array('background' => $background)) + array(array(_IMPACT_INDEX => 1), array('background' => $background)), + [['_cts' => 1], ['background' => $background]] ); } @@ -526,7 +529,7 @@ protected function setConfigForViewIndexes($mockConfig) array( "_id" => "v_testview", "ensureIndexes" => array( - array("rdf:type.u"=>1) + array("rdf:type.u"=>1, '_cts' => 1) ), "from" => "CBD_testing", "type" => "temp:TestType", diff --git a/test/unit/mongo/MongoSearchProviderTest.php b/test/unit/mongo/MongoSearchProviderTest.php index ba58509a..fba4959e 100644 --- a/test/unit/mongo/MongoSearchProviderTest.php +++ b/test/unit/mongo/MongoSearchProviderTest.php @@ -598,6 +598,160 @@ public function testDeleteSearchDocumentsByTypeIdDoNotDeleteNonMatchingDocuments $this->assertEquals(1, $newSearchDocumentCount, "Should have 1 search documents since there is one search document with 'i_search_list' type that does not match delete type."); } + public function testCountSearchDocuments() + { + $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver') + ->setConstructorArgs(['CBD_testing', 'tripod_php_testing']) + ->getMock(); + + $collection = $this->getMockBuilder('\MongoDB\Collection') + ->disableOriginalConstructor() + ->setMethods(['count']) + ->getMock(); + $search = $this->getMockBuilder('\Tripod\Mongo\MongoSearchProvider') + ->setMethods(['getCollectionForSearchSpec']) + ->setConstructorArgs([$tripod]) + ->getMock(); + + $search->expects($this->once()) + ->method('getCollectionForSearchSpec') + ->with('i_search_list') + ->will($this->returnValue($collection)); + + $collection->expects($this->once()) + ->method('count') + ->with(['_id.type' => 'i_search_list']) + ->will($this->returnValue(21)); + + $this->assertEquals(21, $search->count('i_search_list')); + } + + public function testCountSearchDocumentsWithFilters() + { + $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver') + ->setConstructorArgs(['CBD_testing', 'tripod_php_testing']) + ->getMock(); + + $filters = ['_cts' => ['$lte' => new \MongoDB\BSON\UTCDateTime(null)]]; + $query = array_merge(['_id.type' => 'i_search_list'], $filters); + $collection = $this->getMockBuilder('\MongoDB\Collection') + ->disableOriginalConstructor() + ->setMethods(['count']) + ->getMock(); + $search = $this->getMockBuilder('\Tripod\Mongo\MongoSearchProvider') + ->setMethods(['getCollectionForSearchSpec']) + ->setConstructorArgs([$tripod]) + ->getMock(); + + $search->expects($this->once()) + ->method('getCollectionForSearchSpec') + ->with('i_search_list') + ->will($this->returnValue($collection)); + + $collection->expects($this->once()) + ->method('count') + ->with($query) + ->will($this->returnValue(89)); + + $this->assertEquals(89, $search->count('i_search_list', $filters)); + } + + public function testDeleteSearchDocumentsBySearchId() + { + $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver') + ->setConstructorArgs(['CBD_testing', 'tripod_php_testing']) + ->getMock(); + + $collection = $this->getMockBuilder('\MongoDB\Collection') + ->disableOriginalConstructor() + ->setMethods(['deleteMany']) + ->getMock(); + + $deleteResult = $this->getMockBuilder('MongoDB\DeleteResult') + ->setMethods(['getDeletedCount']) + ->disableOriginalConstructor() + ->getMock(); + + $deleteResult->expects($this->once()) + ->method('getDeletedCount') + ->will($this->returnValue(9)); + + $search = $this->getMockBuilder('\Tripod\Mongo\MongoSearchProvider') + ->setMethods(['getCollectionForSearchSpec', 'getSearchDocumentSpecification']) + ->setConstructorArgs([$tripod]) + ->getMock(); + + $search->expects($this->once()) + ->method('getSearchDocumentSpecification') + ->with('i_search_list') + ->will($this->returnValue(['_id' => 'i_search_list'])); + + $search->expects($this->once()) + ->method('getCollectionForSearchSpec') + ->with('i_search_list') + ->will($this->returnValue($collection)); + + $collection->expects($this->once()) + ->method('deleteMany') + ->with(['_id.type' => 'i_search_list']) + ->will($this->returnValue($deleteResult)); + + $this->assertEquals(9, $search->deleteSearchDocumentsByTypeId('i_search_list')); + } + + public function testDeleteSearchDocumentsBySearchIdWithTimestamp() + { + $timestamp = new \MongoDB\BSON\UTCDateTime(null); + + $query = [ + '_id.type' => 'i_search_list', + '$or' => [ + [\_CREATED_TS => ['$lt' => $timestamp]], + [\_CREATED_TS => ['$exists' => false]] + ] + ]; + + $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver') + ->setConstructorArgs(['CBD_testing', 'tripod_php_testing']) + ->getMock(); + + $collection = $this->getMockBuilder('\MongoDB\Collection') + ->disableOriginalConstructor() + ->setMethods(['deleteMany']) + ->getMock(); + + $deleteResult = $this->getMockBuilder('MongoDB\DeleteResult') + ->setMethods(['getDeletedCount']) + ->disableOriginalConstructor() + ->getMock(); + + $deleteResult->expects($this->once()) + ->method('getDeletedCount') + ->will($this->returnValue(9)); + + $search = $this->getMockBuilder('\Tripod\Mongo\MongoSearchProvider') + ->setMethods(['getCollectionForSearchSpec', 'getSearchDocumentSpecification']) + ->setConstructorArgs([$tripod]) + ->getMock(); + + $search->expects($this->once()) + ->method('getSearchDocumentSpecification') + ->with('i_search_list') + ->will($this->returnValue(['_id' => 'i_search_list'])); + + $search->expects($this->once()) + ->method('getCollectionForSearchSpec') + ->with('i_search_list') + ->will($this->returnValue($collection)); + + $collection->expects($this->once()) + ->method('deleteMany') + ->with($query) + ->will($this->returnValue($deleteResult)); + + $this->assertEquals(9, $search->deleteSearchDocumentsByTypeId('i_search_list', $timestamp)); + } + /** * @param \Tripod\Mongo\Driver $tripod * @param array $specs @@ -617,4 +771,4 @@ protected function getCountForSearchSpecs(\Tripod\Mongo\Driver $tripod, $specs = } return $count; } -} \ No newline at end of file +} diff --git a/test/unit/mongo/MongoTripodTablesTest.php b/test/unit/mongo/MongoTripodTablesTest.php index c6e71e95..85553575 100644 --- a/test/unit/mongo/MongoTripodTablesTest.php +++ b/test/unit/mongo/MongoTripodTablesTest.php @@ -1631,6 +1631,132 @@ public function testRemoveTableSpecDoesNotAffectInvalidation() // The table row should still be there, even if the tablespec no longer exists $this->assertGreaterThan(0, $collection->count(array('_id.type'=>'t_resource', 'value._impactIndex'=>array(_ID_RESOURCE=>$uri, _ID_CONTEXT=>$context)))); + } + + public function testCountTables() + { + $collection = $this->getMockBuilder('\MongoDB\Collection') + ->disableOriginalConstructor() + ->setMethods(['count']) + ->getMock(); + $tables = $this->getMockBuilder('\Tripod\Mongo\Composites\Tables') + ->setMethods(['getCollectionForTableSpec']) + ->setConstructorArgs(['tripod_php_testing', $collection, 'http://example.com/']) + ->getMock(); + + $tables->expects($this->once()) + ->method('getCollectionForTableSpec') + ->with('t_source_count') + ->will($this->returnValue($collection)); + + $collection->expects($this->once()) + ->method('count') + ->with(['_id.type' => 't_source_count']) + ->will($this->returnValue(50)); + + $this->assertEquals(50, $tables->count('t_source_count')); + } + + public function testCountTablesWithFilters() + { + $filters = ['_cts' => ['$lte' => new \MongoDB\BSON\UTCDateTime(null)]]; + $query = array_merge(['_id.type' => 't_source_count'], $filters); + $collection = $this->getMockBuilder('\MongoDB\Collection') + ->disableOriginalConstructor() + ->setMethods(['count']) + ->getMock(); + $tables = $this->getMockBuilder('\Tripod\Mongo\Composites\Tables') + ->setMethods(['getCollectionForTableSpec']) + ->setConstructorArgs(['tripod_php_testing', $collection, 'http://example.com/']) + ->getMock(); + + $tables->expects($this->once()) + ->method('getCollectionForTableSpec') + ->with('t_source_count') + ->will($this->returnValue($collection)); + + $collection->expects($this->once()) + ->method('count') + ->with($query) + ->will($this->returnValue(37)); + + $this->assertEquals(37, $tables->count('t_source_count', $filters)); + } + + public function testDeleteTableRowsByTableId() + { + $collection = $this->getMockBuilder('\MongoDB\Collection') + ->disableOriginalConstructor() + ->setMethods(['deleteMany']) + ->getMock(); + + $deleteResult = $this->getMockBuilder('MongoDB\DeleteResult') + ->setMethods(['getDeletedCount']) + ->disableOriginalConstructor() + ->getMock(); + + $deleteResult->expects($this->once()) + ->method('getDeletedCount') + ->will($this->returnValue(2)); + + $tables = $this->getMockBuilder('\Tripod\Mongo\Composites\Tables') + ->setMethods(['getCollectionForTableSpec']) + ->setConstructorArgs(['tripod_php_testing', $collection, 'http://example.com/']) + ->getMock(); + + $tables->expects($this->once()) + ->method('getCollectionForTableSpec') + ->with('t_source_count') + ->will($this->returnValue($collection)); + + $collection->expects($this->once()) + ->method('deleteMany') + ->with(['_id.type' => 't_source_count']) + ->will($this->returnValue($deleteResult)); + + $this->assertEquals(2, $tables->deleteTableRowsByTableId('t_source_count')); + } + + public function testDeleteTableRowsByTableIdWithTimestamp() + { + $timestamp = new \MongoDB\BSON\UTCDateTime(null); + + $query = [ + '_id.type' => 't_source_count', + '$or' => [ + [\_CREATED_TS => ['$lt' => $timestamp]], + [\_CREATED_TS => ['$exists' => false]] + ] + ]; + $collection = $this->getMockBuilder('\MongoDB\Collection') + ->disableOriginalConstructor() + ->setMethods(['deleteMany']) + ->getMock(); + + $deleteResult = $this->getMockBuilder('MongoDB\DeleteResult') + ->setMethods(['getDeletedCount']) + ->disableOriginalConstructor() + ->getMock(); + + $deleteResult->expects($this->once()) + ->method('getDeletedCount') + ->will($this->returnValue(11)); + + $tables = $this->getMockBuilder('\Tripod\Mongo\Composites\Tables') + ->setMethods(['getCollectionForTableSpec']) + ->setConstructorArgs(['tripod_php_testing', $collection, 'http://example.com/']) + ->getMock(); + + $tables->expects($this->once()) + ->method('getCollectionForTableSpec') + ->with('t_source_count') + ->will($this->returnValue($collection)); + + $collection->expects($this->once()) + ->method('deleteMany') + ->with($query) + ->will($this->returnValue($deleteResult)); + $this->assertEquals(11, $tables->deleteTableRowsByTableId('t_source_count', $timestamp)); } } diff --git a/test/unit/mongo/MongoTripodViewsTest.php b/test/unit/mongo/MongoTripodViewsTest.php index e26cb60b..918bf484 100644 --- a/test/unit/mongo/MongoTripodViewsTest.php +++ b/test/unit/mongo/MongoTripodViewsTest.php @@ -108,7 +108,9 @@ public function testGenerateView() ['typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array']] ); $actualView = $mongo->selectCollection('tripod_php_testing','views')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/3SplCtWGPqEyXcDiyhHQpA',"c"=>'http://talisaspire.com/',"type"=>'v_resource_full'))); - $this->assertEquals($expectedView,$actualView); + $this->assertEquals($expectedView['_id'], $actualView['_id']); + $this->assertEquals($expectedView['value'], $actualView['value']); + $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']); } /** @@ -179,7 +181,9 @@ public function testGenerateViewWithFilterRemovesFilteredDataButKeepsResourcesIn ['typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array']] ); $actualView = $mongo->selectCollection('tripod_php_testing','views')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_filter1'))); - $this->assertEquals($expectedView,$actualView); + $this->assertEquals($expectedView['_id'], $actualView['_id']); + $this->assertEquals($expectedView['value'], $actualView['value']); + $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']); } /** @@ -244,7 +248,9 @@ public function testGenerateViewWithFilterOnLiteralValue() $actualView = $mongo->selectCollection('tripod_php_testing','views')->findOne( array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_filter2')) ); - $this->assertEquals($expectedView,$actualView); + $this->assertEquals($expectedView['_id'], $actualView['_id']); + $this->assertEquals($expectedView['value'], $actualView['value']); + $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']); } /** @@ -315,7 +321,9 @@ public function testGenerateViewCorrectlyAfterUpdateAffectsFilter() ['typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array']] ); $actualView = $mongo->selectCollection('tripod_php_testing','views')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_filter1'))); - $this->assertEquals($expectedView,$actualView); + $this->assertEquals($expectedView['_id'], $actualView['_id']); + $this->assertEquals($expectedView['value'], $actualView['value']); + $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']); // Modify http://talisaspire.com/works/filter1 so that it is a Chapter (included in the view) not a Book (excluded from the view) $oldGraph = new \Tripod\ExtendedGraph(); @@ -382,7 +390,9 @@ public function testGenerateViewCorrectlyAfterUpdateAffectsFilter() ); $updatedView = $mongo->selectCollection('tripod_php_testing','views')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_filter1'))); - $this->assertEquals($expectedUpdatedView,$updatedView); + $this->assertEquals($expectedUpdatedView['_id'], $updatedView['_id']); + $this->assertEquals($expectedUpdatedView['value'], $updatedView['value']); + $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $updatedView['_cts']); } /** @@ -458,7 +468,9 @@ public function testGenerateViewContainingRdfSequence() ); $actualView = $mongo->selectCollection('tripod_php_testing','views')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_rdfsequence'))); - $this->assertEquals($expectedView,$actualView); + $this->assertEquals($expectedView['_id'], $actualView['_id']); + $this->assertEquals($expectedView['value'], $actualView['value']); + $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']); } public function testGenerateViewWithTTL() @@ -504,7 +516,9 @@ public function testGenerateViewWithTTL() ); // get the view direct from mongo $actualView = \Tripod\Mongo\Config::getInstance()->getCollectionForView('tripod_php_testing', 'v_resource_full_ttl')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/3SplCtWGPqEyXcDiyhHQpA',"c"=>'http://talisaspire.com/',"type"=>'v_resource_full_ttl'))); - $this->assertEquals($expectedView,$actualView); + $this->assertEquals($expectedView['_id'], $actualView['_id']); + $this->assertEquals($expectedView['value'], $actualView['value']); + $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']); } /** @@ -609,7 +623,9 @@ public function testGenerateViewWithCountAggregate() ); $actualView = \Tripod\Mongo\Config::getInstance()->getCollectionForView('tripod_php_testing', 'v_counts')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/works/4d101f63c10a6',"c"=>"http://talisaspire.com/","type"=>'v_counts'))); - $this->assertEquals($expectedView,$actualView); + $this->assertEquals($expectedView['_id'], $actualView['_id']); + $this->assertEquals($expectedView['value'], $actualView['value']); + $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']); } public function testGetViewWithNamespaces() @@ -2008,4 +2024,131 @@ public function testCursorNoExceptionThrownWhenCursorThrowsSomeExceptions() $mockTripodViews->getViewForResources(array($uri1),$viewType,$context); } -} \ No newline at end of file + + public function testCountViews() + { + $collection = $this->getMockBuilder('\MongoDB\Collection') + ->disableOriginalConstructor() + ->setMethods(['count']) + ->getMock(); + $views = $this->getMockBuilder('\Tripod\Mongo\Composites\Views') + ->setMethods(['getCollectionForViewSpec']) + ->setConstructorArgs(['tripod_php_testing', $collection, 'http://example.com/']) + ->getMock(); + + $views->expects($this->once()) + ->method('getCollectionForViewSpec') + ->with('v_some_spec') + ->will($this->returnValue($collection)); + + $collection->expects($this->once()) + ->method('count') + ->with(['_id.type' => 'v_some_spec']) + ->will($this->returnValue(101)); + + $this->assertEquals(101, $views->count('v_some_spec')); + } + + public function testCountViewsWithFilters() + { + $filters = ['_cts' => ['$lte' => new \MongoDB\BSON\UTCDateTime(null)]]; + $query = array_merge(['_id.type' => 'v_some_spec'], $filters); + $collection = $this->getMockBuilder('\MongoDB\Collection') + ->disableOriginalConstructor() + ->setMethods(['count']) + ->getMock(); + $views = $this->getMockBuilder('\Tripod\Mongo\Composites\Views') + ->setMethods(['getCollectionForViewSpec']) + ->setConstructorArgs(['tripod_php_testing', $collection, 'http://example.com/']) + ->getMock(); + + $views->expects($this->once()) + ->method('getCollectionForViewSpec') + ->with('v_some_spec') + ->will($this->returnValue($collection)); + + $collection->expects($this->once()) + ->method('count') + ->with($query) + ->will($this->returnValue(101)); + + $this->assertEquals(101, $views->count('v_some_spec', $filters)); + } + + public function testDeleteViewsByViewId() + { + $collection = $this->getMockBuilder('\MongoDB\Collection') + ->disableOriginalConstructor() + ->setMethods(['deleteMany']) + ->getMock(); + + $deleteResult = $this->getMockBuilder('MongoDB\DeleteResult') + ->setMethods(['getDeletedCount']) + ->disableOriginalConstructor() + ->getMock(); + + $deleteResult->expects($this->once()) + ->method('getDeletedCount') + ->will($this->returnValue(30)); + + $views = $this->getMockBuilder('\Tripod\Mongo\Composites\Views') + ->setMethods(['getCollectionForViewSpec']) + ->setConstructorArgs(['tripod_php_testing', $collection, 'http://example.com/']) + ->getMock(); + + $views->expects($this->once()) + ->method('getCollectionForViewSpec') + ->with('v_resource_full') + ->will($this->returnValue($collection)); + + $collection->expects($this->once()) + ->method('deleteMany') + ->with(['_id.type' => 'v_resource_full']) + ->will($this->returnValue($deleteResult)); + + $this->assertEquals(30, $views->deleteViewsByViewId('v_resource_full')); + } + + public function testDeleteViewsByViewIdWithTimestamp() + { + $timestamp = new \MongoDB\BSON\UTCDateTime(null); + + $query = [ + '_id.type' => 'v_resource_full', + '$or' => [ + [\_CREATED_TS => ['$lt' => $timestamp]], + [\_CREATED_TS => ['$exists' => false]] + ] + ]; + $collection = $this->getMockBuilder('\MongoDB\Collection') + ->disableOriginalConstructor() + ->setMethods(['deleteMany']) + ->getMock(); + + $deleteResult = $this->getMockBuilder('MongoDB\DeleteResult') + ->setMethods(['getDeletedCount']) + ->disableOriginalConstructor() + ->getMock(); + + $deleteResult->expects($this->once()) + ->method('getDeletedCount') + ->will($this->returnValue(30)); + + $views = $this->getMockBuilder('\Tripod\Mongo\Composites\Views') + ->setMethods(['getCollectionForViewSpec']) + ->setConstructorArgs(['tripod_php_testing', $collection, 'http://example.com/']) + ->getMock(); + + $views->expects($this->once()) + ->method('getCollectionForViewSpec') + ->with('v_resource_full') + ->will($this->returnValue($collection)); + + $collection->expects($this->once()) + ->method('deleteMany') + ->with($query) + ->will($this->returnValue($deleteResult)); + + $this->assertEquals(30, $views->deleteViewsByViewId('v_resource_full', $timestamp)); + } +}