diff --git a/composer.json b/composer.json index dab691ac..20419223 100644 --- a/composer.json +++ b/composer.json @@ -25,6 +25,7 @@ "mongodb/mongodb": "1.0.4" }, "require-dev": { - "phpunit/phpunit": "4.1.*" + "phpunit/phpunit": "4.1.*", + "squizlabs/php_codesniffer": "3.2.*" } } diff --git a/src/mongo/delegates/Views.class.php b/src/mongo/delegates/Views.class.php index ce810417..f405a4f4 100644 --- a/src/mongo/delegates/Views.class.php +++ b/src/mongo/delegates/Views.class.php @@ -28,7 +28,7 @@ class Views extends CompositeBase * @param null $stat * @param string $readPreference */ - function __construct($storeName, Collection $collection,$defaultContext,$stat=null,$readPreference = ReadPreference::RP_PRIMARY) // todo: $collection -> podname + public function __construct($storeName, Collection $collection,$defaultContext,$stat=null,$readPreference = ReadPreference::RP_PRIMARY) // todo: $collection -> podname { $this->storeName = $storeName; $this->labeller = new Labeller(); @@ -75,40 +75,58 @@ public function getTypesInSpecifications() * @param string $contextAlias * @return array|mixed */ - public function findImpactedComposites(Array $resourcesAndPredicates, $contextAlias) + public function findImpactedComposites(array $resourcesAndPredicates, $contextAlias) { - $resources = array_keys($resourcesAndPredicates); - // This should never happen, but in the event that we have been passed an empty array or something - if(empty($resources)) - { - return array(); + if (empty($resourcesAndPredicates)) { + return []; } $contextAlias = $this->getContextAlias($contextAlias); // belt and braces // build a filter - will be used for impactIndex detection and finding direct views to re-gen - $filter = array(); - foreach ($resources as $resource) - { + $filter = []; + $changedTypes = []; + $typeKeys = [RDF_TYPE, $this->labeller->uri_to_alias(RDF_TYPE)]; + foreach ($resourcesAndPredicates as $resource => $predicates) { $resourceAlias = $this->labeller->uri_to_alias($resource); // build $filter for queries to impact index - $filter[] = array("r"=>$resourceAlias,"c"=>$contextAlias); + $filter[] = [_ID_RESOURCE => $resourceAlias, _ID_CONTEXT => $contextAlias]; + $rdfTypePredicates = array_intersect($predicates, $typeKeys); + if (!empty($rdfTypePredicates)) { + $changedTypes[] = $resourceAlias; + } } // first re-gen views where resources appear in the impact index - $query = array("value."._IMPACT_INDEX=>array('$in'=>$filter)); + $query = ['value.' . _IMPACT_INDEX => ['$in' => $filter]]; + + if (!empty($changedTypes)) { + $query = ['$or' => [$query]]; + foreach ($changedTypes as $resourceAlias) { + $query['$or'][] = [ + _ID_KEY . '.' . _ID_RESOURCE => $resourceAlias, + _ID_KEY . '.' . _ID_CONTEXT => $contextAlias + ]; + } + } - $affectedViews = array(); - foreach($this->config->getCollectionsForViews($this->storeName) as $collection) - { + $affectedViews = []; + foreach ($this->config->getCollectionsForViews($this->storeName) as $collection) { $t = new \Tripod\Timer(); $t->start(); - $views = $collection->find($query, array('projection' => array("_id"=>true))); + $views = $collection->find($query, ['projection' => ['_id'=>true]]); $t->stop(); - $this->timingLog(MONGO_FIND_IMPACTED, array('duration'=>$t->result(), 'query'=>$query, 'storeName'=>$this->storeName, 'collection'=>$collection)); - foreach($views as $v) - { + $this->timingLog( + MONGO_FIND_IMPACTED, + [ + 'duration' => $t->result(), + 'query' => $query, + 'storeName' => $this->storeName, + 'collection' => $collection + ] + ); + foreach ($views as $v) { $affectedViews[] = $v; } } @@ -292,11 +310,15 @@ public function generateViews($resources,$context=null) foreach ($resources as $resource) { $resourceAlias = $this->labeller->uri_to_alias($resource); - + $this->getLogger()->warning( + 'Generating views', + ['store' => $this->storeName, '_id' => $resourceAlias] + ); // delete any views this resource is involved in. It's type may have changed so it's not enough just to regen it with it's new type below. foreach (Config::getInstance()->getViewSpecifications($this->storeName) as $type=>$spec) { if($spec['from']==$this->podName){ + $this->config->getCollectionForView($this->storeName, $type, $this->readPreference) ->deleteOne(array("_id" => array("r"=>$resourceAlias,"c"=>$contextAlias,"type"=>$type))); } @@ -402,7 +424,7 @@ public function deleteViewsByViewId($viewId, $timestamp = null) * @throws \Tripod\Exceptions\ViewException * @return array */ - public function generateView($viewId,$resource=null,$context=null,$queueName=null) + public function generateView($viewId, $resource = null, $context = null, $queueName = null) { $contextAlias = $this->getContextAlias($context); $viewSpec = Config::getInstance()->getViewSpecification($this->storeName, $viewId); @@ -479,15 +501,19 @@ public function generateView($viewId,$resource=null,$context=null,$queueName=nul $buildImpactIndex=true; if (isset($viewSpec['ttl'])) { $buildImpactIndex=false; - $value[_EXPIRES] = \Tripod\Mongo\DateUtil::getMongoDate($this->getExpirySecFromNow($viewSpec['ttl']) * 1000); + if (is_int($viewSpec['ttl']) && $viewSpec['ttl'] > 0) { + $value[_EXPIRES] = \Tripod\Mongo\DateUtil::getMongoDate( + $this->getExpirySecFromNow($viewSpec['ttl']) * 1000 + ); + } } else { $value[_IMPACT_INDEX] = array($doc['_id']); } - $this->doJoins($doc,$viewSpec['joins'],$value,$from,$contextAlias,$buildImpactIndex); + $this->doJoins($doc, $viewSpec['joins'], $value, $from, $contextAlias, $buildImpactIndex); // add top level properties - $value[_GRAPHS][] = $this->extractProperties($doc,$viewSpec,$from); + $value[_GRAPHS][] = $this->extractProperties($doc, $viewSpec, $from); $generatedView['value'] = $value; @@ -501,7 +527,7 @@ public function generateView($viewId,$resource=null,$context=null,$queueName=nul 'duration'=>$t->result(), 'filter'=>$filter, 'from'=>$from)); - $this->getStat()->timer(MONGO_CREATE_VIEW.".$viewId",$t->result()); + $this->getStat()->timer(MONGO_CREATE_VIEW.".$viewId", $t->result()); $stat = ['count' => $count]; if (isset($jobOptions[ApplyOperation::TRACKING_KEY])) { diff --git a/test/unit/mongo/MongoTripodConfigTest.php b/test/unit/mongo/MongoTripodConfigTest.php index 8a9b11b4..ab88b968 100644 --- a/test/unit/mongo/MongoTripodConfigTest.php +++ b/test/unit/mongo/MongoTripodConfigTest.php @@ -973,23 +973,28 @@ public function testSearchConfigNotPresent() public function testGetAllTypesInSpecifications() { - $types = $this->tripodConfig->getAllTypesInSpecifications("tripod_php_testing"); - $this->assertEquals(11, count($types), "There should be 11 types based on the configured view, table and search specifications in config.json"); - $expectedValues = array( - "acorn:Resource", - "acorn:ResourceForTruncating", - "acorn:Work", - "http://talisaspire.com/schema#Work2", - "acorn:Work2", - "bibo:Book", - "resourcelist:List", - "spec:User", - "bibo:Document", - "baseData:Wibble", - "baseData:DocWithSequence" + $types = $this->tripodConfig->getAllTypesInSpecifications('tripod_php_testing'); + $this->assertEquals( + 12, + count($types), + 'There should be 12 types based on the configured view, table and search specifications in config.json' ); - - foreach($expectedValues as $expected){ + $expectedValues = [ + 'acorn:Resource', + 'acorn:ResourceForTruncating', + 'acorn:Work', + 'http://talisaspire.com/schema#Work2', + 'acorn:Work2', + 'bibo:Book', + 'resourcelist:List', + 'spec:User', + 'bibo:Document', + 'baseData:Wibble', + 'baseData:DocWithSequence', + 'dctype:Event' + ]; + + foreach ($expectedValues as $expected) { $this->assertContains($expected, $types, "List of types should have contained $expected"); } } @@ -1334,7 +1339,7 @@ public function testTransactionLogIsWrittenToCorrectDBAndCollection() $transactionColletion = $transactionMongo->selectCollection($newConfig['transaction_log']['database'], $newConfig['transaction_log']['collection']); $transactionCount = $transactionColletion->count(); $transactionExampleDocument = $transactionColletion->findOne(); - $this->assertEquals(24, $transactionCount); + $this->assertEquals(26, $transactionCount); $this->assertContains('transaction_', $transactionExampleDocument["_id"]); } @@ -1758,4 +1763,4 @@ function() $mockConfig->getCollectionForCBD('tripod_php_testing', 'CBD_testing', ReadPreference::RP_SECONDARY_PREFERRED); $mockConfig->getCollectionForCBD('tripod_php_testing', 'CBD_testing', ReadPreference::RP_NEAREST); } -} \ No newline at end of file +} diff --git a/test/unit/mongo/MongoTripodTestBase.php b/test/unit/mongo/MongoTripodTestBase.php index e1ff84dd..4a41d97f 100644 --- a/test/unit/mongo/MongoTripodTestBase.php +++ b/test/unit/mongo/MongoTripodTestBase.php @@ -84,7 +84,7 @@ protected function getConfigLocation() protected function setUp() { - date_default_timezone_set('Europe/London'); + date_default_timezone_set('UTC'); $config = json_decode(file_get_contents($this->getConfigLocation()), true); if(getenv('TRIPOD_DATASOURCE_RS1_CONFIG')) @@ -479,4 +479,4 @@ public function loadConfig(array $config) { parent::loadConfig($config); } -} \ No newline at end of file +} diff --git a/test/unit/mongo/MongoTripodViewsTest.php b/test/unit/mongo/MongoTripodViewsTest.php index 918bf484..e94fcfd0 100644 --- a/test/unit/mongo/MongoTripodViewsTest.php +++ b/test/unit/mongo/MongoTripodViewsTest.php @@ -4,8 +4,8 @@ require_once 'src/mongo/Driver.class.php'; require_once 'src/mongo/delegates/Views.class.php'; -use \Tripod\Mongo\Composites\Views; use \MongoDB\Client; +use Tripod\ExtendedGraph; class MongoTripodViewsTest extends MongoTripodTestBase { /** @@ -521,6 +521,116 @@ public function testGenerateViewWithTTL() $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']); } + public function testNonExpiringViewWithNegativeTTL() + { + $views = new \Tripod\Mongo\Composites\Views( + $this->viewsConstParams[0], + $this->viewsConstParams[1], + $this->viewsConstParams[2] + ); + + $view = $views->getViewForResource( + 'http://talisaspire.com/events/1234', + 'v_event_no_expiration' + ); + + // should have no impact index and no _expires + $expectedView = [ + '_id' => [ + 'r' => 'http://talisaspire.com/events/1234', + 'c '=> 'http://talisaspire.com/', + 'type' => 'v_event_no_expiration' + ], + 'value' => [ + _GRAPHS => [ + [ + '_id' => [ + 'r' => 'http://talisaspire.com/events/1234', + 'c'=> 'http://talisaspire.com/' + ], + 'rdf:type' => ['u' => 'dctype:Event'], + 'dct:references' => ['u' => 'http://talisaspire.com/resources/1234'], + 'dct:created' => ['l' => '2018-04-09T00:00:00Z'], + 'dct:title' => ['l' => 'A significant event'] + ], + [ + '_id' => [ + 'r' => 'http://talisaspire.com/resources/1234', + 'c' => 'http://talisaspire.com/' + ], + 'dct:title' => ['l' => 'A real piece of work'], + 'dct:creator' => ['l' => 'Anne Author'] + ] + ], + ] + ]; + // get the view direct from mongo + $actualView = \Tripod\Mongo\Config::getInstance() + ->getCollectionForView('tripod_php_testing', 'v_event_no_expiration') + ->findOne( + [ + '_id' => [ + 'r' => 'http://talisaspire.com/events/1234', + 'c' => 'http://talisaspire.com/', + 'type' => 'v_event_no_expiration' + ] + ] + ); + $this->assertEquals( + $expectedView['_id'], + $actualView['_id'], + '_id does not match expected', + 0.0, + 10, + true + ); + $this->assertContains( + $expectedView['value'][_GRAPHS][0], + $actualView['value'][_GRAPHS] + ); + $this->assertContains( + $expectedView['value'][_GRAPHS][1], + $actualView['value'][_GRAPHS] + ); + $this->assertCount(2, $actualView['value'][_GRAPHS]); + $this->assertArrayNotHasKey(_EXPIRES, $actualView['value']); + $this->assertArrayNotHasKey(_IMPACT_INDEX, $actualView['value']); + $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']); + + // Fetch the joined resource and change it + $graph = $this->tripod->describeResource('http://talisaspire.com/resources/1234'); + + $updatedGraph = new ExtendedGraph($graph->to_ntriples()); + $updatedGraph->replace_literal_triple( + 'http://talisaspire.com/resources/1234', + 'http://purl.org/dc/terms/title', + 'A real piece of work', + 'A literal treasure' + ); + + // This should not affect the view at all + $this->tripod->saveChanges($graph, $updatedGraph); + + $view = $views->getViewForResource( + 'http://talisaspire.com/events/1234', + 'v_event_no_expiration' + ); + + // get the view direct from mongo, it should the same as earlier + $actualView2 = \Tripod\Mongo\Config::getInstance() + ->getCollectionForView('tripod_php_testing', 'v_event_no_expiration') + ->findOne( + [ + '_id' => [ + 'r' => 'http://talisaspire.com/events/1234', + 'c' => 'http://talisaspire.com/', + 'type' => 'v_event_no_expiration' + ] + ] + ); + $this->assertEquals($actualView, $actualView2); + } + /** * This test covers a bug we found where maxJoins causes a difference between the URIs included in the right hand * side vs. the left hand side of the join. Consider the data: @@ -837,70 +947,75 @@ public function testDeletionOfResourceTriggersViewRegeneration() $uri2 = 'http://example.com/resources/' . uniqid(); $originalGraph->add_resource_triple($uri2, RDF_TYPE, $labeller->qname_to_uri('bibo:Document')); - $originalGraph->add_literal_triple($uri2, $labeller->qname_to_uri('dct:subject'), 'Things grouped by no specific criteria'); + $originalGraph->add_literal_triple( + $uri2, + $labeller->qname_to_uri('dct:subject'), + 'Things grouped by no specific criteria' + ); $originalGraph->add_resource_triple($uri1, $labeller->qname_to_uri('dct:isVersionOf'), $uri2); - $tripod = new \Tripod\Mongo\Driver('CBD_testing', 'tripod_php_testing', array('defaultContext'=>$context)); + $tripod = new \Tripod\Mongo\Driver('CBD_testing', 'tripod_php_testing', ['defaultContext' => $context]); $tripod->saveChanges(new \Tripod\ExtendedGraph(), $originalGraph); - $collections = \Tripod\Mongo\Config::getInstance()->getCollectionsForViews('tripod_php_testing', array('v_resource_full', 'v_resource_full_ttl', 'v_resource_to_single_source')); + $collections = \Tripod\Mongo\Config::getInstance()->getCollectionsForViews( + 'tripod_php_testing', + ['v_resource_full', 'v_resource_full_ttl', 'v_resource_to_single_source'] + ); - foreach($collections as $collection) - { - $this->assertGreaterThan(0, $collection->count(array('_id.r'=>$labeller->uri_to_alias($uri1), '_id.c'=>$context))); + foreach ($collections as $collection) { + $this->assertGreaterThan( + 0, + $collection->count(['_id.r' => $labeller->uri_to_alias($uri1), '_id.c' => $context]) + ); } - $subjectsAndPredicatesOfChange = array( - $labeller->uri_to_alias($uri1)=>array( - 'rdf:type','searchterms:topic','dct:isVersionOf' - ) - ); + $subjectsAndPredicatesOfChange = [ + $labeller->uri_to_alias($uri1) =>[ + 'rdf:type', 'searchterms:topic','dct:isVersionOf' + ] + ]; /** @var \Tripod\Mongo\Driver|PHPUnit_Framework_MockObject_MockObject $mockTripod */ $mockTripod = $this->getMock( '\Tripod\Mongo\Driver', - array( - 'getDataUpdater', 'getComposite' - ), - array( + ['getDataUpdater', 'getComposite'], + [ 'CBD_testing', 'tripod_php_testing', - array( - 'defaultContext'=>$context, - OP_ASYNC=>array( - OP_TABLES=>true, - OP_VIEWS=>false, - OP_SEARCH=>true - ) - ) - ) + [ + 'defaultContext' => $context, + OP_ASYNC => [ + OP_TABLES => true, + OP_VIEWS => false, + OP_SEARCH => true + ] + ] + ] ); $mockTripodUpdates = $this->getMock( '\Tripod\Mongo\Updates', - array( - 'processSyncOperations', - 'queueAsyncOperations' - ), - array( + ['processSyncOperations', 'queueAsyncOperations'], + [ $mockTripod, - array( - OP_ASYNC=>array( + [ + OP_ASYNC => [ OP_TABLES=>true, OP_VIEWS=>false, OP_SEARCH=>true - ) - ) - ) + ] + ] + ] ); - $mockViews = $this->getMock('\Tripod\Mongo\Composites\Views', - array('generateViewsForResourcesOfType'), - array( + $mockViews = $this->getMock( + '\Tripod\Mongo\Composites\Views', + ['generateViewsForResourcesOfType'], + [ 'tripod_php_testing', \Tripod\Mongo\Config::getInstance()->getCollectionForCBD('tripod_php_testing', 'CBD_testing'), $context - ) + ] ); $mockTripod->expects($this->once()) @@ -938,34 +1053,44 @@ public function testDeletionOfResourceTriggersViewRegeneration() $view = $mockTripod->getComposite(OP_VIEWS); $this->assertInstanceOf('\Tripod\Mongo\Composites\Views', $view); - $expectedImpactedSubjects = array( + $expectedImpactedSubjects = [ new \Tripod\Mongo\ImpactedSubject( - array( + [ _ID_RESOURCE=>$labeller->uri_to_alias($uri1), _ID_CONTEXT=>$context - ), + ], OP_VIEWS, 'tripod_php_testing', 'CBD_testing', // Don't include v_resource_full_ttl, because TTL views don't include impactIndex - array('v_resource_full', 'v_resource_to_single_source', 'v_resource_filter1', 'v_resource_filter2', 'v_resource_rdfsequence') + [ + 'v_resource_full', + 'v_resource_full_ttl', + 'v_resource_to_single_source', + 'v_resource_filter1', + 'v_resource_filter2', + 'v_resource_rdfsequence' + ] ) - ); + ]; $impactedSubjects = $view->getImpactedSubjects($subjectsAndPredicatesOfChange, $context); $this->assertEquals($expectedImpactedSubjects, $impactedSubjects); - foreach($impactedSubjects as $subject) - { + foreach ($impactedSubjects as $subject) { $view->update($subject); } // This should be 0, because we mocked the actual adding of the regenerated view. If it's zero, however, // it means we successfully deleted the views with $uri1 in the impactIndex - foreach($collections as $collection) - { - $this->assertEquals(0, $collection->count(array('value._impactIndex'=>array('r'=>$labeller->uri_to_alias($uri1), 'c'=>$context)))); + foreach ($collections as $collection) { + $this->assertEquals( + 0, + $collection->count( + ['value._impactIndex' => ['r' => $labeller->uri_to_alias($uri1), 'c' => $context]] + ) + ); } } diff --git a/test/unit/mongo/data/config.json b/test/unit/mongo/data/config.json index c25596ea..b8d78c30 100644 --- a/test/unit/mongo/data/config.json +++ b/test/unit/mongo/data/config.json @@ -221,6 +221,18 @@ "include": ["dct:subject", "rdf:type"] } } + }, + { + "_id": "v_event_no_expiration", + "from": "CBD_testing", + "type": "dctype:Event", + "ttl": -1, + "include": ["rdf:type", "dct:created", "dct:title", "dct:references"], + "joins": { + "dct:references" : { + "include": ["dct:title", "dct:creator"] + } + } } ], "search_config": { diff --git a/test/unit/mongo/data/resources.json b/test/unit/mongo/data/resources.json index 7cbeafa9..a7c6771e 100644 --- a/test/unit/mongo/data/resources.json +++ b/test/unit/mongo/data/resources.json @@ -845,5 +845,41 @@ "u":"http://talisaspire.com/schema#Work" } ] - } + }, + { + "_id": { + "r": "http://talisaspire.com/events/1234", + "c": "http://talisaspire.com/" + }, + "rdf:type": { + "u": "dctype:Event" + }, + "dct:created": { + "l": "2018-04-09T00:00:00Z" + }, + "dct:title": { + "l": "A significant event" + }, + "dct:references": { + "u": "http://talisaspire.com/resources/1234" + } + }, + { + "_id": { + "r": "http://talisaspire.com/resources/1234", + "c": "http://talisaspire.com/" + }, + "rdf:type": { + "u": "dctype:Text" + }, + "dct:created": { + "l": "2018-04-09T00:00:00Z" + }, + "dct:title": { + "l": "A real piece of work" + }, + "dct:creator": { + "l": "Anne Author" + } + } ] \ No newline at end of file