Skip to content

Commit

Permalink
Merge pull request #125 from talis/allow_nonexpiring_ttl_views
Browse files Browse the repository at this point in the history 
Non-invalidating views
  • Loading branch information
@rsinger
rsinger authored Apr 16, 2018
2 parents e41f890 + 92488b4 commit 4ae950e
Show file tree
Hide file tree
Showing 7 changed files with 301 additions and 96 deletions.
3 changes: 2 additions & 1 deletion composer.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
"mongodb/mongodb": "1.0.4"
},
"require-dev": {
"phpunit/phpunit": "4.1.*"
"phpunit/phpunit": "4.1.*",
"squizlabs/php_codesniffer": "3.2.*"
}
}
76 changes: 51 additions & 25 deletions src/mongo/delegates/Views.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class Views extends CompositeBase
* @param null $stat
* @param string $readPreference
*/
function __construct($storeName, Collection $collection,$defaultContext,$stat=null,$readPreference = ReadPreference::RP_PRIMARY) // todo: $collection -> podname
public function __construct($storeName, Collection $collection,$defaultContext,$stat=null,$readPreference = ReadPreference::RP_PRIMARY) // todo: $collection -> podname
{
$this->storeName = $storeName;
$this->labeller = new Labeller();
Expand Down Expand Up @@ -75,40 +75,58 @@ public function getTypesInSpecifications()
* @param string $contextAlias
* @return array|mixed
*/
public function findImpactedComposites(Array $resourcesAndPredicates, $contextAlias)
public function findImpactedComposites(array $resourcesAndPredicates, $contextAlias)
{
$resources = array_keys($resourcesAndPredicates);

// This should never happen, but in the event that we have been passed an empty array or something
if(empty($resources))
{
return array();
if (empty($resourcesAndPredicates)) {
return [];
}

$contextAlias = $this->getContextAlias($contextAlias); // belt and braces

// build a filter - will be used for impactIndex detection and finding direct views to re-gen
$filter = array();
foreach ($resources as $resource)
{
$filter = [];
$changedTypes = [];
$typeKeys = [RDF_TYPE, $this->labeller->uri_to_alias(RDF_TYPE)];
foreach ($resourcesAndPredicates as $resource => $predicates) {
$resourceAlias = $this->labeller->uri_to_alias($resource);
// build $filter for queries to impact index
$filter[] = array("r"=>$resourceAlias,"c"=>$contextAlias);
$filter[] = [_ID_RESOURCE => $resourceAlias, _ID_CONTEXT => $contextAlias];
$rdfTypePredicates = array_intersect($predicates, $typeKeys);
if (!empty($rdfTypePredicates)) {
$changedTypes[] = $resourceAlias;
}
}

// first re-gen views where resources appear in the impact index
$query = array("value."._IMPACT_INDEX=>array('$in'=>$filter));
$query = ['value.' . _IMPACT_INDEX => ['$in' => $filter]];

if (!empty($changedTypes)) {
$query = ['$or' => [$query]];
foreach ($changedTypes as $resourceAlias) {
$query['$or'][] = [
_ID_KEY . '.' . _ID_RESOURCE => $resourceAlias,
_ID_KEY . '.' . _ID_CONTEXT => $contextAlias
];
}
}

$affectedViews = array();
foreach($this->config->getCollectionsForViews($this->storeName) as $collection)
{
$affectedViews = [];
foreach ($this->config->getCollectionsForViews($this->storeName) as $collection) {
$t = new \Tripod\Timer();
$t->start();
$views = $collection->find($query, array('projection' => array("_id"=>true)));
$views = $collection->find($query, ['projection' => ['_id'=>true]]);
$t->stop();
$this->timingLog(MONGO_FIND_IMPACTED, array('duration'=>$t->result(), 'query'=>$query, 'storeName'=>$this->storeName, 'collection'=>$collection));
foreach($views as $v)
{
$this->timingLog(
MONGO_FIND_IMPACTED,
[
'duration' => $t->result(),
'query' => $query,
'storeName' => $this->storeName,
'collection' => $collection
]
);
foreach ($views as $v) {
$affectedViews[] = $v;
}
}
Expand Down Expand Up @@ -292,11 +310,15 @@ public function generateViews($resources,$context=null)
foreach ($resources as $resource)
{
$resourceAlias = $this->labeller->uri_to_alias($resource);

$this->getLogger()->warning(
'Generating views',
['store' => $this->storeName, '_id' => $resourceAlias]
);
// delete any views this resource is involved in. It's type may have changed so it's not enough just to regen it with it's new type below.
foreach (Config::getInstance()->getViewSpecifications($this->storeName) as $type=>$spec)
{
if($spec['from']==$this->podName){

$this->config->getCollectionForView($this->storeName, $type, $this->readPreference)
->deleteOne(array("_id" => array("r"=>$resourceAlias,"c"=>$contextAlias,"type"=>$type)));
}
Expand Down Expand Up @@ -402,7 +424,7 @@ public function deleteViewsByViewId($viewId, $timestamp = null)
* @throws \Tripod\Exceptions\ViewException
* @return array
*/
public function generateView($viewId,$resource=null,$context=null,$queueName=null)
public function generateView($viewId, $resource = null, $context = null, $queueName = null)
{
$contextAlias = $this->getContextAlias($context);
$viewSpec = Config::getInstance()->getViewSpecification($this->storeName, $viewId);
Expand Down Expand Up @@ -479,15 +501,19 @@ public function generateView($viewId,$resource=null,$context=null,$queueName=nul
$buildImpactIndex=true;
if (isset($viewSpec['ttl'])) {
$buildImpactIndex=false;
$value[_EXPIRES] = \Tripod\Mongo\DateUtil::getMongoDate($this->getExpirySecFromNow($viewSpec['ttl']) * 1000);
if (is_int($viewSpec['ttl']) && $viewSpec['ttl'] > 0) {
$value[_EXPIRES] = \Tripod\Mongo\DateUtil::getMongoDate(
$this->getExpirySecFromNow($viewSpec['ttl']) * 1000
);
}
} else {
$value[_IMPACT_INDEX] = array($doc['_id']);
}

$this->doJoins($doc,$viewSpec['joins'],$value,$from,$contextAlias,$buildImpactIndex);
$this->doJoins($doc, $viewSpec['joins'], $value, $from, $contextAlias, $buildImpactIndex);

// add top level properties
$value[_GRAPHS][] = $this->extractProperties($doc,$viewSpec,$from);
$value[_GRAPHS][] = $this->extractProperties($doc, $viewSpec, $from);

$generatedView['value'] = $value;

Expand All @@ -501,7 +527,7 @@ public function generateView($viewId,$resource=null,$context=null,$queueName=nul
'duration'=>$t->result(),
'filter'=>$filter,
'from'=>$from));
$this->getStat()->timer(MONGO_CREATE_VIEW.".$viewId",$t->result());
$this->getStat()->timer(MONGO_CREATE_VIEW.".$viewId", $t->result());

$stat = ['count' => $count];
if (isset($jobOptions[ApplyOperation::TRACKING_KEY])) {
Expand Down
41 changes: 23 additions & 18 deletions test/unit/mongo/MongoTripodConfigTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -973,23 +973,28 @@ public function testSearchConfigNotPresent()

public function testGetAllTypesInSpecifications()
{
$types = $this->tripodConfig->getAllTypesInSpecifications("tripod_php_testing");
$this->assertEquals(11, count($types), "There should be 11 types based on the configured view, table and search specifications in config.json");
$expectedValues = array(
"acorn:Resource",
"acorn:ResourceForTruncating",
"acorn:Work",
"http://talisaspire.com/schema#Work2",
"acorn:Work2",
"bibo:Book",
"resourcelist:List",
"spec:User",
"bibo:Document",
"baseData:Wibble",
"baseData:DocWithSequence"
$types = $this->tripodConfig->getAllTypesInSpecifications('tripod_php_testing');
$this->assertEquals(
12,
count($types),
'There should be 12 types based on the configured view, table and search specifications in config.json'
);

foreach($expectedValues as $expected){
$expectedValues = [
'acorn:Resource',
'acorn:ResourceForTruncating',
'acorn:Work',
'http://talisaspire.com/schema#Work2',
'acorn:Work2',
'bibo:Book',
'resourcelist:List',
'spec:User',
'bibo:Document',
'baseData:Wibble',
'baseData:DocWithSequence',
'dctype:Event'
];

foreach ($expectedValues as $expected) {
$this->assertContains($expected, $types, "List of types should have contained $expected");
}
}
Expand Down Expand Up @@ -1334,7 +1339,7 @@ public function testTransactionLogIsWrittenToCorrectDBAndCollection()
$transactionColletion = $transactionMongo->selectCollection($newConfig['transaction_log']['database'], $newConfig['transaction_log']['collection']);
$transactionCount = $transactionColletion->count();
$transactionExampleDocument = $transactionColletion->findOne();
$this->assertEquals(24, $transactionCount);
$this->assertEquals(26, $transactionCount);
$this->assertContains('transaction_', $transactionExampleDocument["_id"]);
}

Expand Down Expand Up @@ -1758,4 +1763,4 @@ function()
$mockConfig->getCollectionForCBD('tripod_php_testing', 'CBD_testing', ReadPreference::RP_SECONDARY_PREFERRED);
$mockConfig->getCollectionForCBD('tripod_php_testing', 'CBD_testing', ReadPreference::RP_NEAREST);
}
}
}
4 changes: 2 additions & 2 deletions test/unit/mongo/MongoTripodTestBase.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ protected function getConfigLocation()

protected function setUp()
{
date_default_timezone_set('Europe/London');
date_default_timezone_set('UTC');

$config = json_decode(file_get_contents($this->getConfigLocation()), true);
if(getenv('TRIPOD_DATASOURCE_RS1_CONFIG'))
Expand Down Expand Up @@ -479,4 +479,4 @@ public function loadConfig(array $config)
{
parent::loadConfig($config);
}
}
}
Loading

0 comments on commit 4ae950e

Please sign in to comment.