diff --git a/.travis.yml b/.travis.yml index 3a44cd74..102d01c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ php: - 7.0 - 5.6 - 5.5 -- 5.4 services: - redis - docker diff --git a/composer.json b/composer.json index 7b323ef3..9659031e 100644 --- a/composer.json +++ b/composer.json @@ -18,17 +18,17 @@ } ], "require": { - "php" : ">=5.4", + "php" : ">=5.5", "semsol/arc2": "v2.2.4", "chrisboulton/php-resque": "dev-master#98fde571db008a8b48e73022599d1d1c07d4a7b5", "monolog/monolog" : "~1.13", "mongodb/mongodb": "1.0.4" }, "require-dev": { - "phpunit/phpunit": "4.1.*", + "phpunit/phpunit": "4.8.", "squizlabs/php_codesniffer": "3.2.*" - }, + }, "autoload": { "classmap": ["src/"] - } + } } diff --git a/docker-compose.yml b/docker-compose.yml index 3cf8dd7f..95fae1a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,4 @@ mongo: - image: mongo/2.6.12:latest + image: rossfsinger/mongo-2.6.12:latest ports: - "27017:27017" \ No newline at end of file diff --git a/src/mongo/Config.class.php b/src/mongo/Config.class.php index 62ecd5c7..7108981f 100644 --- a/src/mongo/Config.class.php +++ b/src/mongo/Config.class.php @@ -38,61 +38,61 @@ class Config implements IConfigInstance * The defined database indexes, keyed by database name * @var array */ - private $indexes = array(); + private $indexes = []; /** * @var array */ - private $cardinality = array(); + private $cardinality = []; /** * The connection strings for each defined database * @var array */ - private $dbConfig = array(); + private $dbConfig = []; /** * All of the defined viewSpecs * @var array */ - private $viewSpecs = array(); + private $viewSpecs = []; /** * All of the defined tableSpecs * @var array */ - private $tableSpecs = array(); + private $tableSpecs = []; /** * All of the defined searchDocSpecs * @var array */ - protected $searchDocSpecs = array(); + protected $searchDocSpecs = []; /** * Defined database configuration: dbname, collections, etc. * @var array */ - private $databases = array(); + private $databases = []; /** * All defined namespaces * * @var array */ - protected $ns = array(); + protected $ns = []; /** * The transaction log db config * @var array */ - protected $tConfig = array(); + protected $tConfig = []; /** * The value should be the name of a class that implement iTripodSearchProvider keyed by storename * @var array */ - protected $searchProviderClassName = array(); + protected $searchProviderClassName = []; /** * All of the predicates associated with a particular spec document @@ -105,12 +105,12 @@ class Config implements IConfigInstance * A simple map between collection names and the database name they belong to * @var array */ - protected $collectionDatabases = array(); + protected $collectionDatabases = []; /** * @var array */ - protected $activeMongoConnections = array(); + protected $activeMongoConnections = []; /** * @var string @@ -120,12 +120,12 @@ class Config implements IConfigInstance /** * @var array */ - protected $dataSources = array(); + protected $dataSources = []; /** * @var array */ - protected $podConnections = array(); + protected $podConnections = []; const VALIDATE_MIN = 'MIN'; const VALIDATE_MAX = 'MAX'; @@ -140,17 +140,25 @@ class Config implements IConfigInstance * Database connections, keyed by datasource, so we're not inadvertently opening many db connections through getDatabase() * @var array */ - protected $connections = array(); + protected $connections = []; /** * @var int */ protected $mongoCursorTimeout = 30000; + protected $batchSize = [ + OP_TABLES => 100, + OP_SEARCH => 100, + OP_VIEWS => 25 + ]; + /** * Config should not be instantiated directly: use Config::getInstance() */ - private function __construct() {} + private function __construct() + { + } /** * @return int @@ -177,107 +185,84 @@ public function setMongoCursorTimeout($mongoCursorTimeout) protected function loadConfig(array $config) { $this->config = $config; - if (array_key_exists('namespaces',$config)) - { + if (array_key_exists('namespaces', $config)) { $this->ns = $config['namespaces']; } - $this->defaultContext = $this->getMandatoryKey("defaultContext",$config); + $this->defaultContext = $this->getMandatoryKey('defaultContext', $config); - foreach($this->getMandatoryKey('data_sources', $config) as $source=>$c) - { - if(!array_key_exists('type', $c)) - { + foreach ($this->getMandatoryKey('data_sources', $config) as $source => $c) { + if (!array_key_exists('type', $c)) { throw new \Tripod\Exceptions\ConfigException("No 'type' set for data source $source"); } - if(!array_key_exists('connection', $c)) - { + if (!array_key_exists('connection', $c)) { throw new \Tripod\Exceptions\ConfigException("No connection information set for data source $source"); } $this->dataSources[$source] = $c; } - $transactionConfig = $this->getMandatoryKey("transaction_log",$config); - $this->tConfig["data_source"] = $this->getMandatoryKey('data_source', $transactionConfig, 'transaction_log'); - if(!isset($this->dataSources[$this->tConfig['data_source']])) - { + $transactionConfig = $this->getMandatoryKey('transaction_log', $config); + $this->tConfig['data_source'] = $this->getMandatoryKey('data_source', $transactionConfig, 'transaction_log'); + if (!isset($this->dataSources[$this->tConfig['data_source']])) { throw new \Tripod\Exceptions\ConfigException("Transaction log data source, " . $this->tConfig['data_source'] . ", was not defined"); } - $this->tConfig["database"] = $this->getMandatoryKey("database",$transactionConfig,'transaction_log'); - $this->tConfig["collection"] = $this->getMandatoryKey("collection",$transactionConfig,'transaction_log'); + $this->tConfig["database"] = $this->getMandatoryKey("database", $transactionConfig, 'transaction_log'); + $this->tConfig["collection"] = $this->getMandatoryKey("collection", $transactionConfig, 'transaction_log'); // A 'pod' corresponds to a logical database - $this->databases = $this->getMandatoryKey("stores",$config); + $this->databases = $this->getMandatoryKey("stores", $config); $defaultDB = null; - foreach ($this->databases as $storeName=>$storeConfig) - { - $this->dbConfig[$storeName] = array ("data_source"=>$this->getMandatoryKey("data_source",$storeConfig)); - if(isset($storeConfig['database']) && !empty($storeConfig['database'])) - { + foreach ($this->databases as $storeName => $storeConfig) { + $this->dbConfig[$storeName] = ["data_source" => $this->getMandatoryKey("data_source", $storeConfig)]; + if (isset($storeConfig['database']) && !empty($storeConfig['database'])) { $this->dbConfig[$storeName]["database"]=$storeConfig['database']; - } - else - { + } else { $this->dbConfig[$storeName]["database"] = $storeName; } - $this->cardinality[$storeName] = array(); - $this->indexes[$storeName] = array(); - $this->podConnections[$storeName] = array(); - if(isset($storeConfig["pods"])) - { - foreach($storeConfig["pods"] as $podName=>$podConfig) - { + $this->cardinality[$storeName] = []; + $this->indexes[$storeName] = []; + $this->podConnections[$storeName] = []; + if (isset($storeConfig["pods"])) { + foreach ($storeConfig["pods"] as $podName => $podConfig) { $dataSource = (isset($podConfig['data_source']) ? $podConfig['data_source'] : $storeConfig['data_source']); $this->podConnections[$storeName][$podName] = $dataSource; // Set cardinality, also checking against defined namespaces - if (array_key_exists('cardinality', $podConfig)) - { + if (array_key_exists('cardinality', $podConfig)) { // Test that the namespace exists for each cardinality rule defined $cardinality = $podConfig['cardinality']; - foreach ($cardinality as $qname=>$cardinalityValue) - { + foreach ($cardinality as $qname => $cardinalityValue) { $namespaces = explode(':', $qname); // just grab the first element $namespace = array_shift($namespaces); - if (array_key_exists($namespace, $this->ns)) - { + if (array_key_exists($namespace, $this->ns)) { $this->cardinality[$storeName][$podName][] = $cardinality; - } - else - { + } else { throw new \Tripod\Exceptions\ConfigException("Cardinality '{$qname}' does not have the namespace defined"); } } - } - else - { - $this->cardinality[$storeName][$podName] = array(); + } else { + $this->cardinality[$storeName][$podName] = []; } - $this->cardinality[$storeName][$podName] = (array_key_exists("cardinality",$podConfig)) ? $podConfig['cardinality'] : array(); + $this->cardinality[$storeName][$podName] = array_key_exists("cardinality", $podConfig) ? $podConfig['cardinality'] : []; // Ensure indexes are legal - if (array_key_exists("indexes",$podConfig)) - { - $this->indexes[$storeName][$podName] = array(); - foreach($podConfig["indexes"] as $indexName=>$indexFields) - { + if (array_key_exists("indexes", $podConfig)) { + $this->indexes[$storeName][$podName] = []; + foreach ($podConfig["indexes"] as $indexName => $indexFields) { // check no more than 1 indexField is an array to ensure Mongo will be able to create compound indexes - if (count($indexFields)>1) - { + if (count($indexFields) > 1) { $fieldsThatAreArrays = 0; - foreach ($indexFields as $field=>$fieldVal) - { - $cardinalityField = str_replace('.value','',$field); - if (!array_key_exists($cardinalityField,$this->cardinality[$storeName][$podName])||$this->cardinality[$storeName][$podName][$cardinalityField]!=1) - { + foreach ($indexFields as $field => $fieldVal) { + $cardinalityField = str_replace('.value', '', $field); + if (!array_key_exists($cardinalityField, $this->cardinality[$storeName][$podName]) || + $this->cardinality[$storeName][$podName][$cardinalityField] != 1) { $fieldsThatAreArrays++; } - if ($fieldsThatAreArrays>1) - { + if ($fieldsThatAreArrays > 1) { throw new \Tripod\Exceptions\ConfigException("Compound index $indexName has more than one field with cardinality > 1 - mongo will not be able to build this index"); } } @@ -289,29 +274,31 @@ protected function loadConfig(array $config) } } } - $searchConfig = (array_key_exists("search_config",$storeConfig)) ? $storeConfig["search_config"] : array(); - $this->searchDocSpecs[$storeName] = array(); - if(!empty($searchConfig)){ + if (isset($storeConfig['batch_sizes'])) { + foreach ([OP_TABLES, OP_SEARCH, OP_VIEWS] as $op) { + if (isset($storeConfig['batch_sizes'][$op]) && is_numeric($storeConfig['batch_sizes'][$op])) { + $this->batchSize[$op] = intval($storeConfig['batch_sizes'][$op]); + } + } + } + $searchConfig = array_key_exists("search_config", $storeConfig) ? $storeConfig["search_config"] : []; + $this->searchDocSpecs[$storeName] = []; + if (!empty($searchConfig)) { $this->searchProviderClassName[$storeName] = $this->getMandatoryKey('search_provider', $searchConfig, 'search'); // Load search doc specs if search_config is set $searchDocSpecs = $this->getMandatoryKey('search_specifications', $searchConfig, 'search'); - foreach ($searchDocSpecs as $spec) - { - if(!isset($spec[_ID_KEY])) - { + foreach ($searchDocSpecs as $spec) { + if (!isset($spec[_ID_KEY])) { throw new \Tripod\Exceptions\ConfigException("Search document spec does not contain " . _ID_KEY); } - if(!isset($spec['from']) || !in_array($spec['from'], $this->getPods($storeName))) - { + if (!isset($spec['from']) || !in_array($spec['from'], $this->getPods($storeName))) { throw new \Tripod\Exceptions\ConfigException("'" . $spec[_ID_KEY] . "[\"from\"]' property not set or references an undefined pod"); } - if(!isset($spec['filter'])) - { + if (!isset($spec['filter'])) { throw new \Tripod\Exceptions\ConfigException("'" . $spec[_ID_KEY] . "[\"filter\"]' property not set"); } - if(!isset($spec['fields']) && !isset($spec['joins'])) - { + if (!isset($spec['fields']) && !isset($spec['joins'])) { throw new \Tripod\Exceptions\ConfigException("'" . $spec[_ID_KEY] . "' contains no 'fields' or 'joins' properties"); } @@ -334,8 +321,8 @@ protected function loadConfig(array $config) } // Load view specs - $viewSpecs = (array_key_exists("view_specifications",$storeConfig)) ? $storeConfig["view_specifications"] : array(); - $this->viewSpecs[$storeName] = array(); + $viewSpecs = (array_key_exists("view_specifications",$storeConfig)) ? $storeConfig["view_specifications"] : []; + $this->viewSpecs[$storeName] = []; foreach ($viewSpecs as $spec) { if(!isset($spec[_ID_KEY])) @@ -366,8 +353,8 @@ protected function loadConfig(array $config) } // Load table specs - $tableSpecs = (array_key_exists("table_specifications",$storeConfig)) ? $storeConfig["table_specifications"] : array(); - $this->tableSpecs[$storeName] = array(); + $tableSpecs = (array_key_exists("table_specifications",$storeConfig)) ? $storeConfig["table_specifications"] : []; + $this->tableSpecs[$storeName] = []; foreach ($tableSpecs as $spec) { $this->validateTableSpec($spec); @@ -763,7 +750,7 @@ protected function validateComputedArithmeticSpec(array $spec, array $availableF protected function getFieldNamesInSpec(array $spec) { - $fieldNames = array(); + $fieldNames = []; if(isset($spec['fields'])) { foreach($spec['fields'] as $field) @@ -813,7 +800,7 @@ protected function getFieldNamesInSpec(array $spec) */ protected function getDefinedPredicatesInSpecs($storename) { - $predicates = array(); + $predicates = []; $specs = array_merge($this->getTableSpecifications($storename), $this->getSearchDocumentSpecifications($storename)); foreach($specs as $spec) { @@ -834,7 +821,7 @@ protected function getDefinedPredicatesInSpecs($storename) */ protected function getDefinedPredicatesInSpecBlock(array $block) { - $predicates = array(); + $predicates = []; // If the spec has a "type" property, include rdf:type if(isset($block['type'])) { @@ -933,7 +920,7 @@ protected function getDefinedPredicatesInSpecBlock(array $block) */ protected function getPredicateAliasesFromPredicateProperty($predicate) { - $predicates = array(); + $predicates = []; if(is_string($predicate) && !empty($predicate)) { $predicates[] = $this->getLabeller()->uri_to_alias($predicate); @@ -957,7 +944,7 @@ protected function getPredicateAliasesFromPredicateProperty($predicate) */ protected function getPredicatesFromPredicateFunctions($array) { - $predicates = array(); + $predicates = []; if(is_array($array)) { if(isset($array['predicates'])) @@ -978,7 +965,7 @@ protected function getPredicatesFromPredicateFunctions($array) */ protected function getPredicatesFromFilterCondition($filter) { - $predicates = array(); + $predicates = []; $regex = "/(^|\b)(\w+\:\w+)\.(l|u)(\b|$)/"; foreach($filter as $key=>$condition) { @@ -1030,7 +1017,7 @@ public function getDefinedPredicatesInSpec($storename, $specId) { return $this->specPredicates[$storename][$specId]; } - return array(); + return []; } /** @@ -1144,7 +1131,7 @@ public function getIndexesGroupedByCollection($storeName) } // also add the indexes for any views/tables - $tableIndexes = array(); + $tableIndexes = []; foreach ($this->getTableSpecifications($storeName) as $tspec) { if (array_key_exists("ensureIndexes",$tspec)) @@ -1152,7 +1139,7 @@ public function getIndexesGroupedByCollection($storeName) // Indexes should be keyed by data_source if(!isset($tableIndexes[$tspec['to_data_source']])) { - $tableIndexes[$tspec['to_data_source']] = array(); + $tableIndexes[$tspec['to_data_source']] = []; } foreach ($tspec["ensureIndexes"] as $index) { @@ -1162,7 +1149,7 @@ public function getIndexesGroupedByCollection($storeName) } $indexes[TABLE_ROWS_COLLECTION] = $tableIndexes; - $viewIndexes = array(); + $viewIndexes = []; foreach ($this->getViewSpecifications($storeName) as $vspec) { if (array_key_exists("ensureIndexes",$vspec)) @@ -1170,7 +1157,7 @@ public function getIndexesGroupedByCollection($storeName) // Indexes should be keyed by data_source if(!isset($viewIndexes[$vspec['to_data_source']])) { - $viewIndexes[$vspec['to_data_source']] = array(); + $viewIndexes[$vspec['to_data_source']] = []; } foreach ($vspec["ensureIndexes"] as $index) { @@ -1229,7 +1216,7 @@ public function isPodWithinStore($storeName,$pod) */ public function getPods($storeName) { - return (array_key_exists($storeName,$this->podConnections)) ? array_keys($this->podConnections[$storeName]) : array(); + return (array_key_exists($storeName,$this->podConnections)) ? array_keys($this->podConnections[$storeName]) : []; } /** @@ -1404,13 +1391,13 @@ public function getSearchDocumentSpecifications($storeName, $type=null, $justRet if(!isset($this->searchDocSpecs[$storeName]) || empty($this->searchDocSpecs[$storeName])) { - return array(); + return []; } - $specs = array(); + $specs = []; if(empty($type)){ if($justReturnSpecId){ - $specIds = array(); + $specIds = []; foreach($this->searchDocSpecs[$storeName] as $specId=>$spec){ $specIds[] = $specId; } @@ -1475,7 +1462,7 @@ public function getTableSpecification($storeName, $tid) */ public function getTableSpecifications($storeName) { - return (isset($this->tableSpecs[$storeName]) ? $this->tableSpecs[$storeName] : array()); + return (isset($this->tableSpecs[$storeName]) ? $this->tableSpecs[$storeName] : []); } /** @@ -1486,7 +1473,7 @@ public function getTableSpecifications($storeName) */ public function getViewSpecifications($storeName) { - return (isset($this->viewSpecs[$storeName]) ? $this->viewSpecs[$storeName] : array()); + return (isset($this->viewSpecs[$storeName]) ? $this->viewSpecs[$storeName] : []); } /** @@ -1569,7 +1556,7 @@ protected function getLabeller() */ private function getSpecificationTypes(Array $specifications, $podName=null) { - $types = array(); + $types = []; foreach($specifications as $spec){ if(!empty($podName)){ @@ -1659,7 +1646,7 @@ private function isConnectionStringValidForRepSet($connStr) */ private function findFieldsInTableSpec($fieldName, $spec) { - $fields = array(); + $fields = []; if(is_array($spec) && !empty($spec)) { if(array_key_exists($fieldName, $spec)) @@ -1743,7 +1730,7 @@ protected function getConnectionForDataSource($dataSource) { throw new \Tripod\Exceptions\ConfigException("Data source '{$dataSource}' not in configuration"); } - $connectionOptions = array(); + $connectionOptions = []; $ds = $this->dataSources[$dataSource]; $connectionOptions['connectTimeoutMS'] = (isset($ds['connectTimeoutMS']) ? $ds['connectTimeoutMS'] : DEFAULT_MONGO_CONNECT_TIMEOUT_MS); @@ -1884,17 +1871,17 @@ public function getCollectionForTable($storeName, $tableId, $readPreference = Re * @throws \Tripod\Exceptions\ConfigException * @return Collection[] */ - public function getCollectionsForTables($storeName, array $tables = array(), $readPreference = ReadPreference::RP_PRIMARY_PREFERRED) + public function getCollectionsForTables($storeName, array $tables = [], $readPreference = ReadPreference::RP_PRIMARY_PREFERRED) { if(!isset($this->tableSpecs[$storeName])) { - return array(); + return []; } if(empty($tables)) { $tables = array_keys($this->tableSpecs[$storeName]); } - $dataSources = array(); + $dataSources = []; foreach($tables as $table) { if(isset($this->tableSpecs[$storeName][$table])) @@ -1907,7 +1894,7 @@ public function getCollectionsForTables($storeName, array $tables = array(), $re } } - $collections = array(); + $collections = []; foreach(array_unique($dataSources) as $dataSource) { $collections[] = $this->getMongoCollection( @@ -1925,17 +1912,17 @@ public function getCollectionsForTables($storeName, array $tables = array(), $re * @throws \Tripod\Exceptions\ConfigException * @return Collection[] */ - public function getCollectionsForViews($storeName, array $views = array(), $readPreference = ReadPreference::RP_PRIMARY_PREFERRED) + public function getCollectionsForViews($storeName, array $views = [], $readPreference = ReadPreference::RP_PRIMARY_PREFERRED) { if(!isset($this->viewSpecs[$storeName])) { - return array(); + return []; } if(empty($views)) { $views = array_keys($this->viewSpecs[$storeName]); } - $dataSources = array(); + $dataSources = []; foreach($views as $view) { if(isset($this->viewSpecs[$storeName][$view])) @@ -1948,7 +1935,7 @@ public function getCollectionsForViews($storeName, array $views = array(), $read } } - $collections = array(); + $collections = []; foreach(array_unique($dataSources) as $dataSource) { $collections[] = $this->getMongoCollection( @@ -1966,17 +1953,17 @@ public function getCollectionsForViews($storeName, array $views = array(), $read * @throws \Tripod\Exceptions\ConfigException * @return Collection[] */ - public function getCollectionsForSearch($storeName, array $searchSpecIds = array(), $readPreference = ReadPreference::RP_PRIMARY_PREFERRED) + public function getCollectionsForSearch($storeName, array $searchSpecIds = [], $readPreference = ReadPreference::RP_PRIMARY_PREFERRED) { if(!isset($this->searchDocSpecs[$storeName])) { - return array(); + return []; } if(empty($searchSpecIds)) { $searchSpecIds = array_keys($this->searchDocSpecs[$storeName]); } - $dataSources = array(); + $dataSources = []; foreach($searchSpecIds as $searchSpec) { if(isset($this->searchDocSpecs[$storeName][$searchSpec])) @@ -1989,7 +1976,7 @@ public function getCollectionsForSearch($storeName, array $searchSpecIds = array } } - $collections = array(); + $collections = []; foreach(array_unique($dataSources) as $dataSource) { $collections[] = $this->getMongoCollection( @@ -2187,4 +2174,15 @@ public function serialize() { return $this->config; } + + /** + * Return the maximum batch size for async operations + * + * @param string $operation Async operation, e.g. OP_TABLES, OP_VIEWS + * @return integer + */ + public function getBatchSize($operation) + { + return isset($this->batchSize[$operation]) ? $this->batchSize[$operation] : 1; + } } diff --git a/src/mongo/Driver.class.php b/src/mongo/Driver.class.php index f493eaa1..e249d985 100755 --- a/src/mongo/Driver.class.php +++ b/src/mongo/Driver.class.php @@ -61,7 +61,7 @@ class Driver extends DriverBase implements \Tripod\IDriver *
  • readPreference: The Read preference to set for Mongo: Default is ReadPreference::RP_PRIMARY_PREFERRED
    • *
  • retriesToGetLock: Retries to do when unable to get lock on a document, default is 20
    • */ - public function __construct($podName, $storeName, $opts=array()) + public function __construct($podName, $storeName, $opts = []) { $opts = array_merge(array( diff --git a/src/mongo/IConfigInstance.php b/src/mongo/IConfigInstance.php index 8ad37ed2..9e5f1137 100644 --- a/src/mongo/IConfigInstance.php +++ b/src/mongo/IConfigInstance.php @@ -281,6 +281,14 @@ public function getCollectionForJobGroups($storeName, $readPreference = ReadPref */ public function getTransactionLogDatabase($readPreference = ReadPreference::RP_PRIMARY_PREFERRED); + /** + * Return the maximum batch size for async operations + * + * @param string $operation Async operation, e.g. OP_TABLES, OP_VIEWS + * @return integer + */ + public function getBatchSize($operation); + /** * @return string */ diff --git a/src/mongo/base/CompositeBase.class.php b/src/mongo/base/CompositeBase.class.php index bf1d050e..08838ac7 100644 --- a/src/mongo/base/CompositeBase.class.php +++ b/src/mongo/base/CompositeBase.class.php @@ -2,6 +2,8 @@ namespace Tripod\Mongo\Composites; +use \Tripod\Mongo\JobGroup; + /** * Class CompositeBase * @package Tripod\Mongo\Composites @@ -12,70 +14,65 @@ abstract class CompositeBase extends \Tripod\Mongo\DriverBase implements \Tripod * @var \Tripod\Mongo\Jobs\ApplyOperation */ protected $applyOperation; + /** * Returns an array of ImpactedSubjects based on the subjects and predicates of change * @param array $subjectsAndPredicatesOfChange * @param string $contextAlias * @return \Tripod\Mongo\ImpactedSubject[] */ - public function getImpactedSubjects(Array $subjectsAndPredicatesOfChange,$contextAlias) + public function getImpactedSubjects(array $subjectsAndPredicatesOfChange, $contextAlias) { - $candidates = array(); - $filter = array(); - $subjectsToAlias = array(); - foreach(array_keys($subjectsAndPredicatesOfChange) as $s){ + $candidates = []; + $filter = []; + $subjectsToAlias = []; + foreach (array_keys($subjectsAndPredicatesOfChange) as $s) { $resourceAlias = $this->labeller->uri_to_alias($s); $subjectsToAlias[$s] = $resourceAlias; // build $filter for queries to impact index - $filter[] = array(_ID_RESOURCE=>$resourceAlias,_ID_CONTEXT=>$contextAlias); + $filter[] = [_ID_RESOURCE=>$resourceAlias, _ID_CONTEXT=>$contextAlias]; } - $query = array(_ID_KEY=>array('$in'=>$filter)); - $docs = $this->getCollection()->find($query, array( - 'projection' => array(_ID_KEY=>true, 'rdf:type'=>true) - )); + $query = [_ID_KEY => ['$in' => $filter]]; + $docs = $this->getCollection()->find( + $query, + ['projection' => [_ID_KEY => true, 'rdf:type' => true]] + ); $types = $this->getTypesInSpecifications(); - if($this->getCollection()->count($query) !== 0 ) { - foreach($docs as $doc) - { + if ($this->getCollection()->count($query) !== 0) { + foreach ($docs as $doc) { $docResource = $doc[_ID_KEY][_ID_RESOURCE]; $docContext = $doc[_ID_KEY][_ID_CONTEXT]; $docHash = md5($docResource.$docContext); - $docTypes = array(); - if(isset($doc["rdf:type"])) { - if(isset($doc["rdf:type"][VALUE_URI])){ - $docTypes[] = $doc["rdf:type"][VALUE_URI]; + $docTypes = []; + if (isset($doc['rdf:type'])) { + if (isset($doc['rdf:type'][VALUE_URI])) { + $docTypes[] = $doc['rdf:type'][VALUE_URI]; } else { - foreach($doc["rdf:type"] as $t){ - if(isset($t[VALUE_URI])) - { + foreach ($doc['rdf:type'] as $t) { + if (isset($t[VALUE_URI])) { $docTypes[] = $t[VALUE_URI]; } } } } - $currentSubjectProperties = array(); - if(isset($subjectsAndPredicatesOfChange[$docResource])) - { + $currentSubjectProperties = []; + if (isset($subjectsAndPredicatesOfChange[$docResource])) { $currentSubjectProperties = $subjectsAndPredicatesOfChange[$docResource]; - } - elseif(isset($subjectsToAlias[$docResource]) && - isset($subjectsAndPredicatesOfChange[$subjectsToAlias[$docResource]])) - { + } elseif (isset($subjectsToAlias[$docResource]) && + isset($subjectsAndPredicatesOfChange[$subjectsToAlias[$docResource]])) { $currentSubjectProperties = $subjectsAndPredicatesOfChange[$subjectsToAlias[$docResource]]; } - foreach($docTypes as $type) - { - if($this->checkIfTypeShouldTriggerOperation($type, $types, $currentSubjectProperties)) { - if(!array_key_exists($this->getPodName(), $candidates)) - { - $candidates[$this->getPodName()] = array(); + foreach ($docTypes as $type) { + if ($this->checkIfTypeShouldTriggerOperation($type, $types, $currentSubjectProperties)) { + if (!array_key_exists($this->getPodName(), $candidates)) { + $candidates[$this->getPodName()] = []; } - if(!array_key_exists($docHash, $candidates[$this->getPodName()])){ - $candidates[$this->getPodName()][$docHash] = array('id'=>$doc[_ID_KEY]); + if (!array_key_exists($docHash, $candidates[$this->getPodName()])) { + $candidates[$this->getPodName()][$docHash] = ['id'=>$doc[_ID_KEY]]; } } } @@ -83,40 +80,37 @@ public function getImpactedSubjects(Array $subjectsAndPredicatesOfChange,$contex } // add to this any composites - foreach($this->findImpactedComposites($subjectsAndPredicatesOfChange, $contextAlias) as $doc) { + foreach ($this->findImpactedComposites($subjectsAndPredicatesOfChange, $contextAlias) as $doc) { $spec = $this->getSpecification($this->storeName, $doc[_ID_KEY]['type']); - if(is_array($spec) && array_key_exists('from', $spec)){ - if(!array_key_exists($spec['from'], $candidates)) - { - $candidates[$spec['from']] = array(); + if (is_array($spec) && array_key_exists('from', $spec)) { + if (!array_key_exists($spec['from'], $candidates)) { + $candidates[$spec['from']] = []; } $docHash = md5($doc[_ID_KEY][_ID_RESOURCE] . $doc[_ID_KEY][_ID_CONTEXT]); - if(!array_key_exists($docHash, $candidates[$spec['from']])){ - $candidates[$spec['from']][$docHash] = array( - 'id'=>array( + if (!array_key_exists($docHash, $candidates[$spec['from']])) { + $candidates[$spec['from']][$docHash] = [ + 'id' => [ _ID_RESOURCE=>$doc[_ID_KEY][_ID_RESOURCE], _ID_CONTEXT=>$doc[_ID_KEY][_ID_CONTEXT], - ) - ); + ] + ]; } - if(!array_key_exists('specTypes', $candidates[$spec['from']][$docHash])) { - $candidates[$spec['from']][$docHash]['specTypes'] = array(); + if (!array_key_exists('specTypes', $candidates[$spec['from']][$docHash])) { + $candidates[$spec['from']][$docHash]['specTypes'] = []; } // Save the specification type so we only have to regen resources in that table type - if(!in_array($doc[_ID_KEY][_ID_TYPE], $candidates[$spec['from']][$docHash]['specTypes'])) - { + if (!in_array($doc[_ID_KEY][_ID_TYPE], $candidates[$spec['from']][$docHash]['specTypes'])) { $candidates[$spec['from']][$docHash]['specTypes'][] = $doc[_ID_KEY][_ID_TYPE]; } } } // convert operations to subjects - $impactedSubjects = array(); - foreach(array_keys($candidates) as $podName){ - foreach($candidates[$podName] as $candidate) - { - $specTypes = (isset($candidate['specTypes']) ? $candidate['specTypes'] : array()); + $impactedSubjects = []; + foreach (array_keys($candidates) as $podName) { + foreach ($candidates[$podName] as $candidate) { + $specTypes = (isset($candidate['specTypes']) ? $candidate['specTypes'] : []); $impactedSubjects[] = new \Tripod\Mongo\ImpactedSubject($candidate['id'], $this->getOperationType(), $this->getStoreName(), $podName, $specTypes); } } @@ -128,14 +122,14 @@ public function getImpactedSubjects(Array $subjectsAndPredicatesOfChange,$contex * Returns an array of the rdf types that will trigger the specification * @return array */ - public abstract function getTypesInSpecifications(); + abstract public function getTypesInSpecifications(); /** * @param array $resourcesAndPredicates * @param string $contextAlias * @return mixed // @todo: This may eventually return a either a Cursor or array */ - public abstract function findImpactedComposites(Array $resourcesAndPredicates,$contextAlias); + abstract public function findImpactedComposites(array $resourcesAndPredicates, $contextAlias); /** * Returns the specification config @@ -143,7 +137,7 @@ public abstract function findImpactedComposites(Array $resourcesAndPredicates,$c * @param string $specId The specification id * @return array|null */ - public abstract function getSpecification($storeName, $specId); + abstract public function getSpecification($storeName, $specId); /** * Test if the a particular type appears in the array of types associated with a particular spec and that the changeset @@ -153,20 +147,20 @@ public abstract function getSpecification($storeName, $specId); * @param array $subjectPredicates * @return bool */ - protected function checkIfTypeShouldTriggerOperation($rdfType, array $validTypes, Array $subjectPredicates) + protected function checkIfTypeShouldTriggerOperation($rdfType, array $validTypes, array $subjectPredicates) { // We don't know if this is an alias or a fqURI, nor what is in the valid types, necessarily - $types = array($rdfType); - try - { + $types = [$rdfType]; + try { $types[] = $this->labeller->qname_to_uri($rdfType); + } catch (\Tripod\Exceptions\LabellerException $e) { + // Not a qname, apparently } - catch(\Tripod\Exceptions\LabellerException $e) {} - try - { + try { $types[] = $this->labeller->uri_to_alias($rdfType); + } catch (\Tripod\Exceptions\LabellerException $e) { + // Not a declared uri, apparently } - catch(\Tripod\Exceptions\LabellerException $e) {} $intersectingTypes = array_unique(array_intersect($types, $validTypes)); // If views have a matching type *at all*, the operation is triggered @@ -175,15 +169,38 @@ protected function checkIfTypeShouldTriggerOperation($rdfType, array $validTypes /** * For mocking - * + * * @return \Tripod\Mongo\Jobs\ApplyOperation */ protected function getApplyOperation() { - if(!isset($this->applyOperation)) - { + if (!isset($this->applyOperation)) { $this->applyOperation = new \Tripod\Mongo\Jobs\ApplyOperation(); } return $this->applyOperation; } + + /** + * Queues a batch of ImpactedSubjects in a single ApplyOperation job + * + * @param \Tripod\Mongo\ImpactedSubject[] $subjects Array of ImpactedSubjects + * @param string $queueName Queue name + * @param array $jobOptions Job options + * @return void + */ + protected function queueApplyJob(array $subjects, $queueName, array $jobOptions) + { + $this->getApplyOperation()->createJob($subjects, $queueName, $jobOptions); + } + + /** + * For mocking + * + * @param string $storeName + * @return JobGroup + */ + protected function getJobGroup($storeName) + { + return new JobGroup($storeName); + } } diff --git a/src/mongo/delegates/SearchIndexer.class.php b/src/mongo/delegates/SearchIndexer.class.php index 29f2b175..19a2c675 100644 --- a/src/mongo/delegates/SearchIndexer.class.php +++ b/src/mongo/delegates/SearchIndexer.class.php @@ -10,9 +10,10 @@ use Tripod\Mongo\ImpactedSubject; use Tripod\Mongo\Labeller; use Tripod\Mongo\Jobs\ApplyOperation; -use Tripod\Mongo\JobGroup; use \MongoDB\Driver\ReadPreference; use \MongoDB\Collection; +use Tripod\Mongo\Config; +use Tripod\Mongo\Driver; /** * Class SearchIndexer @@ -44,13 +45,7 @@ public function __construct(\Tripod\Mongo\Driver $tripod, $readPreference = Read $this->labeller = new Labeller(); $this->stat = $tripod->getStat(); $this->config = $this->getConfigInstance(); - $provider = $this->config->getSearchProviderClassName($this->tripod->getStoreName()); - - if (class_exists($provider)) { - $this->configuredProvider = new $provider($this->tripod); - } else { - throw new \Tripod\Exceptions\SearchException("Did not recognise Search Provider, or could not find class: $provider"); - } + $this->setSearchProvider($this->tripod, $this->config); $this->readPreference = $readPreference; } @@ -97,7 +92,7 @@ public function getOperationType() */ public function getSpecification($storeName, $specId) { - return $this->config->getSearchDocumentSpecification($storeName,$specId); + return $this->config->getSearchDocumentSpecification($storeName, $specId); } /** @@ -107,7 +102,7 @@ public function getSpecification($storeName, $specId) * @param string $podName * @param array | string | null $specType */ - public function generateAndIndexSearchDocuments($resourceUri, $context, $podName, $specType = array()) + public function generateAndIndexSearchDocuments($resourceUri, $context, $podName, $specType = []) { $mongoCollection = $this->config->getCollectionForCBD($this->storeName, $podName); @@ -118,42 +113,37 @@ public function generateAndIndexSearchDocuments($resourceUri, $context, $podName $searchProvider->deleteDocument($resourceUri, $context, $specType); // null means delete all documents for this resource //2. regenerate search documents for this resource - $documentsToIndex = array(); + $documentsToIndex = []; // first work out what its type is $query = array("_id"=>array( 'r'=>$this->labeller->uri_to_alias($resourceUri), 'c'=>$this->getContextAlias($context) )); - $resourceAndType = $mongoCollection->find($query,array( - 'projection' => array("_id"=>1,"rdf:type"=>1), - 'maxTimeMS' => $this->config->getMongoCursorTimeout() - )); - foreach ($resourceAndType as $rt) - { - if (array_key_exists("rdf:type",$rt)) - { - - $rdfTypes = array(); + $resourceAndType = $mongoCollection->find( + $query, + [ + 'projection' => ["_id" => 1, "rdf:type" => 1], + 'maxTimeMS' => $this->config->getMongoCursorTimeout() + ] + ); + foreach ($resourceAndType as $rt) { + if (array_key_exists("rdf:type", $rt)) { + $rdfTypes = []; - if (array_key_exists('u',$rt["rdf:type"])) - { + if (array_key_exists('u', $rt["rdf:type"])) { $rdfTypes[] = $rt["rdf:type"]['u']; - } - else - { + } else { // an array of types - foreach ($rt["rdf:type"] as $type) - { + foreach ($rt["rdf:type"] as $type) { $rdfTypes[] = $type['u']; } } $docs = $searchDocGenerator->generateSearchDocumentsBasedOnRdfTypes($rdfTypes, $resourceUri, $context); - foreach($docs as $d){ + foreach ($docs as $d) { $documentsToIndex[] = $d; } - } } @@ -192,35 +182,34 @@ public function generateSearchDocuments( // default collection $from = (isset($spec["from"])) ? $spec["from"] : $this->podName; - $types = array(); - if (is_array($spec["type"])) - { - foreach ($spec["type"] as $type) - { + $types = []; + if (is_array($spec["type"])) { + foreach ($spec["type"] as $type) { $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($type)); $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($type)); } - } - else - { + } else { $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($spec["type"])); $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($spec["type"])); } $filter = array('$or'=> $types); - if (isset($resource)) - { + if (isset($resource)) { $filter["_id"] = array(_ID_RESOURCE=>$this->labeller->uri_to_alias($resource),_ID_CONTEXT=>$contextAlias); } - $count = $this->config->getCollectionForCBD($this->getStoreName(), $from)->count($filter); - $docs = $this->config->getCollectionForCBD($this->getStoreName(), $from)->find($filter, array( - 'maxTimeMS' => $this->config->getMongoCursorTimeout() - )); + $count = $this->getConfigInstance()->getCollectionForCBD($this->getStoreName(), $from)->count($filter); + $docs = $this->getConfigInstance() + ->getCollectionForCBD($this->getStoreName(), $from) + ->find( + $filter, + ['maxTimeMS' => $this->getConfigInstance()->getMongoCursorTimeout()] + ); $jobOptions = []; + $subjects = []; if ($queueName && !$resourceUri) { $jobOptions['statsConfig'] = $this->getStatsConfig(); - $jobGroup = new JobGroup($this->storeName); + $jobGroup = $this->getJobGroup($this->storeName); $jobOptions[ApplyOperation::TRACKING_KEY] = $jobGroup->getId()->__toString(); $jobGroup->setJobCount($count); } @@ -234,7 +223,12 @@ public function generateSearchDocuments( array($searchDocumentType) ); - $this->getApplyOperation()->createJob(array($subject), $queueName, $jobOptions); + $subjects[] = $subject; + // Queue ApplyOperations jobs in batches rather than individually + if (count($subjects) >= $this->getConfigInstance()->getBatchSize(OP_SEARCH)) { + $this->queueApplyJob($subjects, $queueName, $jobOptions); + $subjects = []; + } } else { $this->generateAndIndexSearchDocuments( $doc[_ID_KEY][_ID_RESOURCE], @@ -245,13 +239,17 @@ public function generateSearchDocuments( } } + if (!empty($subjects)) { + $this->queueApplyJob($subjects, $queueName, $jobOptions); + } + $t->stop(); $this->timingLog(MONGO_CREATE_TABLE, array( 'type'=>$spec['type'], 'duration'=>$t->result(), 'filter'=>$filter, 'from'=>$from)); - $this->getStat()->timer(MONGO_CREATE_SEARCH_DOC.".$searchDocumentType",$t->result()); + $this->getStat()->timer(MONGO_CREATE_SEARCH_DOC.".$searchDocumentType", $t->result()); $stat = ['count' => $count]; if (isset($jobOptions[ApplyOperation::TRACKING_KEY])) { @@ -265,7 +263,7 @@ public function generateSearchDocuments( * @param string $context * @return array|mixed */ - public function findImpactedComposites(Array $resourcesAndPredicates, $context) + public function findImpactedComposites(array $resourcesAndPredicates, $context) { return $this->getSearchProvider()->findImpactedDocuments($resourcesAndPredicates, $context); } @@ -276,7 +274,7 @@ public function findImpactedComposites(Array $resourcesAndPredicates, $context) */ public function deleteSearchDocumentsByTypeId($typeId) { - return $this->getSearchProvider()->deleteSearchDocumentsByTypeId($typeId); + return $this->getSearchProvider()->deleteSearchDocumentsByTypeId($typeId); } @@ -293,7 +291,7 @@ protected function getSearchProvider() * @param string $context * @return \Tripod\Mongo\SearchDocuments */ - protected function getSearchDocumentGenerator(Collection $collection, $context ) + protected function getSearchDocumentGenerator(Collection $collection, $context) { return new \Tripod\Mongo\SearchDocuments($this->storeName, $collection, $context, $this->tripod->getStat()); } @@ -302,14 +300,38 @@ protected function getSearchDocumentGenerator(Collection $collection, $context ) * @param array $input * @return array */ - protected function deDupe(Array $input) + protected function deDupe(array $input) { - $output = array(); - foreach($input as $i){ - if(!in_array($i, $output)){ + $output = []; + foreach ($input as $i) { + if (!in_array($i, $output)) { $output[] = $i; } } return $output; } + + /** + * For mocking + * + * @param Driver $tripod Mongo Tripod Driver + * @param \Tripod\Mongo\IConfigInstance $config Mongo Tripod ConfigInstance + * @return void + * @throws \Tripod\Exceptions\SearchException If provider class cannot be found + */ + protected function setSearchProvider(Driver $tripod, \Tripod\Mongo\IConfigInstance $config = null) + { + if (is_null($config)) { + $config = $this->getConfigInstance(); + } + + $provider = $config->getSearchProviderClassName($tripod->getStoreName()); + if (class_exists($provider)) { + $this->configuredProvider = new $provider($tripod); + } else { + throw new \Tripod\Exceptions\SearchException( + "Did not recognise Search Provider, or could not find class: $provider" + ); + } + } } diff --git a/src/mongo/delegates/Tables.class.php b/src/mongo/delegates/Tables.class.php index ea8314b4..40b81945 100644 --- a/src/mongo/delegates/Tables.class.php +++ b/src/mongo/delegates/Tables.class.php @@ -7,7 +7,6 @@ use \Tripod\Mongo\Jobs\ApplyOperation; use \Tripod\Mongo\ImpactedSubject; use \Tripod\Mongo\Labeller; -use \Tripod\Mongo\JobGroup; use \MongoDB\Driver\ReadPreference; use \MongoDB\Collection; @@ -64,7 +63,7 @@ class Tables extends CompositeBase /** * @var array */ - protected $temporaryFields = array(); + protected $temporaryFields = []; /** * Construct accepts actual objects rather than strings as this class is a delegate of @@ -104,7 +103,7 @@ public function update(ImpactedSubject $subject) $resourceUri = $resource[_ID_RESOURCE]; $context = $resource[_ID_CONTEXT]; - $this->generateTableRowsForResource($resourceUri,$context,$subject->getSpecTypes()); + $this->generateTableRowsForResource($resourceUri, $context, $subject->getSpecTypes()); } /** @@ -126,7 +125,7 @@ public function findImpactedComposites(array $resourcesAndPredicates, $contextAl { $contextAlias = $this->getContextAlias($contextAlias); // belt and braces - $tablePredicates = array(); + $tablePredicates = []; foreach ($this->getConfigInstance()->getTableSpecifications($this->storeName) as $tableSpec) { if (isset($tableSpec[_ID_KEY])) { @@ -136,8 +135,8 @@ public function findImpactedComposites(array $resourcesAndPredicates, $contextAl } // build a filter - will be used for impactIndex detection and finding direct tables to re-gen - $tableFilters = array(); - $resourceFilters = array(); + $tableFilters = []; + $resourceFilters = []; foreach ($resourcesAndPredicates as $resource => $resourcePredicates) { $resourceAlias = $this->labeller->uri_to_alias($resource); $id = array(_ID_RESOURCE=>$resourceAlias,_ID_CONTEXT=>$contextAlias); @@ -149,11 +148,9 @@ public function findImpactedComposites(array $resourcesAndPredicates, $contextAl } else { foreach ($tablePredicates as $tableType => $predicates) { // Only look for table rows if the changed predicates are actually defined in the tablespec - if(array_intersect($resourcePredicates, $predicates)) - { - if(!isset($tableFilters[$tableType])) - { - $tableFilters[$tableType] = array(); + if (array_intersect($resourcePredicates, $predicates)) { + if (!isset($tableFilters[$tableType])) { + $tableFilters[$tableType] = []; } // build $filter for queries to impact index $tableFilters[$tableType][] = $id; @@ -163,15 +160,11 @@ public function findImpactedComposites(array $resourcesAndPredicates, $contextAl } - if(empty($tableFilters) && !empty($resourceFilters)) - { + if (empty($tableFilters) && !empty($resourceFilters)) { $query = array("value."._IMPACT_INDEX=>array('$in'=>$resourceFilters)); - } - else - { - $query = array(); - foreach($tableFilters as $tableType=>$filters) - { + } else { + $query = []; + foreach ($tableFilters as $tableType => $filters) { // first re-gen table rows where resources appear in the impact index $query[] = array("value."._IMPACT_INDEX=>array('$in'=>$filters), '_id.'._ID_TYPE=>$tableType); } @@ -193,10 +186,10 @@ public function findImpactedComposites(array $resourcesAndPredicates, $contextAl if(empty($query)) { - return array(); + return []; } - $affectedTableRows = array(); + $affectedTableRows = []; foreach($this->config->getCollectionsForTables($this->storeName) as $collection) { @@ -242,7 +235,7 @@ public function getOperationType() * @param int $limit * @return array */ - public function getTableRows($tableSpecId,$filter=array(),$sortBy=array(),$offset=0,$limit=10) + public function getTableRows($tableSpecId,$filter=[],$sortBy=[],$offset=0,$limit=10) { $t = new \Tripod\Timer(); $t->start(); @@ -251,7 +244,7 @@ public function getTableRows($tableSpecId,$filter=array(),$sortBy=array(),$offse $collection = $this->config->getCollectionForTable($this->storeName, $tableSpecId, $this->readPreference); - $findOptions = array(); + $findOptions = []; if (!empty($limit)) { $findOptions['skip'] = (int) $offset; $findOptions['limit'] = (int) $limit; @@ -261,7 +254,7 @@ public function getTableRows($tableSpecId,$filter=array(),$sortBy=array(),$offse } $results = $collection->find($filter, $findOptions); - $rows = array(); + $rows = []; foreach ($results as $doc) { if (array_key_exists(_IMPACT_INDEX,$doc['value'])) unset($doc['value'][_IMPACT_INDEX]); // remove impact index from client @@ -289,7 +282,7 @@ public function getTableRows($tableSpecId,$filter=array(),$sortBy=array(),$offse * @param array $filter * @return array */ - public function distinct($tableSpecId, $fieldName, array $filter=array()) + public function distinct($tableSpecId, $fieldName, array $filter=[]) { $t = new \Tripod\Timer(); $t->start(); @@ -324,7 +317,7 @@ protected function deleteTableRowsForResource($resource, $context=null, $specTyp $resourceAlias = $this->labeller->uri_to_alias($resource); $contextAlias = $this->getContextAlias($context); $query = array(_ID_KEY . '.' . _ID_RESOURCE => $resourceAlias, _ID_KEY . '.' . _ID_CONTEXT => $contextAlias); - $specNames = array(); + $specNames = []; $specTypes = $this->config->getTableSpecifications($this->storeName); if (empty($specType)) { $specNames = array_keys($specTypes); @@ -395,14 +388,14 @@ public function deleteTableRowsByTableId($tableId, $timestamp = null) { * @param string|null $context * @param array $specTypes */ - protected function generateTableRowsForResource($resource, $context=null, $specTypes=array()) + protected function generateTableRowsForResource($resource, $context=null, $specTypes=[]) { $resourceAlias = $this->labeller->uri_to_alias($resource); $contextAlias = $this->getContextAlias($context); $this->deleteTableRowsForResource($resource, $context, $specTypes); - $filter = array(); + $filter = []; $filter[] = array("r"=>$resourceAlias,"c"=>$contextAlias); // now go through the types @@ -444,7 +437,7 @@ protected function generateTableRowsForResource($resource, $context=null, $specT * @param array $specTypes * @return mixed */ - public function generateTableRowsForType($rdfType,$subject=null,$context=null, $specTypes = array()) + public function generateTableRowsForType($rdfType,$subject=null,$context=null, $specTypes = []) { $rdfType = $this->labeller->qname_to_alias($rdfType); $rdfTypeAlias = $this->labeller->uri_to_alias($rdfType); @@ -456,7 +449,7 @@ public function generateTableRowsForType($rdfType,$subject=null,$context=null, $ } else { - $tableSpecs = array(); + $tableSpecs = []; foreach($specTypes as $specType) { $spec = $this->getConfigInstance()->getTableSpecification($this->storeName, $specType); @@ -502,11 +495,11 @@ public function generateTableRows($tableType, $resource = null, $context = null, { $t = new \Tripod\Timer(); $t->start(); - $this->temporaryFields = array(); + $this->temporaryFields = []; $tableSpec = $this->getConfigInstance()->getTableSpecification($this->storeName, $tableType); - $collection = $this->config->getCollectionForTable($this->storeName, $tableType); + $collection = $this->getConfigInstance()->getCollectionForTable($this->storeName, $tableType); - if ($tableSpec==null) { + if (empty($tableSpec)) { $this->debugLog("Could not find a table specification for $tableType"); return null; } @@ -515,32 +508,37 @@ public function generateTableRows($tableType, $resource = null, $context = null, $contextAlias = $this->getContextAlias($context); // default collection - $from = (isset($tableSpec["from"])) ? $tableSpec["from"] : $this->podName; + $from = isset($tableSpec["from"]) ? $tableSpec["from"] : $this->podName; - $types = array(); + $types = []; if (is_array($tableSpec["type"])) { foreach ($tableSpec["type"] as $type) { - $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($type)); - $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($type)); + $types[] = ["rdf:type.u" => $this->labeller->qname_to_alias($type)]; + $types[] = ["rdf:type.u" => $this->labeller->uri_to_alias($type)]; } } else { - $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($tableSpec["type"])); - $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($tableSpec["type"])); + $types[] = ["rdf:type.u" => $this->labeller->qname_to_alias($tableSpec["type"])]; + $types[] = ["rdf:type.u" => $this->labeller->uri_to_alias($tableSpec["type"])]; } - $filter = array('$or'=> $types); + $filter = ['$or' => $types]; if (isset($resource)) { - $filter["_id"] = array(_ID_RESOURCE=>$this->labeller->uri_to_alias($resource),_ID_CONTEXT=>$contextAlias); + $filter["_id"] = [ + _ID_RESOURCE => $this->labeller->uri_to_alias($resource), + _ID_CONTEXT => $contextAlias + ]; } + // @todo Change this to a command when we upgrade MongoDB to 1.1+ - $count = $this->config->getCollectionForCBD($this->storeName, $from)->count($filter); - $docs = $this->config->getCollectionForCBD($this->storeName, $from)->find($filter, array( - 'maxTimeMS' => 1000000 - )); + $count = $this->getConfigInstance()->getCollectionForCBD($this->storeName, $from)->count($filter); + $docs = $this->getConfigInstance() + ->getCollectionForCBD($this->storeName, $from) + ->find($filter, ['maxTimeMS' => 1000000]); $jobOptions = []; + $subjects = []; if ($queueName && !$resource && ($this->stat || !empty($this->statsConfig))) { $jobOptions['statsConfig'] = $this->getStatsConfig(); - $jobGroup = new JobGroup($this->storeName); + $jobGroup = $this->getJobGroup($this->storeName); $jobOptions[ApplyOperation::TRACKING_KEY] = $jobGroup->getId()->__toString(); $jobGroup->setJobCount($count); } @@ -554,8 +552,11 @@ public function generateTableRows($tableType, $resource = null, $context = null, $from, array($tableType) ); - - $this->getApplyOperation()->createJob(array($subject), $queueName, $jobOptions); + $subjects[] = $subject; + if (count($subjects) >= $this->getConfigInstance()->getBatchSize(OP_TABLES)) { + $this->queueApplyJob($subjects, $queueName, $jobOptions); + $subjects = []; + } } else { // set up ID $generatedRow = [ @@ -569,12 +570,12 @@ public function generateTableRows($tableType, $resource = null, $context = null, // everything must go in the value object todo: this is a hang over from map reduce days, engineer out once we have stability on new PHP method for M/R $value = ['_id' => $doc['_id']]; $this->addIdToImpactIndex($doc['_id'], $value); // need to add the doc to the impact index to be consistent with views/search etc. this is needed for discovering impacted operations - $this->addFields($doc,$tableSpec,$value); + $this->addFields($doc, $tableSpec, $value); if (isset($tableSpec['joins'])) { - $this->doJoins($doc,$tableSpec['joins'],$value,$from,$contextAlias); + $this->doJoins($doc, $tableSpec['joins'], $value, $from, $contextAlias); } if (isset($tableSpec['counts'])) { - $this->doCounts($doc,$tableSpec['counts'],$value); + $this->doCounts($doc, $tableSpec['counts'], $value); } if (isset($tableSpec['computed_fields'])) { @@ -588,13 +589,17 @@ public function generateTableRows($tableType, $resource = null, $context = null, } } + if (!empty($subjects)) { + $this->queueApplyJob($subjects, $queueName, $jobOptions); + } + $t->stop(); $this->timingLog(MONGO_CREATE_TABLE, array( 'type'=>$tableSpec['type'], 'duration'=>$t->result(), 'filter'=>$filter, 'from'=>$from)); - $this->getStat()->timer(MONGO_CREATE_TABLE.".$tableType",$t->result()); + $this->getStat()->timer(MONGO_CREATE_TABLE.".$tableType", $t->result()); $stat = ['count' => $count]; if (isset($jobOptions[ApplyOperation::TRACKING_KEY])) { @@ -641,7 +646,7 @@ protected function truncatingSave(Collection $collection, array $generatedRow) protected function truncateFields(Collection $collection, array &$generatedRow) { // Find the name of any indexed fields - $indexedFields = array(); + $indexedFields = []; $indexesGroupedByCollection = $this->config->getIndexesGroupedByCollection($this->storeName); if (isset($indexesGroupedByCollection) && isset($indexesGroupedByCollection[$collection->getCollectionName()])) { @@ -911,7 +916,7 @@ protected function rewriteVariableValue($value, array &$dest, $setType=null) $function = array_keys($value); return $this->getComputedValue($function[0], $value, $dest); } - $aryValue = array(); + $aryValue = []; foreach($value as $v) { $aryValue[] = $this->rewriteVariableValue($v, $dest); @@ -1129,7 +1134,7 @@ protected function addFields($source,$spec,&$dest) */ protected function generateValues($source, $f, $predicate, &$dest) { - $values = array(); + $values = []; if (isset($source[$predicate][VALUE_URI]) && !empty($source[$predicate][VALUE_URI])) { $values[] = $source[$predicate][VALUE_URI]; @@ -1175,7 +1180,7 @@ protected function generateValues($source, $f, $predicate, &$dest) { // convert from single value to array of values $existingVal = $dest[$f['fieldName']]; - $dest[$f['fieldName']] = array(); + $dest[$f['fieldName']] = []; $dest[$f['fieldName']][] = $existingVal; $dest[$f['fieldName']][] = $v; } @@ -1190,7 +1195,7 @@ protected function generateValues($source, $f, $predicate, &$dest) */ protected function getPredicateFunctions($array) { - $predicateFunctions = array(); + $predicateFunctions = []; if(is_array($array)) { if(isset($array['predicates'])) @@ -1217,7 +1222,7 @@ protected function getPredicateFunctions($array) * @throws \Exception * @return mixed */ - private function applyModifier($modifier, $value, $options = array()) + private function applyModifier($modifier, $value, $options = []) { try { @@ -1271,7 +1276,7 @@ protected function doJoins($source,$joins,&$dest,$from,$contextAlias) // to join on it. However, we need to think about different combinations of // nested joins in different points of the view spec and see if this would // complicate things. Needs a unit test or two. - $joinUris = array(); + $joinUris = []; if (isset($source[$predicate][VALUE_URI])) { // single value for join @@ -1291,7 +1296,7 @@ protected function doJoins($source,$joins,&$dest,$from,$contextAlias) } } - $recursiveJoins = array(); + $recursiveJoins = []; $collection = (isset($ruleset['from']) ? $this->config->getCollectionForCBD($this->storeName, $ruleset['from']) : $this->config->getCollectionForCBD($this->storeName, $from) diff --git a/src/mongo/delegates/Views.class.php b/src/mongo/delegates/Views.class.php index c1ba8f1b..4a462b69 100644 --- a/src/mongo/delegates/Views.class.php +++ b/src/mongo/delegates/Views.class.php @@ -7,7 +7,6 @@ use \Tripod\Mongo\Labeller; use \MongoDB\Driver\ReadPreference; use \MongoDB\Collection; -use Tripod\Mongo\JobGroup; /** * Class Views @@ -153,7 +152,7 @@ public function getViews(Array $filter, $viewType) { if (strpos($predicate,'$')===0) { - $values = array(); + $values = []; foreach ($object as $obj) { foreach ($obj as $p=>$o) $values[] = array('value.'._GRAPHS.'.'.$p=>$o); @@ -240,7 +239,7 @@ public function getViewForResources(Array $resources,$viewType,$context=null) $missingSubjects = array_diff($resources,$returnedSubjects); if (!empty($missingSubjects)) { - $regrabResources = array(); + $regrabResources = []; foreach($missingSubjects as $missingSubject) { $viewSpec = $this->getConfigInstance()->getViewSpecification($this->storeName, $viewType); @@ -285,7 +284,7 @@ public function getViewForResources(Array $resources,$viewType,$context=null) private function createTripodViewIdsFromResourceUris($resourceUriOrArray,$context,$viewType) { $contextAlias = $this->getContextAlias($context); - $ret = array(); + $ret = []; foreach($resourceUriOrArray as $resource) { $ret[] = array("r"=>$this->labeller->uri_to_alias($resource),"c"=>$contextAlias,"type"=>$viewType); @@ -303,7 +302,7 @@ public function generateViews($resources,$context=null) $contextAlias = $this->getContextAlias($context); // build a filter - will be used for impactIndex detection and finding direct views to re-gen - $filter = array(); + $filter = []; foreach ($resources as $resource) { $resourceAlias = $this->labeller->uri_to_alias($resource); @@ -431,13 +430,13 @@ public function generateView($viewId, $resource = null, $context = null, $queueN $t->start(); $from = $this->getFromCollectionForViewSpec($viewSpec); - $collection = $this->config->getCollectionForView($this->storeName, $viewId); + $collection = $this->getConfigInstance()->getCollectionForView($this->storeName, $viewId); if (!isset($viewSpec['joins'])) { throw new \Tripod\Exceptions\ViewException('Could not find any joins in view specification - usecase better served with select()'); } - $types = array(); // this is used to filter the CBD table to speed up the view creation + $types = []; // this is used to filter the CBD table to speed up the view creation if (is_array($viewSpec["type"])) { foreach ($viewSpec["type"] as $type) { $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($type)); @@ -454,17 +453,18 @@ public function generateView($viewId, $resource = null, $context = null, $queueN } // @todo Change this to a command when we upgrade MongoDB to 1.1+ - $count = $this->config->getCollectionForCBD($this->storeName, $from)->count($filter); - $docs = $this->config->getCollectionForCBD($this->storeName, $from)->find($filter, array( + $count = $this->getConfigInstance()->getCollectionForCBD($this->storeName, $from)->count($filter); + $docs = $this->getConfigInstance()->getCollectionForCBD($this->storeName, $from)->find($filter, array( 'maxTimeMS' => $this->getConfigInstance()->getMongoCursorTimeout() )); $jobOptions = []; + $subjects = []; if ($queueName && !$resource) { $jobOptions['statsConfig'] = $this->getStatsConfig(); - $jobGroup = new JobGroup($this->storeName); + $jobGroup = $this->getJobGroup($this->storeName); $jobOptions[ApplyOperation::TRACKING_KEY] = $jobGroup->getId()->__toString(); $jobGroup->setJobCount($count); } @@ -477,8 +477,11 @@ public function generateView($viewId, $resource = null, $context = null, $queueN $from, array($viewId) ); - - $this->getApplyOperation()->createJob(array($subject), $queueName, $jobOptions); + $subjects[] = $subject; + if (count($subjects) >= $this->getConfigInstance()->getBatchSize(OP_VIEWS)) { + $this->queueApplyJob($subjects, $queueName, $jobOptions); + $subjects = []; + } } else { // Set up view meta information $generatedView = [ @@ -489,9 +492,9 @@ public function generateView($viewId, $resource = null, $context = null, $queueN ], \_CREATED_TS => \Tripod\Mongo\DateUtil::getMongoDate() ]; - $value = array(); // everything must go in the value object todo: this is a hang over from map reduce days, engineer out once we have stability on new PHP method for M/R + $value = []; // everything must go in the value object todo: this is a hang over from map reduce days, engineer out once we have stability on new PHP method for M/R - $value[_GRAPHS] = array(); + $value[_GRAPHS] = []; $buildImpactIndex=true; if (isset($viewSpec['ttl'])) { @@ -516,6 +519,10 @@ public function generateView($viewId, $resource = null, $context = null, $queueN } } + if (!empty($subjects)) { + $this->queueApplyJob($subjects, $queueName, $jobOptions); + } + $t->stop(); $this->timingLog(MONGO_CREATE_VIEW, array( 'view'=>$viewSpec['type'], @@ -557,7 +564,7 @@ protected function doJoins($source, $joins, &$dest, $from, $contextAlias, $build // to join on it. However, we need to think about different combinations of // nested joins in different points of the view spec and see if this would // complicate things. Needs a unit test or two. - $joinUris = array(); + $joinUris = []; if (isset($source[$predicate][VALUE_URI])) { // single value for join @@ -578,7 +585,7 @@ protected function doJoins($source, $joins, &$dest, $from, $contextAlias, $build } } - $recursiveJoins = array(); + $recursiveJoins = []; $collection = ( isset($ruleset['from']) ? $this->config->getCollectionForCBD($this->storeName, $ruleset['from']) @@ -682,7 +689,7 @@ protected function matchesFilter($linkMatchType, $linkMatchValue, $filterType, $ */ protected function extractProperties($source,$viewSpec,$from) { - $obj = array(); + $obj = []; if (isset($viewSpec['include'])) { $obj['_id'] = $source['_id']; @@ -742,7 +749,7 @@ protected function extractProperties($source,$viewSpec,$from) } if(isset($source[$p]) && isset($source[$p][$i])) { - if (!isset($obj[$p])) $obj[$p] = array(); + if (!isset($obj[$p])) $obj[$p] = []; $obj[$p][] = $source[$p][$i]; } } @@ -770,7 +777,7 @@ protected function extractProperties($source,$viewSpec,$from) } if($val && isset($val[$i])) { - if (!$obj[$p]) $obj[$p] = array(); + if (!$obj[$p]) $obj[$p] = []; $obj[$p][] = $val[$i]; } } diff --git a/test/unit/mongo/MongoTripodSearchIndexerTest.php b/test/unit/mongo/MongoTripodSearchIndexerTest.php index 7b9c86ef..322f5f8a 100644 --- a/test/unit/mongo/MongoTripodSearchIndexerTest.php +++ b/test/unit/mongo/MongoTripodSearchIndexerTest.php @@ -13,8 +13,7 @@ protected function setUp() parent::setUp(); $this->tripod = new \Tripod\Mongo\Driver("CBD_testing", "tripod_php_testing", array("async"=>array(OP_VIEWS=>true, OP_TABLES=>true, OP_SEARCH=>false))); - foreach(\Tripod\Config::getInstance()->getCollectionsForSearch($this->tripod->getStoreName()) as $collection) - { + foreach (\Tripod\Config::getInstance()->getCollectionsForSearch($this->tripod->getStoreName()) as $collection) { $collection->drop(); } $this->loadResourceDataViaTripod(); @@ -499,4 +498,85 @@ public function testSavingMultipleNewEntitiesResultsInOneImpactedSubject() $this->assertEquals($expectedImpactedSubjects, $impactedSubjects); } -} \ No newline at end of file + public function testBatchSearchDocumentsGeneration() + { + $count = 234; + $docs = []; + + $configOptions = json_decode(file_get_contents(__DIR__ . '/data/config.json'), true); + + for ($i = 0; $i < $count; $i++) { + $docs[] = ['_id' => ['r' => 'tenantLists:batch' . $i, 'c' => 'tenantContexts:DefaultGraph']]; + } + + $fakeCursor = new ArrayIterator($docs); + /** @var \PHPUnit_Framework_MockObject_MockObject|TripodTestConfig $configInstance */ + $configInstance = $this->getMockBuilder('TripodTestConfig') + ->setMethods(['getCollectionForCBD']) + ->disableOriginalConstructor() + ->getMock(); + $configInstance->loadConfig($configOptions); + + /** @var \PHPUnit_Framework_MockObject_MockObject|\MongoDB\Collection $collection */ + $collection = $this->getMockBuilder('\MongoDB\Collection') + ->setMethods(['count', 'find']) + ->disableOriginalConstructor() + ->getMock(); + $collection->expects($this->atLeastOnce())->method('count')->willReturn($count); + $collection->expects($this->atLeastOnce())->method('find')->willReturn($fakeCursor); + + /** @var \PHPUnit_Framework_MockObject_MockObject|\Tripod\Mongo\JobGroup $jobGroup */ + $jobGroup = $this->getMockBuilder('\Tripod\Mongo\JobGroup') + ->setMethods(['setJobCount']) + ->setConstructorArgs(['tripod_php_testing']) + ->getMock(); + $jobGroup->expects($this->once())->method('setJobCount')->with($count); + + $configInstance->expects($this->atLeastOnce())->method('getCollectionForCBD')->willReturn($collection); + /** @var \PHPUnit_Framework_MockObject_MockObject|\Tripod\Mongo\Driver $tripod */ + $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver') + ->setMethods(['getConfigInstance',]) + ->setConstructorArgs(['tripod_php_testing', 'CBD_testing']) + ->disableOriginalConstructor() + ->getMock(); + + /** @var \PHPUnit_Framework_MockObject_MockObject|\Tripod\Mongo\Composites\SearchIndexer $search */ + $search = $this->getMockBuilder('\Tripod\Mongo\Composites\SearchIndexer') + ->setMethods(['setSearchProvider', 'getConfigInstance', 'queueApplyJob', 'getJobGroup']) + ->setConstructorArgs([$tripod]) + ->getMock(); + $search->expects($this->atLeastOnce())->method('getConfigInstance')->willReturn($configInstance); + $search->expects($this->once())->method('getJobGroup')->willReturn($jobGroup); + $search->expects($this->exactly(3))->method('queueApplyJob') + ->withConsecutive( + [ + $this->logicalAnd( + $this->isType('array'), + $this->containsOnlyInstancesOf('\Tripod\Mongo\ImpactedSubject'), + $this->countOf(100) + ), + 'TESTQUEUE', + $this->isType('array') + ], + [ + $this->logicalAnd( + $this->isType('array'), + $this->containsOnlyInstancesOf('\Tripod\Mongo\ImpactedSubject'), + $this->countOf(100) + ), + 'TESTQUEUE', + $this->isType('array') + ], + [ + $this->logicalAnd( + $this->isType('array'), + $this->containsOnlyInstancesOf('\Tripod\Mongo\ImpactedSubject'), + $this->countOf(34) + ), + 'TESTQUEUE', + $this->isType('array') + ] + ); + $search->generateSearchDocuments('i_search_list', null, null, 'TESTQUEUE'); + } +} diff --git a/test/unit/mongo/MongoTripodTablesTest.php b/test/unit/mongo/MongoTripodTablesTest.php index dbef6ec7..c184309a 100644 --- a/test/unit/mongo/MongoTripodTablesTest.php +++ b/test/unit/mongo/MongoTripodTablesTest.php @@ -1,6 +1,7 @@ tripodTransactionLog = new \Tripod\Mongo\TransactionLog(); $this->tripodTransactionLog->purgeAllTransactions(); - $this->tripod = new \Tripod\Mongo\Driver($this->defaultPodName, $this->defaultStoreName, array("async"=>array(OP_VIEWS=>false, OP_TABLES=>false, OP_SEARCH=>false))); + $this->tripod = new \Tripod\Mongo\Driver( + $this->defaultPodName, + $this->defaultStoreName, + ["async" => [OP_VIEWS=>false, OP_TABLES => false, OP_SEARCH => false]] + ); $this->getTripodCollection($this->tripod)->drop(); $this->tripod->setTransactionLog($this->tripodTransactionLog); $this->loadResourceDataViaTripod(); - $this->tablesConstParams = array($this->tripod->getStoreName(),$this->getTripodCollection($this->tripod),'http://talisaspire.com/'); + $this->tablesConstParams = [ + $this->tripod->getStoreName(), + $this->getTripodCollection($this->tripod), + 'http://talisaspire.com/' + ]; - $this->tripodTables = new \Tripod\Mongo\Composites\Tables($this->tripod->getStoreName(),$this->getTripodCollection($this->tripod),null); // pass null context, should default to http://talisaspire.com + $this->tripodTables = new \Tripod\Mongo\Composites\Tables( + $this->tripod->getStoreName(), + $this->getTripodCollection($this->tripod), + null // pass null context, should default to http://talisaspire.com + ); // purge tables foreach (\Tripod\Config::getInstance()->getCollectionsForTables($this->tripod->getStoreName()) as $collection) { @@ -303,6 +316,74 @@ public function testGenerateTableRows() $this->assertTrue(isset($result['isbn13']),"Result does not contain isbn13"); } + public function testBatchTableRowGeneration() + { + $count = 234; + $docs = []; + + $configOptions = json_decode(file_get_contents(__DIR__ . '/data/config.json'), true); + + for ($i = 0; $i < $count; $i++) { + $docs[] = ['_id' => ['r' => 'tenantLists:batch' . $i, 'c' => 'tenantContexts:DefaultGraph']]; + } + + $fakeCursor = new ArrayIterator($docs); + /** @var \PHPUnit_Framework_MockObject_MockObject|TripodTestConfig $configInstance */ + $configInstance = $this->getMockBuilder('TripodTestConfig') + ->setMethods(['getCollectionForTable', 'getCollectionForCBD']) + ->disableOriginalConstructor() + ->getMock(); + $configInstance->loadConfig($configOptions); + + /** @var \PHPUnit_Framework_MockObject_MockObject|\MongoDB\Collection $collection */ + $collection = $this->getMockBuilder('\MongoDB\Collection') + ->setMethods(['count', 'find']) + ->disableOriginalConstructor() + ->getMock(); + $collection->expects($this->atLeastOnce())->method('count')->willReturn($count); + $collection->expects($this->atLeastOnce())->method('find')->willReturn($fakeCursor); + + $configInstance->expects($this->atLeastOnce())->method('getCollectionForCBD')->willReturn($collection); + + /** @var \PHPUnit_Framework_MockObject_MockObject|\Tripod\Mongo\Composites\Tables $tables */ + $tables = $this->getMockBuilder('\Tripod\Mongo\Composites\Tables') + ->setMethods(['getConfigInstance', 'queueApplyJob']) + ->setConstructorArgs(['tripod_php_testing', $collection, 'tenantContexts:DefaultGraph']) + ->getMock(); + $tables->expects($this->atLeastOnce())->method('getConfigInstance')->willReturn($configInstance); + $tables->expects($this->exactly(3))->method('queueApplyJob') + ->withConsecutive( + [ + $this->logicalAnd( + $this->isType('array'), + $this->containsOnlyInstancesOf('\Tripod\Mongo\ImpactedSubject'), + $this->countOf(100) + ), + 'TESTQUEUE', + $this->isType('array') + ], + [ + $this->logicalAnd( + $this->isType('array'), + $this->containsOnlyInstancesOf('\Tripod\Mongo\ImpactedSubject'), + $this->countOf(100) + ), + 'TESTQUEUE', + $this->isType('array') + ], + [ + $this->logicalAnd( + $this->isType('array'), + $this->containsOnlyInstancesOf('\Tripod\Mongo\ImpactedSubject'), + $this->countOf(34) + ), + 'TESTQUEUE', + $this->isType('array') + ] + ); + $tables->generateTableRows('t_resource', null, null, 'TESTQUEUE'); + } + public function testGetTableRowsSort() { $this->tripodTables->generateTableRows("t_resource"); diff --git a/test/unit/mongo/MongoTripodViewsTest.php b/test/unit/mongo/MongoTripodViewsTest.php index 2bf3c513..3b987edf 100644 --- a/test/unit/mongo/MongoTripodViewsTest.php +++ b/test/unit/mongo/MongoTripodViewsTest.php @@ -2276,4 +2276,135 @@ public function testDeleteViewsByViewIdWithTimestamp() $this->assertEquals(30, $views->deleteViewsByViewId('v_resource_full', $timestamp)); } + + public function testBatchViewGeneration() + { + $count = 234; + $docs = []; + + $configOptions = json_decode(file_get_contents(__DIR__ . '/data/config.json'), true); + + for ($i = 0; $i < $count; $i++) { + $docs[] = ['_id' => ['r' => 'tenantLists:batch' . $i, 'c' => 'tenantContexts:DefaultGraph']]; + } + + $fakeCursor = new ArrayIterator($docs); + /** @var \PHPUnit_Framework_MockObject_MockObject|TripodTestConfig $configInstance */ + $configInstance = $this->getMockBuilder('TripodTestConfig') + ->setMethods(['getCollectionForView', 'getCollectionForCBD']) + ->disableOriginalConstructor() + ->getMock(); + $configInstance->loadConfig($configOptions); + + /** @var \PHPUnit_Framework_MockObject_MockObject|\MongoDB\Collection $collection */ + $collection = $this->getMockBuilder('\MongoDB\Collection') + ->setMethods(['count', 'find']) + ->disableOriginalConstructor() + ->getMock(); + $collection->expects($this->atLeastOnce())->method('count')->willReturn($count); + $collection->expects($this->atLeastOnce())->method('find')->willReturn($fakeCursor); + + $configInstance->expects($this->atLeastOnce())->method('getCollectionForCBD')->willReturn($collection); + + /** @var \PHPUnit_Framework_MockObject_MockObject|\Tripod\Mongo\Composites\Views $views */ + $views = $this->getMockBuilder('\Tripod\Mongo\Composites\Views') + ->setMethods(['getConfigInstance', 'queueApplyJob']) + ->setConstructorArgs(['tripod_php_testing', $collection, 'tenantContexts:DefaultGraph']) + ->getMock(); + $views->expects($this->atLeastOnce())->method('getConfigInstance')->willReturn($configInstance); + $views->expects($this->exactly(10))->method('queueApplyJob') + ->withConsecutive( + [ + $this->logicalAnd( + $this->isType('array'), + $this->containsOnlyInstancesOf('\Tripod\Mongo\ImpactedSubject'), + $this->countOf(25) + ), + 'TESTQUEUE', + $this->isType('array') + ], + [ + $this->logicalAnd( + $this->isType('array'), + $this->containsOnlyInstancesOf('\Tripod\Mongo\ImpactedSubject'), + $this->countOf(25) + ), + 'TESTQUEUE', + $this->isType('array') + ], + [ + $this->logicalAnd( + $this->isType('array'), + $this->containsOnlyInstancesOf('\Tripod\Mongo\ImpactedSubject'), + $this->countOf(25) + ), + 'TESTQUEUE', + $this->isType('array') + ], + [ + $this->logicalAnd( + $this->isType('array'), + $this->containsOnlyInstancesOf('\Tripod\Mongo\ImpactedSubject'), + $this->countOf(25) + ), + 'TESTQUEUE', + $this->isType('array') + ], + [ + $this->logicalAnd( + $this->isType('array'), + $this->containsOnlyInstancesOf('\Tripod\Mongo\ImpactedSubject'), + $this->countOf(25) + ), + 'TESTQUEUE', + $this->isType('array') + ], + [ + $this->logicalAnd( + $this->isType('array'), + $this->containsOnlyInstancesOf('\Tripod\Mongo\ImpactedSubject'), + $this->countOf(25) + ), + 'TESTQUEUE', + $this->isType('array') + ], + [ + $this->logicalAnd( + $this->isType('array'), + $this->containsOnlyInstancesOf('\Tripod\Mongo\ImpactedSubject'), + $this->countOf(25) + ), + 'TESTQUEUE', + $this->isType('array') + ], + [ + $this->logicalAnd( + $this->isType('array'), + $this->containsOnlyInstancesOf('\Tripod\Mongo\ImpactedSubject'), + $this->countOf(25) + ), + 'TESTQUEUE', + $this->isType('array') + ], + [ + $this->logicalAnd( + $this->isType('array'), + $this->containsOnlyInstancesOf('\Tripod\Mongo\ImpactedSubject'), + $this->countOf(25) + ), + 'TESTQUEUE', + $this->isType('array') + ], + [ + $this->logicalAnd( + $this->isType('array'), + $this->containsOnlyInstancesOf('\Tripod\Mongo\ImpactedSubject'), + $this->countOf(9) + ), + 'TESTQUEUE', + $this->isType('array') + ] + ); + $views->generateView('v_resource_full', null, null, 'TESTQUEUE'); + } }