From 4fa81cc0b0456c4999a6ae3ecc24eb8110faba9c Mon Sep 17 00:00:00 2001 From: Grzegorz Rajchman Date: Fri, 9 Aug 2024 15:41:35 +0100 Subject: [PATCH] PLT-842 Update Mongo driver connection options (#148) BREAKING CHANGES - Remove Tripod\Mongo\Config::getConnStr, getTransactionLogConnStr - These were public methods but weren't part of the interface. Instead, getDatabase or getCollectionFor should be used. Features - Allow MongoDB connection string to include options - Add missing methods to IConfigInstance - Don't force default connectTimeoutMS if already set in connection string - Correct config instance type in docblocks - Correct Labeller type in MongoGraph - Test against PHP 7.4, MongoDB replica sets - Add test to cover JobBase::getTripodConfig --- .circleci/config.yml | 158 +++++++++++++-- composer.json | 9 +- docker-compose.clusters.yml | 16 ++ docker-compose.yml | 31 +-- docker/Dockerfile-php55 | 14 +- docker/Dockerfile-php73 | 15 +- docker/Dockerfile-php74 | 18 ++ src/Config.php | 2 +- src/ITripodConfig.php | 2 +- src/ITripodConfigSerializer.php | 2 +- src/TripodConfigFactory.php | 4 +- src/mongo/Config.class.php | 131 +++--------- src/mongo/IConfigInstance.php | 24 +++ src/mongo/MongoGraph.class.php | 5 + src/mongo/base/JobBase.class.php | 8 +- src/mongo/util/TriplesUtil.class.php | 7 +- test/unit/mongo/ApplyOperationTest.php | 22 +-- .../mongo/DiscoverImpactedSubjectsTest.php | 22 +-- test/unit/mongo/EnsureIndexesTest.php | 4 +- test/unit/mongo/JobBaseTest.php | 28 +++ test/unit/mongo/MongoSearchProviderTest.php | 8 +- test/unit/mongo/MongoTripodConfigUnitTest.php | 187 ++++-------------- .../mongo/MongoTripodSearchIndexerTest.php | 2 +- test/unit/mongo/MongoTripodTablesTest.php | 10 +- test/unit/mongo/MongoTripodViewsTest.php | 79 +++----- test/unit/mongo/TestJobBase.php | 27 +++ 26 files changed, 431 insertions(+), 404 deletions(-) create mode 100644 docker-compose.clusters.yml create mode 100644 docker/Dockerfile-php74 create mode 100644 test/unit/mongo/JobBaseTest.php create mode 100644 test/unit/mongo/TestJobBase.php diff --git a/.circleci/config.yml b/.circleci/config.yml index cac5ed72..3f8cdda0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,33 +1,155 @@ version: 2.1 -jobs: - test: +commands: + setup_replica_set: parameters: - php_version: + set_name: + type: string + member_hosts: type: string - docker: - - image: talis/tripod-php:<< parameters.php_version >>-latest - environment: - RESQUE_SERVER: redis - - image: mongo:3.2.21 - name: mongodb - - image: redis:6.2.6 - name: redis steps: - - checkout - - run: composer install + - run: + name: Setup a replica set + no_output_timeout: "2m" + environment: + RS_NAME: << parameters.set_name >> + MEMBERS: << parameters.member_hosts >> + command: | + IFS=',' read -r -a MEMBERS_ARR \<<<"$MEMBERS" + INITIATOR=${MEMBERS_ARR[0]} + MEMBERS_JS='' + for i in "${!MEMBERS_ARR[@]}"; do + MEMBERS_JS="${MEMBERS_JS}$(printf '{ _id: %d, host: "%s" },' $i "${MEMBERS_ARR[$i]}")" + done + + MONGO_CLI="$(command -v mongosh mongo | head -n1)" + _mongo() { "$MONGO_CLI" --quiet --host "$@"; } + + for member in "${MEMBERS_ARR[@]}"; do + echo "Waiting for $member" + until _mongo "$member" \<<<'db.adminCommand("ping")' | grep 'ok'; do + sleep 1 + done + done + + echo "Initiating replica set $RS_NAME..." + _mongo "$INITIATOR" \<<<"$(printf 'rs.initiate({ _id: "%s", members: [%s] })' "$RS_NAME" "$MEMBERS_JS")" | tee /dev/stderr | grep -E 'ok|already initialized' + echo "Waiting for primary..." + _mongo "$INITIATOR" \<<<'while (true) { if (rs.status().members.some(({ state }) => state === 1)) { break; } sleep(1000); }' + echo "Waiting for secondaries..." + _mongo "$INITIATOR" \<<<'while (true) { if (rs.status().members.every(({state}) => state == 1 || state == 2)) { break; } sleep(1000); }' + echo "Checking status..." + _mongo "$INITIATOR" \<<<'rs.status();' | tee /dev/stderr | grep "$RS_NAME" + echo "Replica set configured!" + + check_mongodb_lib_version: + steps: + - run: + name: Check mongodb ext+lib parity + command: | + php -r 'echo phpversion("mongodb"), PHP_EOL;' + grep '"name": "mongodb/mongodb",' composer.lock -A1 + + run_test: + steps: - run: composer test -- --log-junit test-results/junit.xml - store_test_results: path: test-results/junit.xml - store_artifacts: path: test-results/junit.xml +jobs: + test: + parameters: + php_version: { type: string } + mongo_version: { type: string } + docker: + - image: talis/tripod-php:<< parameters.php_version >>-latest + - { name: mongodb, image: mongo:<< parameters.mongo_version >> } + - { name: redis, image: redis:6.2.6 } + environment: + RESQUE_SERVER: redis + steps: + - checkout + - run: composer install + - check_mongodb_lib_version + - run_test + + test-multiple-stores: + docker: + - image: talis/tripod-php:php74-latest + - { name: mongo1, image: mongo:4.4.29 } + - { name: mongo2, image: mongo:5.0.28 } + - { name: redis, image: redis:6.2.6 } + environment: + RESQUE_SERVER: redis + TRIPOD_DATASOURCE_RS1_CONFIG: | + {"type":"mongo", "connection":"mongodb://mongo1:27017/", "replicaSet":""} + TRIPOD_DATASOURCE_RS2_CONFIG: | + {"type":"mongo", "connection":"mongodb://mongo2:27017/", "replicaSet":""} + steps: + - checkout + - run: composer install + - check_mongodb_lib_version + - run_test + + test-replica-set-mmap: + docker: + - image: talis/tripod-php:php74-latest + - { name: mongo1, image: mongo:3.6.23, command: mongod --storageEngine mmapv1 --smallfiles --replSet=tripod-rs } + - { name: mongo2, image: mongo:3.6.23, command: mongod --storageEngine mmapv1 --smallfiles --replSet=tripod-rs } + - { name: redis, image: redis:6.2.6 } + environment: + RESQUE_SERVER: redis + TRIPOD_DATASOURCE_RS1_CONFIG: | + {"type":"mongo", "connection":"mongodb://mongo1,mongo2/?retryWrites=false", "replicaSet":"tripod-rs"} + TRIPOD_DATASOURCE_RS2_CONFIG: | + {"type":"mongo", "connection":"mongodb://mongo1,mongo2/?retryWrites=false", "replicaSet":"tripod-rs"} + steps: + - checkout + - setup_replica_set: + set_name: tripod-rs + member_hosts: mongo1,mongo2 + - run: composer install + - check_mongodb_lib_version + - run_test + + test-replica-set-wiredtiger: + docker: + - image: talis/tripod-php:php74-latest + - { name: mongo1, image: mongo:5.0.28, command: mongod --replSet=tripod-rs } + - { name: mongo2, image: mongo:5.0.28, command: mongod --replSet=tripod-rs } + - { name: redis, image: redis:6.2.6 } + environment: + RESQUE_SERVER: redis + TRIPOD_DATASOURCE_RS1_CONFIG: | + {"type":"mongo", "connection":"mongodb://mongo1,mongo2/admin?replicaSet=tripod-rs", "replicaSet":""} + TRIPOD_DATASOURCE_RS2_CONFIG: | + {"type":"mongo", "connection":"mongodb://mongo1,mongo2/admin?replicaSet=tripod-rs", "replicaSet":""} + steps: + - checkout + - setup_replica_set: + set_name: tripod-rs + member_hosts: mongo1,mongo2 + - run: composer install + - check_mongodb_lib_version + - run_test + workflows: build_and_test: jobs: - test: - matrix: - parameters: - php_version: - - php55 - - php73 + name: test-php55 + php_version: php55 + mongo_version: 3.2.21 + - test: + name: test-php73 + php_version: php73 + mongo_version: 3.6.23 + - test: + name: test-php74 + php_version: php74 + mongo_version: 4.0.28 + - test-multiple-stores + - test-replica-set-mmap + - test-replica-set-wiredtiger diff --git a/composer.json b/composer.json index 203c5a7b..8645c583 100644 --- a/composer.json +++ b/composer.json @@ -18,18 +18,17 @@ } ], "suggest": { - "mongodb/mongodb": "MongoDB driver library", "resque/php-resque": "Redis backed library for background jobs" }, "require": { "php" : ">=5.5", - "semsol/arc2": "2.2.6", - "monolog/monolog" : "~1.13" + "mongodb/mongodb": "*", + "monolog/monolog" : "~1.13", + "semsol/arc2": "2.2.6" }, "require-dev": { - "resque/php-resque": "v1.3.4", - "mongodb/mongodb": "1.4.3", "phpunit/phpunit": "^4.8", + "resque/php-resque": "v1.3.4", "squizlabs/php_codesniffer": "3.2.*" }, "autoload": { diff --git a/docker-compose.clusters.yml b/docker-compose.clusters.yml new file mode 100644 index 00000000..5a8297c6 --- /dev/null +++ b/docker-compose.clusters.yml @@ -0,0 +1,16 @@ +# An override file to test Tripod on multiple clusters. +# Usage: +# docker compose -f docker-compose.yml -f docker-compose.clusters.yml run --rm -it php74 vendor/bin/phpunit + +services: + php74: + depends_on: + - mongodb2 + environment: + TRIPOD_DATASOURCE_RS1_CONFIG: | + {"type":"mongo", "connection":"mongodb://mongodb:27017/", "replicaSet":""} + TRIPOD_DATASOURCE_RS2_CONFIG: | + {"type":"mongo", "connection":"mongodb://mongodb2:27017/", "replicaSet":""} + + mongodb2: + image: mongo:4.4.29 diff --git a/docker-compose.yml b/docker-compose.yml index f459c91b..c29cae68 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,35 +1,36 @@ -version: "3.7" - x-base-config: &base-config volumes: - ./:/var/tripod-php - - ./vendor:/var/tripod-php/vendor:delegated - links: - - "mongo32:mongodb" - - redis depends_on: - - mongo32 + - mongodb - redis working_dir: /var/tripod-php env_file: .env services: php55: - # build: - # context: ./docker - # dockerfile: Dockerfile-php55 + build: + context: ./docker + dockerfile: Dockerfile-php55 image: talis/tripod-php:php55-latest <<: *base-config php73: - # build: - # context: ./docker - # dockerfile: Dockerfile-php73 + build: + context: ./docker + dockerfile: Dockerfile-php73 image: talis/tripod-php:php73-latest <<: *base-config - mongo32: - image: mongo:3.2.21 + php74: + build: + context: ./docker + dockerfile: Dockerfile-php74 + image: talis/tripod-php:php74-latest + <<: *base-config + + mongodb: + image: mongo:3.6.23 redis: image: redis:6.2.6 diff --git a/docker/Dockerfile-php55 b/docker/Dockerfile-php55 index 698a8643..421fd005 100644 --- a/docker/Dockerfile-php55 +++ b/docker/Dockerfile-php55 @@ -15,8 +15,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ zip \ && rm -rf /var/lib/apt/lists/* -COPY --from=mlocati/php-extension-installer:2.1.58 /usr/bin/install-php-extensions /usr/local/bin/ -RUN install-php-extensions pcntl redis mongodb-1.5.5 && \ - curl https://getcomposer.org/installer >/tmp/composer-setup.php && \ - php /tmp/composer-setup.php --install-dir /usr/local/bin/ --filename composer && \ - rm /tmp/composer-setup.php +RUN curl -sLo - https://www.mongodb.org/static/pgp/server-3.6.asc | apt-key add - \ + && echo "deb http://repo.mongodb.org/apt/debian jessie/mongodb-org/3.6 main" | tee /etc/apt/sources.list.d/mongodb-org-3.6.list \ + && apt-get update && apt-get install -y --no-install-recommends mongodb-org-shell --force-yes \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=mlocati/php-extension-installer:2.3.2 /usr/bin/install-php-extensions /usr/local/bin/ +COPY --from=composer:2.2.20 /usr/bin/composer /usr/bin/composer + +RUN install-php-extensions pcntl mongodb-1.5.5 diff --git a/docker/Dockerfile-php73 b/docker/Dockerfile-php73 index 078b83c9..56b1bced 100644 --- a/docker/Dockerfile-php73 +++ b/docker/Dockerfile-php73 @@ -1,4 +1,4 @@ -FROM php:7.3-cli +FROM php:7.3.33-cli RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ @@ -8,8 +8,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ zip \ && rm -rf /var/lib/apt/lists/* -COPY --from=mlocati/php-extension-installer:2.1.58 /usr/bin/install-php-extensions /usr/local/bin/ -RUN install-php-extensions pcntl redis mongodb-1.5.5 && \ - curl https://getcomposer.org/installer >/tmp/composer-setup.php && \ - php /tmp/composer-setup.php --install-dir /usr/local/bin/ --filename composer && \ - rm /tmp/composer-setup.php +RUN curl -sLo /tmp/mongosh.deb https://downloads.mongodb.com/compass/mongodb-mongosh_2.2.15_amd64.deb \ + && dpkg -i /tmp/mongosh.deb \ + && rm /tmp/mongosh.deb + +COPY --from=mlocati/php-extension-installer:2.3.2 /usr/bin/install-php-extensions /usr/local/bin/ +COPY --from=composer:2.7.7 /usr/bin/composer /usr/local/bin/ + +RUN install-php-extensions pcntl mongodb-1.6.1 diff --git a/docker/Dockerfile-php74 b/docker/Dockerfile-php74 new file mode 100644 index 00000000..45947dac --- /dev/null +++ b/docker/Dockerfile-php74 @@ -0,0 +1,18 @@ +FROM php:7.4.33-cli + +RUN apt-get update && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + git \ + unzip \ + zip \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -sLo /tmp/mongosh.deb https://downloads.mongodb.com/compass/mongodb-mongosh_2.2.15_amd64.deb \ + && dpkg -i /tmp/mongosh.deb \ + && rm /tmp/mongosh.deb + +COPY --from=mlocati/php-extension-installer:2.3.2 /usr/bin/install-php-extensions /usr/local/bin/ +COPY --from=composer:2.7.7 /usr/bin/composer /usr/local/bin/ + +RUN install-php-extensions pcntl mongodb-1.19.3 diff --git a/src/Config.php b/src/Config.php index c83a44c1..aef41b33 100644 --- a/src/Config.php +++ b/src/Config.php @@ -26,7 +26,7 @@ private function __construct() * @uses Config::setConfig() Configuration must be set prior to calling this method. To generate a completely new object, set a new config * @codeCoverageIgnore * @throws \Tripod\Exceptions\ConfigException - * @return ITripodConfig + * @return \Tripod\Mongo\IConfigInstance */ public static function getInstance() { diff --git a/src/ITripodConfig.php b/src/ITripodConfig.php index 215749f5..5d6c6f50 100644 --- a/src/ITripodConfig.php +++ b/src/ITripodConfig.php @@ -7,7 +7,7 @@ interface ITripodConfig /** * Tripod Config instances are singletons, this method gets the existing or instantiates a new one. * - * @return ITripodConfig + * @return \Tripod\Mongo\IConfigInstance */ public static function getInstance(); diff --git a/src/ITripodConfigSerializer.php b/src/ITripodConfigSerializer.php index 67be4980..2d13b7d7 100644 --- a/src/ITripodConfigSerializer.php +++ b/src/ITripodConfigSerializer.php @@ -16,7 +16,7 @@ public function serialize(); * When given a valid config, returns a Tripod Config object * * @param array $config - * @return \Tripod\ITripodConfig + * @return \Tripod\Mongo\IConfigInstance */ public static function deserialize(array $config); } diff --git a/src/TripodConfigFactory.php b/src/TripodConfigFactory.php index 79cda977..003d4261 100644 --- a/src/TripodConfigFactory.php +++ b/src/TripodConfigFactory.php @@ -5,11 +5,11 @@ class TripodConfigFactory { /** - * Factory method to get a Tripod\ITripodConfig instance from either a config array, or a serialized + * Factory method to get a Tripod config instance from either a config array, or a serialized * ITripodConfigSerializer instance * * @param array $config The Tripod config or serialized ITripodConfigSerializer array - * @return \Tripod\ITripodConfig + * @return \Tripod\Mongo\IConfigInstance */ public static function create(array $config) { diff --git a/src/mongo/Config.class.php b/src/mongo/Config.class.php index 7108981f..0a6f5708 100644 --- a/src/mongo/Config.class.php +++ b/src/mongo/Config.class.php @@ -6,13 +6,8 @@ use \MongoDB\Database; use \MongoDB\Collection; use \MongoDB\Driver\ReadPreference; -use \MongoDB\Driver\Command; -use \MongoDB\Driver\Manager; use \MongoDB\Driver\Exception\ConnectionTimeoutException; -use \Tripod\ITripodConfig; -use \Tripod\ITripodConfigSerializer; - /** * Holds the global configuration for Tripod */ @@ -1237,68 +1232,6 @@ public function getDataSourceForPod($storeName, $podName) throw new \Tripod\Exceptions\ConfigException("'{$podName}' not configured for store '{$storeName}'"); } - /** - * Returns the connection string for the supplied database name - * - * @param string $storeName - * @param string|null $podName - * @throws \Tripod\Exceptions\ConfigException - * @return string - */ - public function getConnStr($storeName, $podName = null) - { - if (array_key_exists($storeName,$this->dbConfig)) - { - if(!$podName) - { - return $this->getConnStrForDataSource($this->dbConfig[$storeName]['data_source']); - } - $pods = $this->getPods($storeName); - if(array_key_exists($podName, $pods)) - { - return $this->getConnStrForDataSource($pods[$podName]['data_source']); - } - throw new \Tripod\Exceptions\ConfigException("Collection $podName does not exist for database $storeName"); - } - else - { - throw new \Tripod\Exceptions\ConfigException("Database $storeName does not exist in configuration"); - } - } - - /** - * Returns the transaction log database connection string - * @return string - * @throws \Tripod\Exceptions\ConfigException - */ - public function getTransactionLogConnStr() { - return $this->getConnStrForDataSource($this->tConfig['data_source']); - } - - /** - * @param $dataSource - * @return string - * @throws \Tripod\Exceptions\ConfigException - */ - protected function getConnStrForDataSource($dataSource) - { - if(!array_key_exists($dataSource, $this->dataSources)) - { - throw new \Tripod\Exceptions\ConfigException("Data source '{$dataSource}' not configured"); - } - $ds = $this->dataSources[$dataSource]; - if(array_key_exists("replicaSet", $ds) && !empty($ds["replicaSet"])) { - $connStr = $ds['connection']; - if ($this->isConnectionStringValidForRepSet($connStr)){ - return $connStr; - } else { - throw new \Tripod\Exceptions\ConfigException("Connection string for '{$dataSource}' must include /admin database when connecting to Replica Set"); - } - } else { - return $ds['connection']; - } - } - /** * Returns a replica set name for the database, if one has been defined * @param $datasource @@ -1306,11 +1239,19 @@ protected function getConnStrForDataSource($dataSource) */ public function getReplicaSetName($datasource) { - if($this->isReplicaSet($datasource)) - { + if (!empty($this->dataSources[$datasource]['replicaSet'])) { return $this->dataSources[$datasource]['replicaSet']; } + if (strpos($this->dataSources[$datasource]['connection'], 'replicaSet=') !== false) { + $query = parse_url($this->dataSources[$datasource]['connection'], PHP_URL_QUERY); + $params = []; + parse_str($query, $params); + if (!empty($params['replicaSet'])) { + return $params['replicaSet']; + } + } + return null; } @@ -1321,14 +1262,7 @@ public function getReplicaSetName($datasource) */ public function isReplicaSet($datasource) { - if (array_key_exists($datasource,$this->dataSources)) - { - if(array_key_exists("replicaSet",$this->dataSources[$datasource]) && !empty($this->dataSources[$datasource]["replicaSet"])) { - return true; - } - } - - return false; + return $this->getReplicaSetName($datasource) !== null; } /** @@ -1624,20 +1558,6 @@ private function getMandatoryKey($key,Array $a,$configName='config') return $a[$key]; } - /** - * @param string $connStr - * @return bool - */ - private function isConnectionStringValidForRepSet($connStr) - { - $needle = "/admin"; - if (substr($connStr, -6) === $needle){ - return true; - } else { - return false; - } - } - /** * Finds fields in a table specification * @param string $fieldName @@ -1730,22 +1650,26 @@ protected function getConnectionForDataSource($dataSource) { throw new \Tripod\Exceptions\ConfigException("Data source '{$dataSource}' not in configuration"); } - $connectionOptions = []; - $ds = $this->dataSources[$dataSource]; - $connectionOptions['connectTimeoutMS'] = (isset($ds['connectTimeoutMS']) ? $ds['connectTimeoutMS'] : DEFAULT_MONGO_CONNECT_TIMEOUT_MS); - - if(isset($ds['replicaSet']) && !empty($ds['replicaSet'])) { - $connectionOptions['replicaSet'] = $ds['replicaSet']; - } if(!isset($this->connections[$dataSource])) { + $ds = $this->dataSources[$dataSource]; + $connectionString = $ds['connection']; + $connectionOptions = []; + + if (!empty($ds['connectTimeoutMS']) || strpos($connectionString, 'connectTimeoutMS=') === false) { + $connectionOptions['connectTimeoutMS'] = isset($ds['connectTimeoutMS']) ? $ds['connectTimeoutMS'] : DEFAULT_MONGO_CONNECT_TIMEOUT_MS; + } + + if (!empty($ds['replicaSet'])) { + $connectionOptions['replicaSet'] = $ds['replicaSet']; + } + $retries = 1; $exception = null; do { try { - $connectionString = $ds['connection'] . '?' . http_build_query($connectionOptions); - $this->connections[$dataSource] = $this->getMongoClient($connectionString); + $this->connections[$dataSource] = $this->getMongoClient($connectionString, $connectionOptions); break; } catch (ConnectionTimeoutException $e) { self::getLogger()->error("ConnectionTimeoutException attempt ".$retries.". Retrying...:" . $e->getMessage()); @@ -1769,10 +1693,11 @@ protected function getConnectionForDataSource($dataSource) * @param string $connectionString * @return Client */ - protected function getMongoClient($connectionString) + protected function getMongoClient($connectionString, array $connectionOptions = []) { - $client = new Client($connectionString, - [], + $client = new Client( + $connectionString, + $connectionOptions, ['typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array']] ); return $client; diff --git a/src/mongo/IConfigInstance.php b/src/mongo/IConfigInstance.php index 9e5f1137..f007b88b 100644 --- a/src/mongo/IConfigInstance.php +++ b/src/mongo/IConfigInstance.php @@ -68,6 +68,17 @@ public function isPodWithinStore($storeName, $pod); */ public function getPods($storeName); + /** + * Returns the name of the data source for the request pod. This may be the default for the store or the pod may + * have overridden it in the config. + * + * @param string $storeName + * @param string $podName + * @throws \Tripod\Exceptions\ConfigException + * @return string + */ + public function getDataSourceForPod($storeName, $podName); + /** * Return the view specification document for the supplied id, if it exists * @param string $storeName Store name @@ -156,6 +167,12 @@ public function getDbs(); */ public function getNamespaces(); + /** + * Getter for transaction log connection config + * @return array + */ + public function getTransactionLogConfig(); + /** * @param string $storeName Store name * @return string|null @@ -257,6 +274,13 @@ public function getCollectionsForSearch( */ public function getCollectionForTTLCache($storeName, $readPreference = ReadPreference::RP_PRIMARY_PREFERRED); + /** + * @param string $storeName + * @param string $readPreference + * @return Collection + */ + public function getCollectionForLocks($storeName, $readPreference = ReadPreference::RP_PRIMARY_PREFERRED); + /** * @param string $storeName Store (database) name * @param string $readPreference Mongo read preference diff --git a/src/mongo/MongoGraph.class.php b/src/mongo/MongoGraph.class.php index 23b27827..aa6fa1f7 100644 --- a/src/mongo/MongoGraph.class.php +++ b/src/mongo/MongoGraph.class.php @@ -9,6 +9,11 @@ * @package Tripod\Mongo */ class MongoGraph extends \Tripod\ExtendedGraph { + /** + * @var Labeller + */ + var $_labeller; + /** * Constructor */ diff --git a/src/mongo/base/JobBase.class.php b/src/mongo/base/JobBase.class.php index 0a8888fa..fb7796da 100644 --- a/src/mongo/base/JobBase.class.php +++ b/src/mongo/base/JobBase.class.php @@ -41,7 +41,7 @@ abstract class JobBase extends \Tripod\Mongo\DriverBase protected $mandatoryArgs = []; protected $configRequired = false; - /** @var \Tripod\ITripodConfig */ + /** @var \Tripod\Mongo\IConfigInstance */ protected $tripodConfig; /** @var \Tripod\Timer */ @@ -307,7 +307,7 @@ protected function serializeConfig($configSerializer) * Deserialize a tripodConfigGenerator argument to a Tripod Config object * * @param array $config The serialized Tripod config - * @return \Tripod\ITripodConfig + * @return \Tripod\Mongo\IConfigInstance */ protected function deserializeConfig(array $config) { @@ -333,13 +333,13 @@ protected function setTripodConfig() /** * Returns the Tripod config required by the job * - * @return \Tripod\ITripodConfig + * @return \Tripod\Mongo\IConfigInstance */ protected function getTripodConfig() { if (!isset($this->tripodConfig)) { $this->ensureConfig(); - $this->setConfig(); + $this->setTripodConfig(); } return $this->tripodConfig; } diff --git a/src/mongo/util/TriplesUtil.class.php b/src/mongo/util/TriplesUtil.class.php index 76f6a66b..556c20a5 100644 --- a/src/mongo/util/TriplesUtil.class.php +++ b/src/mongo/util/TriplesUtil.class.php @@ -97,12 +97,7 @@ public function loadTriplesAbout($subject,Array $triples,$storeName,$podName,$co if (array_key_exists($podName, $this->collections)) { $collection = $this->collections[$podName]; } else { - $m = new Client( - \Tripod\Config::getInstance()->getConnStr($storeName), - [], - ['typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array']] - ); - $collection = $m->selectDatabase($storeName)->selectCollection($podName); + $collection = \Tripod\Config::getInstance()->getCollectionForCBD($storeName, $podName); } $graph = new MongoGraph(); diff --git a/test/unit/mongo/ApplyOperationTest.php b/test/unit/mongo/ApplyOperationTest.php index b8796683..c26eccf1 100644 --- a/test/unit/mongo/ApplyOperationTest.php +++ b/test/unit/mongo/ApplyOperationTest.php @@ -17,7 +17,7 @@ public function testMandatoryArgTripodConfig() unset($this->args['tripodConfig']); $job = new \Tripod\Mongo\Jobs\ApplyOperation(); $job->args = $this->args; - $job->job->payload['id'] = uniqid(); + $job->job = new \Resque_Job('queue', ['id' => uniqid()]); $this->setExpectedException( 'Exception', 'Argument tripodConfig or tripodConfigGenerator was not present in supplied job args for job Tripod\Mongo\Jobs\ApplyOperation' @@ -31,7 +31,7 @@ public function testMandatoryArgSubject() unset($this->args['subjects']); $job = new \Tripod\Mongo\Jobs\ApplyOperation(); $job->args = $this->args; - $job->job->payload['id'] = uniqid(); + $job->job = new \Resque_Job('queue', ['id' => uniqid()]); $this->setExpectedException('Exception', "Argument subjects was not present in supplied job args for job Tripod\Mongo\Jobs\ApplyOperation"); $this->performJob($job); } @@ -44,7 +44,7 @@ public function testApplyViewOperation() ->getMock(); $applyOperation->args = $this->args; - $applyOperation->job->payload['id'] = uniqid(); + $applyOperation->job = new \Resque_Job('queue', ['id' => uniqid()]); $statMock = $this->getMockStat( $this->args['statsConfig']['config']['host'], @@ -122,7 +122,7 @@ public function testApplyViewOperationDecrementsJobGroupForBatchOperations() ->getMock(); $applyOperation->args = $this->args; - $applyOperation->job->payload['id'] = uniqid(); + $applyOperation->job = new \Resque_Job('queue', ['id' => uniqid()]); $jobTrackerId = new \MongoDB\BSON\ObjectId(); $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString(); @@ -223,7 +223,7 @@ public function testApplyViewOperationCleanupIfAllGroupJobsComplete() ->getMock(); $applyOperation->args = $this->args; - $applyOperation->job->payload['id'] = uniqid(); + $applyOperation->job = new \Resque_Job('queue', ['id' => uniqid()]); $jobTrackerId = new \MongoDB\BSON\ObjectId(); $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString(); $timestamp = new \MongoDB\BSON\UTCDateTime(hexdec(substr($jobTrackerId, 0, 8)) * 1000); @@ -350,7 +350,7 @@ public function testApplyTableOperation() $this->args['subjects'] = array($impactedSubject->toArray()); $applyOperation->args = $this->args; - $applyOperation->job->payload['id'] = uniqid(); + $applyOperation->job = new \Resque_Job('queue', ['id' => uniqid()]); $subject = $this->getMockBuilder('\Tripod\Mongo\ImpactedSubject') ->setMethods(array('getTripod')) @@ -429,7 +429,7 @@ public function testApplyTableOperationDecrementsJobGroupForBatchOperations() ); $applyOperation->args = $this->args; - $applyOperation->job->payload['id'] = uniqid(); + $applyOperation->job = new \Resque_Job('queue', ['id' => uniqid()]); $jobTrackerId = new \MongoDB\BSON\ObjectId(); $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString(); @@ -533,7 +533,7 @@ public function testApplyTableOperationCleanupIfAllGroupJobsComplete() ); $applyOperation->args = $this->args; - $applyOperation->job->payload['id'] = uniqid(); + $applyOperation->job = new \Resque_Job('queue', ['id' => uniqid()]); $jobTrackerId = new \MongoDB\BSON\ObjectId(); $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString(); @@ -642,7 +642,7 @@ public function testApplySearchOperation() ); $applyOperation->args = $this->args; - $applyOperation->job->payload['id'] = uniqid(); + $applyOperation->job = new \Resque_Job('queue', ['id' => uniqid()]); $subject = $this->getMockBuilder('\Tripod\Mongo\ImpactedSubject') ->setMethods(array('getTripod')) @@ -718,7 +718,7 @@ public function testApplySearchOperationDecrementsJobGroupForBatchOperations() ); $applyOperation->args = $this->args; - $applyOperation->job->payload['id'] = uniqid(); + $applyOperation->job = new \Resque_Job('queue', ['id' => uniqid()]); $jobTrackerId = new \MongoDB\BSON\ObjectId(); $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString(); @@ -815,7 +815,7 @@ public function testApplySearchOperationCleanupIfAllGroupJobsComplete() ); $applyOperation->args = $this->args; - $applyOperation->job->payload['id'] = uniqid(); + $applyOperation->job = new \Resque_Job('queue', ['id' => uniqid()]); $jobTrackerId = new \MongoDB\BSON\ObjectId(); $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString(); diff --git a/test/unit/mongo/DiscoverImpactedSubjectsTest.php b/test/unit/mongo/DiscoverImpactedSubjectsTest.php index ddeafe2b..751184c9 100644 --- a/test/unit/mongo/DiscoverImpactedSubjectsTest.php +++ b/test/unit/mongo/DiscoverImpactedSubjectsTest.php @@ -16,7 +16,7 @@ public function testMandatoryArgTripodConfig() unset($this->args['tripodConfig']); $job = new DiscoverImpactedSubjects(); $job->args = $this->args; - $job->job->payload['id'] = uniqid(); + $job->job = new \Resque_Job('queue', ['id' => uniqid()]); $this->setExpectedException( 'Exception', 'Argument tripodConfig or tripodConfigGenerator was not present in supplied job args for job Tripod\Mongo\Jobs\DiscoverImpactedSubjects' @@ -30,7 +30,7 @@ public function testMandatoryArgStoreName() unset($this->args['storeName']); $job = new DiscoverImpactedSubjects(); $job->args = $this->args; - $job->job->payload['id'] = uniqid(); + $job->job = new \Resque_Job('queue', ['id' => uniqid()]); $this->setExpectedException('Exception', "Argument storeName was not present in supplied job args for job Tripod\Mongo\Jobs\DiscoverImpactedSubjects"); $this->performJob($job); } @@ -41,7 +41,7 @@ public function testMandatoryArgPodName() unset($this->args['podName']); $job = new DiscoverImpactedSubjects(); $job->args = $this->args; - $job->job->payload['id'] = uniqid(); + $job->job = new \Resque_Job('queue', ['id' => uniqid()]); $this->setExpectedException('Exception', "Argument podName was not present in supplied job args for job Tripod\Mongo\Jobs\DiscoverImpactedSubjects"); $this->performJob($job); } @@ -52,7 +52,7 @@ public function testMandatoryArgChanges() unset($this->args['changes']); $job = new DiscoverImpactedSubjects(); $job->args = $this->args; - $job->job->payload['id'] = uniqid(); + $job->job = new \Resque_Job('queue', ['id' => uniqid()]); $this->setExpectedException('Exception', "Argument changes was not present in supplied job args for job Tripod\Mongo\Jobs\DiscoverImpactedSubjects"); $this->performJob($job); } @@ -63,7 +63,7 @@ public function testMandatoryArgOperations() unset($this->args['operations']); $job = new DiscoverImpactedSubjects(); $job->args = $this->args; - $job->job->payload['id'] = uniqid(); + $job->job = new \Resque_Job('queue', ['id' => uniqid()]); $this->setExpectedException('Exception', "Argument operations was not present in supplied job args for job Tripod\Mongo\Jobs\DiscoverImpactedSubjects"); $this->performJob($job); } @@ -74,7 +74,7 @@ public function testMandatoryArgContextAlias() unset($this->args['contextAlias']); $job = new DiscoverImpactedSubjects(); $job->args = $this->args; - $job->job->payload['id'] = uniqid(); + $job->job = new \Resque_Job('queue', ['id' => uniqid()]); $this->setExpectedException('Exception', "Argument contextAlias was not present in supplied job args for job Tripod\Mongo\Jobs\DiscoverImpactedSubjects"); $this->performJob($job); } @@ -142,12 +142,12 @@ public function testSubmitApplyOperationsJob() ->will($this->returnValue($tripod)); $discoverImpactedSubjects->args = $this->args; - $discoverImpactedSubjects->job->payload['id'] = uniqid(); + $discoverImpactedSubjects->job = new \Resque_Job('queue', ['id' => uniqid()]); $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation') ->setMethods(array('createJob')) ->getMock(); - $applyOperation->job->payload['id'] = uniqid(); + $applyOperation->job = new \Resque_Job('queue', ['id' => uniqid()]); $viewSubject = new \Tripod\Mongo\ImpactedSubject( array( @@ -353,7 +353,7 @@ public function testManualQueueNamePersistsThroughJob() $args = $this->args; $args['queue'] = 'TRIPOD_TESTING_QUEUE_' . uniqid(); $discoverImpactedSubjects->args = $args; - $discoverImpactedSubjects->job->payload['id'] = uniqid(); + $discoverImpactedSubjects->job = new \Resque_Job('queue', ['id' => uniqid()]); $tripod->expects($this->exactly(3)) ->method('getComposite') @@ -526,7 +526,7 @@ public function testDiscoverOperationWillSubmitApplyOperationForDistinctQueues() $args = $this->args; $args['operations'] = array(OP_TABLES); $discoverImpactedSubjects->args = $args; - $discoverImpactedSubjects->job->payload['id'] = uniqid(); + $discoverImpactedSubjects->job = new \Resque_Job('queue', ['id' => uniqid()]); $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver') ->setMethods(array('getComposite')) @@ -761,7 +761,7 @@ public function testManuallySpecifiedQueueWillOverrideQueuesDefinedInConfig() $args['operations'] = array(OP_TABLES); $args['queue'] = 'TRIPOD_TESTING_QUEUE_' . uniqid(); $discoverImpactedSubjects->args = $args; - $discoverImpactedSubjects->job->payload['id'] = uniqid(); + $discoverImpactedSubjects->job = new \Resque_Job('queue', ['id' => uniqid()]); $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver') ->setMethods(array('getComposite')) diff --git a/test/unit/mongo/EnsureIndexesTest.php b/test/unit/mongo/EnsureIndexesTest.php index a8e36d98..359a494b 100644 --- a/test/unit/mongo/EnsureIndexesTest.php +++ b/test/unit/mongo/EnsureIndexesTest.php @@ -37,7 +37,7 @@ public function testMandatoryArgs($argument, $argumentName = null) } $job = new \Tripod\Mongo\Jobs\EnsureIndexes(); $job->args = $this->args; - $job->job->payload['id'] = uniqid(); + $job->job = new \Resque_Job('queue', ['id' => uniqid()]); unset($job->args[$argument]); $this->setExpectedException( @@ -224,7 +224,7 @@ protected function createMockJob($methods=array()) ->setMethods($methodsToStub) ->setMockClassName('MockEnsureIndexes') ->getMock(); - $mockEnsureIndexesJob->job->payload['id'] = uniqid(); + $mockEnsureIndexesJob->job = new \Resque_Job('queue', ['id' => uniqid()]); return $mockEnsureIndexesJob; } diff --git a/test/unit/mongo/JobBaseTest.php b/test/unit/mongo/JobBaseTest.php new file mode 100644 index 00000000..f73d70a8 --- /dev/null +++ b/test/unit/mongo/JobBaseTest.php @@ -0,0 +1,28 @@ +args = $this->getArgs(); + $job->job = new \Resque_Job('queue', ['id' => uniqid()]); + + $this->assertInstanceOf(\Tripod\Mongo\IConfigInstance::class, $job->getTripodConfig()); + } + + protected function getArgs() + { + return [ + 'tripodConfig' => \Tripod\Config::getConfig(), + 'storeName' => 'tripod_php_testing', + 'podName' => 'CBD_testing', + 'changes' => ['http://example.com/resources/foo' => ['rdf:type','dct:title']], + 'operations' => [OP_VIEWS, OP_TABLES, OP_SEARCH], + 'contextAlias' => 'http://talisaspire.com/' + ]; + } +} diff --git a/test/unit/mongo/MongoSearchProviderTest.php b/test/unit/mongo/MongoSearchProviderTest.php index e0ecb142..62d19178 100644 --- a/test/unit/mongo/MongoSearchProviderTest.php +++ b/test/unit/mongo/MongoSearchProviderTest.php @@ -607,7 +607,7 @@ public function testCountSearchDocuments() ->getMock(); $collection = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->setMethods(['count']) ->getMock(); $search = $this->getMockBuilder('\Tripod\Mongo\MongoSearchProvider') @@ -637,7 +637,7 @@ public function testCountSearchDocumentsWithFilters() $filters = ['_cts' => ['$lte' => new \MongoDB\BSON\UTCDateTime(null)]]; $query = array_merge(['_id.type' => 'i_search_list'], $filters); $collection = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->setMethods(['count']) ->getMock(); $search = $this->getMockBuilder('\Tripod\Mongo\MongoSearchProvider') @@ -665,7 +665,7 @@ public function testDeleteSearchDocumentsBySearchId() ->getMock(); $collection = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->setMethods(['deleteMany']) ->getMock(); @@ -718,7 +718,7 @@ public function testDeleteSearchDocumentsBySearchIdWithTimestamp() ->getMock(); $collection = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->setMethods(['deleteMany']) ->getMock(); diff --git a/test/unit/mongo/MongoTripodConfigUnitTest.php b/test/unit/mongo/MongoTripodConfigUnitTest.php index 7dd5612e..7988337e 100644 --- a/test/unit/mongo/MongoTripodConfigUnitTest.php +++ b/test/unit/mongo/MongoTripodConfigUnitTest.php @@ -64,79 +64,6 @@ public function testTConfig() $tConfig = $config->getTransactionLogConfig(); $this->assertEquals('tripod_php_testing',$tConfig['database']); $this->assertEquals('transaction_log',$tConfig['collection']); - $this->assertEquals($cfg['data_sources'][$cfg['transaction_log']['data_source']]['connection'],$config->getTransactionLogConnStr()); - } - - public function testTConfigRepSetConnStr() - { - $config=array(); - $config["data_sources"] = array( - "tlog"=>array( - "type"=>"mongo", - "connection"=>"mongodb://tloghost:27017,tloghost:27018/admin", - "replicaSet" => "tlogrepset" - ), - "mongo"=>array("type"=>"mongo","connection" => "mongodb://mongodb") - ); - $config["defaultContext"] = "http://talisaspire.com/"; - $config["stores"] = array( - "tripod_php_testing" => array( - "type"=>"mongo", - "data_source"=>"mongo", - "pods" => array( - "CBD_testing" => array() - ), - ) - ); - $config["transaction_log"] = array( - "database"=>"transactions", - "collection"=>"transaction_log", - "data_source"=>"tlog" - - ); - - \Tripod\Config::setConfig($config); - $mtc = \Tripod\Config::getInstance(); - $this->assertEquals("mongodb://tloghost:27017,tloghost:27018/admin",$mtc->getTransactionLogConnStr()); - } - - public function testTConfigRepSetConnStrThrowsException() - { - $this->setExpectedException( - '\Tripod\Exceptions\ConfigException', - 'Connection string for \'rs1\' must include /admin database when connecting to Replica Set'); - - $config=array(); - $config["defaultContext"] = "http://talisaspire.com/"; - $config["data_sources"] = array( - "mongo1"=>array( - "type"=>"mongo", - "connection"=>"mongodb://mongodb" - ), - "rs1"=>array( - "type"=>"mongo", - "connection"=>"mongodb://tloghost:27017,tloghost:27018", - "replicaSet" => "tlogrepset" - ) - ); - $config["stores"] = array( - "tripod_php_testing" => array( - "data_source"=>"mongo1", - "pods" => array( - "CBD_testing" => array() - ) - ) - ); - $config["transaction_log"] = array( - "database"=>"transactions", - "collection"=>"transaction_log", - "data_source"=>"rs1" - ); - - \Tripod\Config::setConfig($config); - $mtc = \Tripod\Config::getInstance(); - - $connStr = $mtc->getTransactionLogConnStr(); } public function testCardinality() @@ -148,78 +75,6 @@ public function testCardinality() $this->assertEquals(-1,$cardinality,"Expected cardinality of 1 for random:property"); } - public function testGetConnectionString() - { - $this->assertEquals("mongodb://mongodb:27017/",\Tripod\Config::getInstance()->getConnStr("tripod_php_testing")); - } - - public function testGetConnectionStringThrowsException() - { - $this->setExpectedException( - '\Tripod\Exceptions\ConfigException', - 'Database notexists does not exist in configuration'); - $this->assertEquals("mongodb://mongodb:27017/",\Tripod\Config::getInstance()->getConnStr("notexists")); - } - - public function testGetConnectionStringForReplicaSet(){ - $config=array(); - $config["defaultContext"] = "http://talisaspire.com/"; - $config["data_sources"] = array( - "rs"=>array( - "type"=>"mongo", - "connection"=>"mongodb://mongodb:27017,localhost:27018/admin", - "replicaSet" => "myrepset" - ) - ); - $config["transaction_log"] = array("database"=>"transactions","collection"=>"transaction_log","data_source"=>"rs"); - $config["stores"] = array( - "tripod_php_testing" => array( - "pods" => array( - "CBD_testing" => array() - ), - "data_source"=>"rs" - ) - ); - - \Tripod\Config::setConfig($config); - $mtc = \Tripod\Config::getInstance(); - - $this->assertEquals("mongodb://mongodb:27017,localhost:27018/admin",$mtc->getConnStr("tripod_php_testing")); - } - - public function testGetConnectionStringThrowsExceptionForReplicaSet(){ - $this->setExpectedException( - '\Tripod\Exceptions\ConfigException', - 'Connection string for \'rs1\' must include /admin database when connecting to Replica Set'); - $config=array(); - $config["defaultContext"] = "http://talisaspire.com/"; - $config["data_sources"] = array( - "mongo1"=>array( - "type"=>"mongo", - "connection"=>"mongodb://mongodb" - ), - "rs1"=>array( - "type"=>"mongo", - "connection" => "mongodb://mongodb:27017,localhost:27018", - "replicaSet" => "myrepset" - ) - ); - $config["transaction_log"] = array("database"=>"transactions","collection"=>"transaction_log","data_source"=>"mongo1"); - $config["stores"] = array( - "tripod_php_testing" => array( - "data_source"=>"rs1", - "pods" => array( - "CBD_testing" => array() - ), - ) - ); - - \Tripod\Config::setConfig($config); - $mtc = \Tripod\Config::getInstance(); - - $mtc->getConnStr("tripod_php_testing"); - } - public function testCompoundIndexAllArraysThrowsException() { $this->setExpectedException( @@ -876,6 +731,28 @@ public function testGetReplicaSetName() $this->assertNull($mtc->getReplicaSetName("testing_2")); } + public function testGetReplicaSetNameFromConnectionString() + { + \Tripod\Config::setConfig([ + 'defaultContext' => 'http://talisaspire.com/', + 'data_sources' => [ + 'rs1' => [ + 'type' => 'mongo', + 'connection' => 'mongodb://a.foo.com,b.foo.com/?replicaSet=myReplicaSet&authSource=admin', + ], + 'rs2' => [ + 'type' => 'mongo', + 'connection' => 'mongodb://c.foo.com,d.foo.com/?replicaSet=', + ], + ], + 'transaction_log' => ['database' => 'transactions', 'collection' => 'transaction_log', 'data_source' => 'rs1'], + 'stores' => [], + ]); + $mtc = \Tripod\Config::getInstance(); + $this->assertEquals('myReplicaSet', $mtc->getReplicaSetName('rs1')); + $this->assertEquals(null, $mtc->getReplicaSetName('rs2')); + } + public function testGetViewSpecification(){ $expectedVspec = array( "_id"=> "v_resource_full", @@ -1172,8 +1049,8 @@ public function testDataLoadedInConfiguredDataSource() // Create some locks so we have a collection $lCollection = $config->getCollectionForLocks($storeName); $lCollection->drop(); - $lCollection->insert(array(_ID_KEY=>array(_ID_RESOURCE=>'foo',_ID_CONTEXT=>'bar'), _LOCKED_FOR_TRANS=>'foobar')); - $lCollection->insert(array(_ID_KEY=>array(_ID_RESOURCE=>'baz',_ID_CONTEXT=>'bar'), _LOCKED_FOR_TRANS=>'wibble')); + $lCollection->insertOne(array(_ID_KEY=>array(_ID_RESOURCE=>'foo',_ID_CONTEXT=>'bar'), _LOCKED_FOR_TRANS=>'foobar')); + $lCollection->insertOne(array(_ID_KEY=>array(_ID_RESOURCE=>'baz',_ID_CONTEXT=>'bar'), _LOCKED_FOR_TRANS=>'wibble')); $this->tripod->removeInertLocks('foobar', 'reason1'); $collectionsForDataSource = array(); @@ -1206,10 +1083,11 @@ public function testDataLoadedInConfiguredDataSource() foreach($dataSourcesForStore as $source) { - /** @var MongoCollection $collection */ - foreach($config->getDatabase($storeName, $source)->listCollections() as $collection) + $db = $config->getDatabase($storeName, $source); + foreach($db->listCollections() as $collectionInfo) { - $name = $collection->getName(); + $name = $collectionInfo->getName(); + $collection = $db->selectCollection($name); $foundCollections[] = $name; $this->assertContains($name, $collectionsForDataSource[$source], "Source " . $source . " does not include " . $name); switch($name) @@ -1710,11 +1588,12 @@ public function testGetResqueServer() // MongoClient creation tests public function testMongoConnectionNoExceptions() { + /** @var PHPUnit_Framework_MockObject_MockObject&TripodTestConfig */ $mockConfig = $this->getMock('TripodTestConfig', array('getMongoClient')); $mockConfig->loadConfig(json_decode(file_get_contents(dirname(__FILE__).'/data/config.json'), true)); $mockConfig->expects($this->exactly(1)) ->method('getMongoClient') - ->with('mongodb://mongodb:27017/?connectTimeoutMS=20000') + ->with('mongodb://mongodb:27017/', ['connectTimeoutMS' => 20000]) ->will($this->returnCallback( function() { @@ -1729,22 +1608,24 @@ function() public function testMongoConnectionExceptionThrown() { $this->setExpectedException('\MongoDB\Driver\Exception\ConnectionTimeoutException', "Exception thrown when connecting to Mongo"); + /** @var PHPUnit_Framework_MockObject_MockObject&TripodTestConfig */ $mockConfig = $this->getMock('TripodTestConfig', array('getMongoClient')); $mockConfig->loadConfig(json_decode(file_get_contents(dirname(__FILE__).'/data/config.json'), true)); $mockConfig->expects($this->exactly(30)) ->method('getMongoClient') - ->with('mongodb://mongodb:27017/?connectTimeoutMS=20000') + ->with('mongodb://mongodb:27017/', ['connectTimeoutMS' => 20000]) ->will($this->throwException(new ConnectionTimeoutException('Exception thrown when connecting to Mongo'))); $mockConfig->getDatabase('tripod_php_testing', 'rs1', ReadPreference::RP_SECONDARY_PREFERRED); } public function testMongoConnectionNoExceptionThrownWhenConnectionThrowsSomeExceptions() { + /** @var PHPUnit_Framework_MockObject_MockObject&TripodTestConfig */ $mockConfig = $this->getMock('TripodTestConfig', array('getMongoClient')); $mockConfig->loadConfig(json_decode(file_get_contents(dirname(__FILE__).'/data/config.json'), true)); $mockConfig->expects($this->exactly(5)) ->method('getMongoClient') - ->with('mongodb://mongodb:27017/?connectTimeoutMS=20000') + ->with('mongodb://mongodb:27017/', ['connectTimeoutMS' => 20000]) ->will($this->onConsecutiveCalls( $this->throwException(new ConnectionTimeoutException('Exception thrown when connecting to Mongo')), $this->throwException(new ConnectionTimeoutException('Exception thrown when connecting to Mongo')), diff --git a/test/unit/mongo/MongoTripodSearchIndexerTest.php b/test/unit/mongo/MongoTripodSearchIndexerTest.php index 322f5f8a..fe3b7ec4 100644 --- a/test/unit/mongo/MongoTripodSearchIndexerTest.php +++ b/test/unit/mongo/MongoTripodSearchIndexerTest.php @@ -520,7 +520,7 @@ public function testBatchSearchDocumentsGeneration() /** @var \PHPUnit_Framework_MockObject_MockObject|\MongoDB\Collection $collection */ $collection = $this->getMockBuilder('\MongoDB\Collection') ->setMethods(['count', 'find']) - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->getMock(); $collection->expects($this->atLeastOnce())->method('count')->willReturn($count); $collection->expects($this->atLeastOnce())->method('find')->willReturn($fakeCursor); diff --git a/test/unit/mongo/MongoTripodTablesTest.php b/test/unit/mongo/MongoTripodTablesTest.php index fb408f44..5965dd99 100644 --- a/test/unit/mongo/MongoTripodTablesTest.php +++ b/test/unit/mongo/MongoTripodTablesTest.php @@ -338,7 +338,7 @@ public function testBatchTableRowGeneration() /** @var \PHPUnit_Framework_MockObject_MockObject|\MongoDB\Collection $collection */ $collection = $this->getMockBuilder('\MongoDB\Collection') ->setMethods(['count', 'find']) - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->getMock(); $collection->expects($this->atLeastOnce())->method('count')->willReturn($count); $collection->expects($this->atLeastOnce())->method('find')->willReturn($fakeCursor); @@ -1716,7 +1716,7 @@ public function testRemoveTableSpecDoesNotAffectInvalidation() public function testCountTables() { $collection = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->setMethods(['count']) ->getMock(); $tables = $this->getMockBuilder('\Tripod\Mongo\Composites\Tables') @@ -1742,7 +1742,7 @@ public function testCountTablesWithFilters() $filters = ['_cts' => ['$lte' => new \MongoDB\BSON\UTCDateTime(null)]]; $query = array_merge(['_id.type' => 't_source_count'], $filters); $collection = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->setMethods(['count']) ->getMock(); $tables = $this->getMockBuilder('\Tripod\Mongo\Composites\Tables') @@ -1766,7 +1766,7 @@ public function testCountTablesWithFilters() public function testDeleteTableRowsByTableId() { $collection = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->setMethods(['deleteMany']) ->getMock(); @@ -1809,7 +1809,7 @@ public function testDeleteTableRowsByTableIdWithTimestamp() ] ]; $collection = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->setMethods(['deleteMany']) ->getMock(); diff --git a/test/unit/mongo/MongoTripodViewsTest.php b/test/unit/mongo/MongoTripodViewsTest.php index 719836be..84878b99 100644 --- a/test/unit/mongo/MongoTripodViewsTest.php +++ b/test/unit/mongo/MongoTripodViewsTest.php @@ -102,13 +102,8 @@ public function testGenerateView() ) ); // get the view direct from mongo -// $result = $this->tripod->getViewForResource("http://talisaspire.com/resources/3SplCtWGPqEyXcDiyhHQpA","v_resource_full"); - $mongo = new Client( - \Tripod\Config::getInstance()->getConnStr('tripod_php_testing'), - [], - ['typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array']] - ); - $actualView = $mongo->selectCollection('tripod_php_testing','views')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/3SplCtWGPqEyXcDiyhHQpA',"c"=>'http://talisaspire.com/',"type"=>'v_resource_full'))); + $collection = \Tripod\Config::getInstance()->getCollectionForView('tripod_php_testing', 'v_resource_full'); + $actualView = $collection->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/3SplCtWGPqEyXcDiyhHQpA',"c"=>'http://talisaspire.com/',"type"=>'v_resource_full'))); $this->assertEquals($expectedView['_id'], $actualView['_id']); $this->assertEquals($expectedView['value'], $actualView['value']); $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']); @@ -176,12 +171,8 @@ public function testGenerateViewWithFilterRemovesFilteredDataButKeepsResourcesIn ) ); // get the view direct from mongo - $mongo = new Client( - \Tripod\Config::getInstance()->getConnStr('tripod_php_testing'), - [], - ['typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array']] - ); - $actualView = $mongo->selectCollection('tripod_php_testing','views')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_filter1'))); + $collection = \Tripod\Config::getInstance()->getCollectionForView('tripod_php_testing', 'v_resource_filter1'); + $actualView = $collection->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_filter1'))); $this->assertEquals($expectedView['_id'], $actualView['_id']); $this->assertEquals($expectedView['value'], $actualView['value']); $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']); @@ -241,12 +232,8 @@ public function testGenerateViewWithFilterOnLiteralValue() ) ); // get the view direct from mongo - $mongo = new Client( - \Tripod\Config::getInstance()->getConnStr('tripod_php_testing'), - [], - ['typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array']] - ); - $actualView = $mongo->selectCollection('tripod_php_testing','views')->findOne( + $collection = \Tripod\Config::getInstance()->getCollectionForView('tripod_php_testing', 'v_resource_filter2'); + $actualView = $collection->findOne( array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_filter2')) ); $this->assertEquals($expectedView['_id'], $actualView['_id']); @@ -316,12 +303,8 @@ public function testGenerateViewCorrectlyAfterUpdateAffectsFilter() ) ); // get the view direct from mongo - $mongo = new Client( - \Tripod\Config::getInstance()->getConnStr('tripod_php_testing'), - [], - ['typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array']] - ); - $actualView = $mongo->selectCollection('tripod_php_testing','views')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_filter1'))); + $collection = \Tripod\Config::getInstance()->getCollectionForView('tripod_php_testing', 'v_resource_filter1'); + $actualView = $collection->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_filter1'))); $this->assertEquals($expectedView['_id'], $actualView['_id']); $this->assertEquals($expectedView['value'], $actualView['value']); $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']); @@ -390,7 +373,7 @@ public function testGenerateViewCorrectlyAfterUpdateAffectsFilter() ) ); - $updatedView = $mongo->selectCollection('tripod_php_testing','views')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_filter1'))); + $updatedView = $collection->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_filter1'))); $this->assertEquals($expectedUpdatedView['_id'], $updatedView['_id']); $this->assertEquals($expectedUpdatedView['value'], $updatedView['value']); $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $updatedView['_cts']); @@ -462,12 +445,8 @@ public function testGenerateViewContainingRdfSequence() ) ); // get the view direct from mongo - $mongo = new Client( - \Tripod\Config::getInstance()->getConnStr('tripod_php_testing'), - [], - ['typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array']] - ); - $actualView = $mongo->selectCollection('tripod_php_testing','views')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_rdfsequence'))); + $collection = \Tripod\Config::getInstance()->getCollectionForView('tripod_php_testing', 'v_resource_rdfsequence'); + $actualView = $collection->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_rdfsequence'))); $this->assertEquals($expectedView['_id'], $actualView['_id']); $this->assertEquals($expectedView['value'], $actualView['value']); @@ -808,11 +787,11 @@ public function testGetViewForResourcesDoesNotInvokeViewGenerationForMissingReso ->setMethods(['selectCollection']) ->getMock(); $mockColl = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->setMethods(['findOne']) ->getMock(); $mockViewColl = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'view']) ->setMethods(['findOne']) ->getMock(); @@ -875,11 +854,11 @@ public function testGetViewForResourcesInvokesViewGenerationForMissingResources( ->setMethods(['selectCollection']) ->getMock(); $mockColl = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->setMethods(['findOne']) ->getMock(); $mockViewColl = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'view']) ->setMethods(['findOne']) ->getMock(); @@ -1078,7 +1057,7 @@ public function testDeletionOfResourceTriggersViewRegeneration() $impactedSubjects = $view->getImpactedSubjects($subjectsAndPredicatesOfChange, $context); - $this->assertEquals($expectedImpactedSubjects, $impactedSubjects); + $this->assertEquals($expectedImpactedSubjects, $impactedSubjects, '', 0.0, 10, true); foreach ($impactedSubjects as $subject) { $view->update($subject); @@ -1119,7 +1098,7 @@ public function testDeletionOfResourceInImpactIndexTriggersViewRegeneration() $tripod = new \Tripod\Mongo\Driver('CBD_testing', 'tripod_php_testing', array('defaultContext'=>$context)); $tripod->saveChanges(new \Tripod\ExtendedGraph(), $originalGraph); - $collections = \Tripod\Config::getInstance()->getCollectionsForViews('tripod_php_testing', array('v_resource_full', 'v_resource_full_ttl', 'v_resource_to_single_source')); + $collections = \Tripod\Config::getInstance()->getCollectionsForViews('tripod_php_testing', array('v_resource_full', 'v_resource_to_single_source')); foreach($collections as $collection) { @@ -1302,7 +1281,7 @@ public function testUpdateOfResourceInImpactIndexTriggersViewRegeneration() $tripod = new \Tripod\Mongo\Driver('CBD_testing', 'tripod_php_testing', array('defaultContext'=>$context)); $tripod->saveChanges(new \Tripod\ExtendedGraph(), $originalGraph); - $collections = \Tripod\Config::getInstance()->getCollectionsForViews('tripod_php_testing', array('v_resource_full', 'v_resource_full_ttl', 'v_resource_to_single_source')); + $collections = \Tripod\Config::getInstance()->getCollectionsForViews('tripod_php_testing', array('v_resource_full', 'v_resource_to_single_source')); foreach($collections as $collection) { @@ -1968,11 +1947,11 @@ public function testCursorNoExceptions() ->setMethods(['selectCollection']) ->getMock(); $mockColl = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->setMethods(['findOne']) ->getMock(); $mockViewColl = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'view']) ->setMethods(['find']) ->getMock(); $mockCursor = $this->getMock('\ArrayIterator', array('rewind')); @@ -2031,11 +2010,11 @@ public function testCursorExceptionThrown() ->setMethods(['selectCollection']) ->getMock(); $mockColl = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->setMethods(['findOne']) ->getMock(); $mockViewColl = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'view']) ->setMethods(['findOne', 'find']) ->getMock(); $mockCursor = $this->getMock('\ArrayIterator', array('rewind')); @@ -2094,11 +2073,11 @@ public function testCursorNoExceptionThrownWhenCursorThrowsSomeExceptions() ->setMethods(['selectCollection']) ->getMock(); $mockColl = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->setMethods(['findOne']) ->getMock(); $mockViewColl = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'view']) ->setMethods(['find', 'findOne']) ->getMock(); $mockCursor = $this->getMock('\ArrayIterator', array('rewind')); @@ -2155,7 +2134,7 @@ public function testCursorNoExceptionThrownWhenCursorThrowsSomeExceptions() public function testCountViews() { $collection = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->setMethods(['count']) ->getMock(); $views = $this->getMockBuilder('\Tripod\Mongo\Composites\Views') @@ -2181,7 +2160,7 @@ public function testCountViewsWithFilters() $filters = ['_cts' => ['$lte' => new \MongoDB\BSON\UTCDateTime(null)]]; $query = array_merge(['_id.type' => 'v_some_spec'], $filters); $collection = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->setMethods(['count']) ->getMock(); $views = $this->getMockBuilder('\Tripod\Mongo\Composites\Views') @@ -2205,7 +2184,7 @@ public function testCountViewsWithFilters() public function testDeleteViewsByViewId() { $collection = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->setMethods(['deleteMany']) ->getMock(); @@ -2248,7 +2227,7 @@ public function testDeleteViewsByViewIdWithTimestamp() ] ]; $collection = $this->getMockBuilder('\MongoDB\Collection') - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->setMethods(['deleteMany']) ->getMock(); @@ -2301,7 +2280,7 @@ public function testBatchViewGeneration() /** @var \PHPUnit_Framework_MockObject_MockObject|\MongoDB\Collection $collection */ $collection = $this->getMockBuilder('\MongoDB\Collection') ->setMethods(['count', 'find']) - ->disableOriginalConstructor() + ->setConstructorArgs([new \MongoDB\Driver\Manager(), 'db', 'coll']) ->getMock(); $collection->expects($this->atLeastOnce())->method('count')->willReturn($count); $collection->expects($this->atLeastOnce())->method('find')->willReturn($fakeCursor); diff --git a/test/unit/mongo/TestJobBase.php b/test/unit/mongo/TestJobBase.php new file mode 100644 index 00000000..9fb1f5e1 --- /dev/null +++ b/test/unit/mongo/TestJobBase.php @@ -0,0 +1,27 @@ +