PLT-842 Update Mongo driver connection options (#148)

BREAKING CHANGES - Remove Tripod\Mongo\Config::getConnStr, getTransactionLogConnStr - These were public methods but weren't part of the interface. Instead, getDatabase or getCollectionFor<X> should be used. Features - Allow MongoDB connection string to include options - Add missing methods to IConfigInstance - Don't force default connectTimeoutMS if already set in connection string - Correct config instance type in docblocks - Correct Labeller type in MongoGraph - Test against PHP 7.4, MongoDB replica sets - Add test to cover JobBase::getTripodConfig
techfromsage · Aug 9, 2024 · 4fa81cc · 4fa81cc
1 parent d89fdc0
commit 4fa81cc
Show file tree

Hide file tree

Showing 26 changed files with 431 additions and 404 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,33 +1,155 @@
 version: 2.1
 
-jobs:
-  test:
+commands:
+  setup_replica_set:
     parameters:
-      php_version:
+      set_name:
+        type: string
+      member_hosts:
         type: string
-    docker:
-      - image: talis/tripod-php:<< parameters.php_version >>-latest
-        environment:
-          RESQUE_SERVER: redis
-      - image: mongo:3.2.21
-        name: mongodb
-      - image: redis:6.2.6
-        name: redis
     steps:
-      - checkout
-      - run: composer install
+      - run:
+          name: Setup a replica set
+          no_output_timeout: "2m"
+          environment:
+            RS_NAME: << parameters.set_name >>
+            MEMBERS: << parameters.member_hosts >>
+          command: |
+            IFS=',' read -r -a MEMBERS_ARR \<<<"$MEMBERS"
+            INITIATOR=${MEMBERS_ARR[0]}
+            MEMBERS_JS=''
+            for i in "${!MEMBERS_ARR[@]}"; do
+              MEMBERS_JS="${MEMBERS_JS}$(printf '{ _id: %d, host: "%s" },' $i "${MEMBERS_ARR[$i]}")"
+            done
+
+            MONGO_CLI="$(command -v mongosh mongo | head -n1)"
+            _mongo() { "$MONGO_CLI" --quiet --host "$@"; }
+
+            for member in "${MEMBERS_ARR[@]}"; do
+              echo "Waiting for $member"
+              until _mongo "$member" \<<<'db.adminCommand("ping")' | grep 'ok'; do
+                sleep 1
+              done
+            done
+
+            echo "Initiating replica set $RS_NAME..."
+            _mongo "$INITIATOR" \<<<"$(printf 'rs.initiate({ _id: "%s", members: [%s] })' "$RS_NAME" "$MEMBERS_JS")" | tee /dev/stderr | grep -E 'ok|already initialized'
+            echo "Waiting for primary..."
+            _mongo "$INITIATOR" \<<<'while (true) { if (rs.status().members.some(({ state }) => state === 1)) { break; } sleep(1000); }'
+            echo "Waiting for secondaries..."
+            _mongo "$INITIATOR" \<<<'while (true) { if (rs.status().members.every(({state}) => state == 1 || state == 2)) { break; } sleep(1000); }'
+            echo "Checking status..."
+            _mongo "$INITIATOR" \<<<'rs.status();' | tee /dev/stderr | grep "$RS_NAME"
+            echo "Replica set configured!"
+
+  check_mongodb_lib_version:
+    steps:
+      - run:
+          name: Check mongodb ext+lib parity
+          command: |
+            php -r 'echo phpversion("mongodb"), PHP_EOL;'
+            grep '"name": "mongodb/mongodb",' composer.lock -A1
+
+  run_test:
+    steps:
       - run: composer test -- --log-junit test-results/junit.xml
       - store_test_results:
           path: test-results/junit.xml
       - store_artifacts:
           path: test-results/junit.xml
 
+jobs:
+  test:
+    parameters:
+      php_version: { type: string }
+      mongo_version: { type: string }
+    docker:
+      - image: talis/tripod-php:<< parameters.php_version >>-latest
+      - { name: mongodb, image: mongo:<< parameters.mongo_version >> }
+      - { name: redis, image: redis:6.2.6 }
+    environment:
+      RESQUE_SERVER: redis
+    steps:
+      - checkout
+      - run: composer install
+      - check_mongodb_lib_version
+      - run_test
+
+  test-multiple-stores:
+    docker:
+      - image: talis/tripod-php:php74-latest
+      - { name: mongo1, image: mongo:4.4.29 }
+      - { name: mongo2, image: mongo:5.0.28 }
+      - { name: redis, image: redis:6.2.6 }
+    environment:
+      RESQUE_SERVER: redis
+      TRIPOD_DATASOURCE_RS1_CONFIG: |
+        {"type":"mongo", "connection":"mongodb://mongo1:27017/", "replicaSet":""}
+      TRIPOD_DATASOURCE_RS2_CONFIG: |
+        {"type":"mongo", "connection":"mongodb://mongo2:27017/", "replicaSet":""}
+    steps:
+      - checkout
+      - run: composer install
+      - check_mongodb_lib_version
+      - run_test
+
+  test-replica-set-mmap:
+    docker:
+      - image: talis/tripod-php:php74-latest
+      - { name: mongo1, image: mongo:3.6.23, command: mongod --storageEngine mmapv1 --smallfiles --replSet=tripod-rs }
+      - { name: mongo2, image: mongo:3.6.23, command: mongod --storageEngine mmapv1 --smallfiles --replSet=tripod-rs }
+      - { name: redis, image: redis:6.2.6 }
+    environment:
+      RESQUE_SERVER: redis
+      TRIPOD_DATASOURCE_RS1_CONFIG: |
+        {"type":"mongo", "connection":"mongodb://mongo1,mongo2/?retryWrites=false", "replicaSet":"tripod-rs"}
+      TRIPOD_DATASOURCE_RS2_CONFIG: |
+        {"type":"mongo", "connection":"mongodb://mongo1,mongo2/?retryWrites=false", "replicaSet":"tripod-rs"}
+    steps:
+      - checkout
+      - setup_replica_set:
+          set_name: tripod-rs
+          member_hosts: mongo1,mongo2
+      - run: composer install
+      - check_mongodb_lib_version
+      - run_test
+
+  test-replica-set-wiredtiger:
+    docker:
+      - image: talis/tripod-php:php74-latest
+      - { name: mongo1, image: mongo:5.0.28, command: mongod --replSet=tripod-rs }
+      - { name: mongo2, image: mongo:5.0.28, command: mongod --replSet=tripod-rs }
+      - { name: redis, image: redis:6.2.6 }
+    environment:
+      RESQUE_SERVER: redis
+      TRIPOD_DATASOURCE_RS1_CONFIG: |
+        {"type":"mongo", "connection":"mongodb://mongo1,mongo2/admin?replicaSet=tripod-rs", "replicaSet":""}
+      TRIPOD_DATASOURCE_RS2_CONFIG: |
+        {"type":"mongo", "connection":"mongodb://mongo1,mongo2/admin?replicaSet=tripod-rs", "replicaSet":""}
+    steps:
+      - checkout
+      - setup_replica_set:
+          set_name: tripod-rs
+          member_hosts: mongo1,mongo2
+      - run: composer install
+      - check_mongodb_lib_version
+      - run_test
+
 workflows:
   build_and_test:
     jobs:
       - test:
-          matrix:
-            parameters:
-              php_version:
-                - php55
-                - php73
+          name: test-php55
+          php_version: php55
+          mongo_version: 3.2.21
+      - test:
+          name: test-php73
+          php_version: php73
+          mongo_version: 3.6.23
+      - test:
+          name: test-php74
+          php_version: php74
+          mongo_version: 4.0.28
+      - test-multiple-stores
+      - test-replica-set-mmap
+      - test-replica-set-wiredtiger
diff --git a/composer.json b/composer.json
@@ -18,18 +18,17 @@
         }
     ],
     "suggest": {
-        "mongodb/mongodb": "MongoDB driver library",
         "resque/php-resque": "Redis backed library for background jobs"
     },
     "require": {
         "php" : ">=5.5",
-        "semsol/arc2": "2.2.6",
-        "monolog/monolog" : "~1.13"
+        "mongodb/mongodb": "*",
+        "monolog/monolog" : "~1.13",
+        "semsol/arc2": "2.2.6"
     },
     "require-dev": {
-        "resque/php-resque": "v1.3.4",
-        "mongodb/mongodb": "1.4.3",
         "phpunit/phpunit": "^4.8",
+        "resque/php-resque": "v1.3.4",
         "squizlabs/php_codesniffer": "3.2.*"
     },
     "autoload": {

diff --git a/docker-compose.clusters.yml b/docker-compose.clusters.yml
@@ -0,0 +1,16 @@
+# An override file to test Tripod on multiple clusters.
+# Usage:
+# docker compose -f docker-compose.yml -f docker-compose.clusters.yml run --rm -it php74 vendor/bin/phpunit
+
+services:
+  php74:
+    depends_on:
+      - mongodb2
+    environment:
+      TRIPOD_DATASOURCE_RS1_CONFIG: |
+        {"type":"mongo", "connection":"mongodb://mongodb:27017/", "replicaSet":""}
+      TRIPOD_DATASOURCE_RS2_CONFIG: |
+        {"type":"mongo", "connection":"mongodb://mongodb2:27017/", "replicaSet":""}
+
+  mongodb2:
+    image: mongo:4.4.29
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,35 +1,36 @@
-version: "3.7"
-
 x-base-config: &base-config
   volumes:
     - ./:/var/tripod-php
-    - ./vendor:/var/tripod-php/vendor:delegated
-  links:
-    - "mongo32:mongodb"
-    - redis
   depends_on:
-    - mongo32
+    - mongodb
     - redis
   working_dir: /var/tripod-php
   env_file: .env
 
 services:
   php55:
-    # build:
-    #   context: ./docker
-    #   dockerfile: Dockerfile-php55
+    build:
+      context: ./docker
+      dockerfile: Dockerfile-php55
     image: talis/tripod-php:php55-latest
     <<: *base-config
 
   php73:
-    # build:
-    #   context: ./docker
-    #   dockerfile: Dockerfile-php73
+    build:
+      context: ./docker
+      dockerfile: Dockerfile-php73
     image: talis/tripod-php:php73-latest
     <<: *base-config
 
-  mongo32:
-    image: mongo:3.2.21
+  php74:
+    build:
+      context: ./docker
+      dockerfile: Dockerfile-php74
+    image: talis/tripod-php:php74-latest
+    <<: *base-config
+
+  mongodb:
+    image: mongo:3.6.23
 
   redis:
     image: redis:6.2.6
diff --git a/docker/Dockerfile-php55 b/docker/Dockerfile-php55
@@ -15,8 +15,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   zip \
   && rm -rf /var/lib/apt/lists/*
 
-COPY --from=mlocati/php-extension-installer:2.1.58 /usr/bin/install-php-extensions /usr/local/bin/
-RUN install-php-extensions pcntl redis mongodb-1.5.5 && \
-    curl https://getcomposer.org/installer >/tmp/composer-setup.php && \
-    php /tmp/composer-setup.php --install-dir /usr/local/bin/ --filename composer && \
-    rm /tmp/composer-setup.php
+RUN curl -sLo - https://www.mongodb.org/static/pgp/server-3.6.asc | apt-key add - \
+  && echo "deb http://repo.mongodb.org/apt/debian jessie/mongodb-org/3.6 main" | tee /etc/apt/sources.list.d/mongodb-org-3.6.list \
+  && apt-get update && apt-get install -y --no-install-recommends mongodb-org-shell --force-yes \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --from=mlocati/php-extension-installer:2.3.2 /usr/bin/install-php-extensions /usr/local/bin/
+COPY --from=composer:2.2.20 /usr/bin/composer /usr/bin/composer
+
+RUN install-php-extensions pcntl mongodb-1.5.5
diff --git a/docker/Dockerfile-php73 b/docker/Dockerfile-php73
@@ -1,4 +1,4 @@
-FROM php:7.3-cli
+FROM php:7.3.33-cli
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
   ca-certificates \
@@ -8,8 +8,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   zip \
   && rm -rf /var/lib/apt/lists/*
 
-COPY --from=mlocati/php-extension-installer:2.1.58 /usr/bin/install-php-extensions /usr/local/bin/
-RUN install-php-extensions pcntl redis mongodb-1.5.5 && \
-    curl https://getcomposer.org/installer >/tmp/composer-setup.php && \
-    php /tmp/composer-setup.php --install-dir /usr/local/bin/ --filename composer && \
-    rm /tmp/composer-setup.php
+RUN curl -sLo /tmp/mongosh.deb https://downloads.mongodb.com/compass/mongodb-mongosh_2.2.15_amd64.deb \
+  && dpkg -i /tmp/mongosh.deb \
+  && rm /tmp/mongosh.deb
+
+COPY --from=mlocati/php-extension-installer:2.3.2 /usr/bin/install-php-extensions /usr/local/bin/
+COPY --from=composer:2.7.7 /usr/bin/composer /usr/local/bin/
+
+RUN install-php-extensions pcntl mongodb-1.6.1
diff --git a/docker/Dockerfile-php74 b/docker/Dockerfile-php74
@@ -0,0 +1,18 @@
+FROM php:7.4.33-cli
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  ca-certificates \
+  curl \
+  git \
+  unzip \
+  zip \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN curl -sLo /tmp/mongosh.deb https://downloads.mongodb.com/compass/mongodb-mongosh_2.2.15_amd64.deb \
+  && dpkg -i /tmp/mongosh.deb \
+  && rm /tmp/mongosh.deb
+
+COPY --from=mlocati/php-extension-installer:2.3.2 /usr/bin/install-php-extensions /usr/local/bin/
+COPY --from=composer:2.7.7 /usr/bin/composer /usr/local/bin/
+
+RUN install-php-extensions pcntl mongodb-1.19.3
diff --git a/src/Config.php b/src/Config.php
@@ -26,7 +26,7 @@ private function __construct()
      * @uses Config::setConfig() Configuration must be set prior to calling this method. To generate a completely new object, set a new config
      * @codeCoverageIgnore
      * @throws \Tripod\Exceptions\ConfigException
-     * @return ITripodConfig
+     * @return \Tripod\Mongo\IConfigInstance
      */
     public static function getInstance()
     {

diff --git a/src/ITripodConfig.php b/src/ITripodConfig.php
@@ -7,7 +7,7 @@ interface ITripodConfig
     /**
      * Tripod Config instances are singletons, this method gets the existing or instantiates a new one.
      *
-     * @return ITripodConfig
+     * @return \Tripod\Mongo\IConfigInstance
      */
     public static function getInstance();
 

diff --git a/src/ITripodConfigSerializer.php b/src/ITripodConfigSerializer.php
@@ -16,7 +16,7 @@ public function serialize();
      * When given a valid config, returns a Tripod Config object
      *
      * @param array $config
-     * @return \Tripod\ITripodConfig
+     * @return \Tripod\Mongo\IConfigInstance
      */
     public static function deserialize(array $config);
 }
diff --git a/src/TripodConfigFactory.php b/src/TripodConfigFactory.php
@@ -5,11 +5,11 @@
 class TripodConfigFactory
 {
     /**
-     * Factory method to get a Tripod\ITripodConfig instance from either a config array, or a serialized
+     * Factory method to get a Tripod config instance from either a config array, or a serialized
      * ITripodConfigSerializer instance
      *
      * @param array $config The Tripod config or serialized ITripodConfigSerializer array
-     * @return \Tripod\ITripodConfig
+     * @return \Tripod\Mongo\IConfigInstance
      */
     public static function create(array $config)
     {