From d6a48ccd13b5f1493d02d373df5ee6258c31e18b Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 2 Nov 2017 12:15:30 -0400
Subject: [PATCH 01/42] Added created timestamps to composites

---
 .editorconfig                                 | 15 ++++
 src/mongo/delegates/SearchDocuments.class.php |  2 +-
 src/mongo/delegates/Tables.class.php          | 74 +++++++++--------
 src/mongo/delegates/Views.class.php           | 79 ++++++++++---------
 .../providers/MongoSearchProvider.class.php   | 23 ++++--
 5 files changed, 112 insertions(+), 81 deletions(-)
 create mode 100644 .editorconfig

diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 00000000..adbc1517
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,15 @@
+# This file is for unifying the coding style for different editors and IDEs
+# editorconfig.org
+
+# PHP PSR-2 Coding Standards
+# http://www.php-fig.org/psr/psr-2/
+
+root = true
+
+[*.php]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 4
\ No newline at end of file
diff --git a/src/mongo/delegates/SearchDocuments.class.php b/src/mongo/delegates/SearchDocuments.class.php
index 013790b4..716142be 100644
--- a/src/mongo/delegates/SearchDocuments.class.php
+++ b/src/mongo/delegates/SearchDocuments.class.php
@@ -111,7 +111,7 @@ public function generateSearchDocumentBasedOnSpecId($specId, $resource, $context
         $this->debugLog("Processing {$specId}");
 
         // build the document
-        $generatedDocument = array();
+        $generatedDocument = [\_CREATED_TS => new \MongoDB\BSON\UTCDateTime()];
         $this->addIdToImpactIndex($_id, $generatedDocument);
 
         $_id['type'] = $specId;
diff --git a/src/mongo/delegates/Tables.class.php b/src/mongo/delegates/Tables.class.php
index 8df1da74..d19fd607 100644
--- a/src/mongo/delegates/Tables.class.php
+++ b/src/mongo/delegates/Tables.class.php
@@ -358,19 +358,30 @@ protected function deleteTableRowsForResource($resource, $context=null, $specTyp
 
     /**
      * This method will delete all table rows where the _id.type matches the specified $tableId
-     * @param string $tableId
+     * @param string                         $tableId   Table spec ID
+     * @param \MongoDB\BSON\UTCDateTime|null $timestamp Optional timestamp to delete all table rows that are older than
      */
-    public function deleteTableRowsByTableId($tableId) {
+    public function deleteTableRowsByTableId($tableId, $timestamp = null) {
         $t = new \Tripod\Timer();
         $t->start();
         $tableSpec = Config::getInstance()->getTableSpecification($this->storeName, $tableId);
-        if ($tableSpec==null)
-        {
+        if ($tableSpec == null) {
             $this->debugLog("Could not find a table specification for $tableId");
             return;
         }
 
-        $query = array("_id.type"=>$tableId);
+        $query = ['_id.type' => $tableId];
+        if ($timestamp) {
+            if (!($timestamp instanceof \MongoDB\BSON\UTCDateTime)) {
+                $timestamp = new \MongoDB\BSON\UTCDateTime($timestamp);
+            }
+            $query[\_CREATED_TS] = [
+                '$or' => [
+                    ['$lt' => $timestamp],
+                    ['$exists' => false]
+                ]
+            ];
+        }
         $this->config->getCollectionForTable($this->storeName, $tableId)
             ->deleteMany($query);
 
@@ -488,7 +499,7 @@ public function generateTableRowsForType($rdfType,$subject=null,$context=null, $
      * @param string|null $queueName Queue for background bulk generation
      * @return null //@todo: this should be a bool
      */
-    public function generateTableRows($tableType,$resource=null,$context=null,$queueName=null)
+    public function generateTableRows($tableType, $resource = null, $context = null, $queueName = null)
     {
         $t = new \Tripod\Timer();
         $t->start();
@@ -496,8 +507,7 @@ public function generateTableRows($tableType,$resource=null,$context=null,$queue
         $tableSpec = Config::getInstance()->getTableSpecification($this->storeName, $tableType);
         $collection = $this->config->getCollectionForTable($this->storeName, $tableType);
 
-        if ($tableSpec==null)
-        {
+        if ($tableSpec==null) {
             $this->debugLog("Could not find a table specification for $tableType");
             return null;
         }
@@ -509,22 +519,17 @@ public function generateTableRows($tableType,$resource=null,$context=null,$queue
         $from = (isset($tableSpec["from"])) ? $tableSpec["from"] : $this->podName;
 
         $types = array();
-        if (is_array($tableSpec["type"]))
-        {
-            foreach ($tableSpec["type"] as $type)
-            {
+        if (is_array($tableSpec["type"])) {
+            foreach ($tableSpec["type"] as $type) {
                 $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($type));
                 $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($type));
             }
-        }
-        else
-        {
+        } else {
             $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($tableSpec["type"]));
             $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($tableSpec["type"]));
         }
         $filter = array('$or'=> $types);
-        if (isset($resource))
-        {
+        if (isset($resource)) {
             $filter["_id"] = array(_ID_RESOURCE=>$this->labeller->uri_to_alias($resource),_ID_CONTEXT=>$contextAlias);
         }
 
@@ -532,10 +537,8 @@ public function generateTableRows($tableType,$resource=null,$context=null,$queue
             'maxTimeMS' => 1000000
         ));
 
-        foreach ($docs as $doc)
-        {
-            if($queueName && !$resource)
-            {
+        foreach ($docs as $doc) {
+            if ($queueName && !$resource) {
                 $subject = new ImpactedSubject(
                     $doc['_id'],
                     OP_TABLES,
@@ -546,32 +549,33 @@ public function generateTableRows($tableType,$resource=null,$context=null,$queue
 
                 $jobOptions = array();
 
-                if($this->stat || !empty($this->statsConfig))
-                {
+                if($this->stat || !empty($this->statsConfig)) {
                     $jobOptions['statsConfig'] = $this->getStatsConfig();
                 }
 
                 $this->getApplyOperation()->createJob(array($subject), $queueName, $jobOptions);
-            }
-            else
-            {
+            } else {
                 // set up ID
-                $generatedRow = array("_id"=>array(_ID_RESOURCE=>$doc["_id"][_ID_RESOURCE],_ID_CONTEXT=>$doc["_id"][_ID_CONTEXT],_ID_TYPE=>$tableSpec['_id']));
-
-                $value = array('_id'=>$doc['_id']); // everything must go in the value object todo: this is a hang over from map reduce days, engineer out once we have stability on new PHP method for M/R
+                $generatedRow = [
+                    '_id' => [
+                        _ID_RESOURCE => $doc['_id'][_ID_RESOURCE],
+                        _ID_CONTEXT => $doc['_id'][_ID_CONTEXT],
+                        _ID_TYPE=>$tableSpec['_id']
+                    ],
+                    \_CREATED_TS => new \MongoDB\BSON\UTCDateTime()
+                ];
+                // everything must go in the value object todo: this is a hang over from map reduce days, engineer out once we have stability on new PHP method for M/R
+                $value = array('_id'=>$doc['_id'], '_cts' => new \MongoDB\BSON\UTCDateTime());
                 $this->addIdToImpactIndex($doc['_id'], $value); // need to add the doc to the impact index to be consistent with views/search etc. this is needed for discovering impacted operations
                 $this->addFields($doc,$tableSpec,$value);
-                if (isset($tableSpec['joins']))
-                {
+                if (isset($tableSpec['joins'])) {
                     $this->doJoins($doc,$tableSpec['joins'],$value,$from,$contextAlias);
                 }
-                if (isset($tableSpec['counts']))
-                {
+                if (isset($tableSpec['counts'])) {
                     $this->doCounts($doc,$tableSpec['counts'],$value);
                 }
 
-                if (isset($tableSpec['computed_fields']))
-                {
+                if (isset($tableSpec['computed_fields'])) {
                     $this->doComputedFields($tableSpec, $value);
                 }
 
diff --git a/src/mongo/delegates/Views.class.php b/src/mongo/delegates/Views.class.php
index 0067d327..a96da895 100644
--- a/src/mongo/delegates/Views.class.php
+++ b/src/mongo/delegates/Views.class.php
@@ -365,18 +365,30 @@ public function generateViewsForResourcesOfType($rdfType,$resource=null,$context
 
     /**
      * This method will delete all views where the _id.type of the viewmatches the specified $viewId
-     * @param $viewId
+     * @param string                         $viewId    View spec ID
+     * @param \MongoDB\BSON\UTCDateTime|null $timestamp Optional timestamp to delete all views that are older than
      */
-    public function deleteViewsByViewId($viewId){
+    public function deleteViewsByViewId($viewId, $timestamp = null)
+    {
         $viewSpec = Config::getInstance()->getViewSpecification($this->storeName, $viewId);
-        if ($viewSpec==null)
-        {
+        if ($viewSpec == null) {
             $this->debugLog("Could not find a view specification with viewId '$viewId'");
             return;
         }
-
+        $query = ['_id.type' => $viewId];
+        if ($timestamp) {
+            if (!($timestamp instanceof \MongoDB\BSON\UTCDateTime)) {
+                $timestamp = new \MongoDB\BSON\UTCDateTime($timestamp);
+            }
+            $query[\_CREATED_TS] = [
+                '$or' => [
+                    ['$lt' => $timestamp],
+                    ['$exists' => false]
+                ]
+            ];
+        }
         $this->config->getCollectionForView($this->storeName, $viewId)
-            ->deleteMany(array("_id.type"=>$viewId));
+            ->deleteMany($query);
     }
 
     /**
@@ -392,41 +404,32 @@ public function generateView($viewId,$resource=null,$context=null,$queueName=nul
     {
         $contextAlias = $this->getContextAlias($context);
         $viewSpec = Config::getInstance()->getViewSpecification($this->storeName, $viewId);
-        if ($viewSpec==null)
-        {
+        if ($viewSpec == null) {
             $this->debugLog("Could not find a view specification for $resource with viewId '$viewId'");
             return null;
-        }
-        else
-        {
+        } else {
             $t = new \Tripod\Timer();
             $t->start();
 
             $from = $this->getFromCollectionForViewSpec($viewSpec);
             $collection = $this->config->getCollectionForView($this->storeName, $viewId);
 
-            if (!isset($viewSpec['joins']))
-            {
+            if (!isset($viewSpec['joins'])) {
                 throw new \Tripod\Exceptions\ViewException('Could not find any joins in view specification - usecase better served with select()');
             }
 
             $types = array(); // this is used to filter the CBD table to speed up the view creation
-            if (is_array($viewSpec["type"]))
-            {
-                foreach ($viewSpec["type"] as $type)
-                {
+            if (is_array($viewSpec["type"])) {
+                foreach ($viewSpec["type"] as $type) {
                     $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($type));
                     $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($type));
                 }
-            }
-            else
-            {
+            } else {
                 $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($viewSpec["type"]));
                 $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($viewSpec["type"]));
             }
             $filter = array('$or'=> $types);
-            if (isset($resource))
-            {
+            if (isset($resource)) {
                 $resourceAlias = $this->labeller->uri_to_alias($resource);
                 $filter["_id"] = array(_ID_RESOURCE=>$resourceAlias,_ID_CONTEXT=>$contextAlias);
             }
@@ -435,10 +438,8 @@ public function generateView($viewId,$resource=null,$context=null,$queueName=nul
                 'maxTimeMS' => \Tripod\Mongo\Config::getInstance()->getMongoCursorTimeout()
             ));
 
-            foreach ($docs as $doc)
-            {
-                if($queueName && !$resource)
-                {
+            foreach ($docs as $doc) {
+                if ($queueName && !$resource) {
                     $subject = new ImpactedSubject(
                         $doc['_id'],
                         OP_VIEWS,
@@ -448,30 +449,30 @@ public function generateView($viewId,$resource=null,$context=null,$queueName=nul
                     );
 
                     $jobOptions = array();
-                    if($this->stat || !empty($this->statsConfig))
-                    {
+                    if ($this->stat || !empty($this->statsConfig)) {
                         $jobOptions['statsConfig'] = $this->getStatsConfig();
                     }
 
                     $this->getApplyOperation()->createJob(array($subject), $queueName, $jobOptions);
-                }
-                else
-                {
-                    // set up ID
-                    $id = array("_id"=>array(_ID_RESOURCE=>$doc["_id"][_ID_RESOURCE],_ID_CONTEXT=>$doc["_id"][_ID_CONTEXT],_ID_TYPE=>$viewSpec['_id']));
-                    $generatedView = $id;
+                } else {
+                    // Set up view meta information
+                    $generatedView = [
+                        '_id' => [
+                            _ID_RESOURCE => $doc['_id'][_ID_RESOURCE],
+                            _ID_CONTEXT => $doc['_id'][_ID_CONTEXT],
+                            _ID_TYPE=>$viewSpec['_id']
+                        ],
+                        \_CREATED_TS => new \MongoDB\BSON\UTCDateTime()
+                    ];
                     $value = array(); // everything must go in the value object todo: this is a hang over from map reduce days, engineer out once we have stability on new PHP method for M/R
 
                     $value[_GRAPHS] = array();
 
                     $buildImpactIndex=true;
-                    if (isset($viewSpec['ttl']))
-                    {
+                    if (isset($viewSpec['ttl'])) {
                         $buildImpactIndex=false;
                         $value[_EXPIRES] = \Tripod\Mongo\DateUtil::getMongoDate($this->getExpirySecFromNow($viewSpec['ttl']) * 1000);
-                    }
-                    else
-                    {
+                    } else {
                         $value[_IMPACT_INDEX] = array($doc['_id']);
                     }
 
diff --git a/src/mongo/providers/MongoSearchProvider.class.php b/src/mongo/providers/MongoSearchProvider.class.php
index d5895755..f2c6916d 100644
--- a/src/mongo/providers/MongoSearchProvider.class.php
+++ b/src/mongo/providers/MongoSearchProvider.class.php
@@ -336,20 +336,31 @@ public function getSearchCollectionName()
      * Removes all documents from search index based on the specified type id.
      * Here search type id represents to id from, mongo tripod config, that is converted to _id.type in SEARCH_INDEX_COLLECTION
      * If type id is not specified this method will throw an exception.
-     * @param string $typeId search type id
+     * @param string                         $typeId    Search type id
+     * @param \MongoDB\BSON\UTCDateTime|null $timestamp Optional timestamp to delete all search docs that are older than
      * @return bool|array  response returned by mongo
      * @throws \Tripod\Exceptions\Exception if there was an error performing the operation
      */
-    public function deleteSearchDocumentsByTypeId($typeId)
+    public function deleteSearchDocumentsByTypeId($typeId, $timestamp = null)
     {
         $searchSpec = $this->getSearchDocumentSpecification($typeId);
-        if ($searchSpec == null)
-        {
+        if ($searchSpec == null) {
             throw new \Tripod\Exceptions\SearchException("Could not find a search specification for $typeId");
         }
-
+        $query = ['_id.type' => $typeId];
+        if ($timestamp) {
+            if (!($timestamp instanceof \MongoDB\BSON\UTCDateTime)) {
+                $timestamp = new \MongoDB\BSON\UTCDateTime($timestamp);
+            }
+            $query[\_CREATED_TS] = [
+                '$or' => [
+                    ['$lt' => $timestamp],
+                    ['$exists' => false]
+                ]
+            ];
+        }
         return $this->config->getCollectionForSearchDocument($this->storeName, $typeId)
-            ->deleteMany(array("_id.type" => $typeId));
+            ->deleteMany($query);
     }
 
     /**

From 1946895122f1d99437698d7f745567dd20b73907 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 2 Nov 2017 12:23:37 -0400
Subject: [PATCH 02/42] Update mongo for travis; specify min php requirements

---
 .travis.yml   | 2 +-
 composer.json | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 6dc6d668..138bd332 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,7 @@ php:
 services:
 - redis
 before_install:
-- sudo apt-get install -y mongodb-org=2.6.9 mongodb-org-server=2.6.9 mongodb-org-shell=2.6.9 mongodb-org-mongos=2.6.9 mongodb-org-tools=2.6.9
+- sudo apt-get install -y mongodb-org=2.6.* mongodb-org-server=2.6.* mongodb-org-shell=2.6.* mongodb-org-mongos=2.6.* mongodb-org-tools=2.6.*
 - echo "extension = mongodb.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini
 - sleep 15
 install:
diff --git a/composer.json b/composer.json
index ba2e11a8..5d941e4a 100644
--- a/composer.json
+++ b/composer.json
@@ -18,6 +18,7 @@
         }
     ],
     "require": {
+        "php" : ">=5.4",
         "semsol/arc2": "v2.2.4",
         "chrisboulton/php-resque": "dev-master#98fde571db008a8b48e73022599d1d1c07d4a7b5",
         "monolog/monolog" : "~1.13",

From a54d944d354b43f7d2bc66919d5cb95482ceac94 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 2 Nov 2017 12:26:39 -0400
Subject: [PATCH 03/42] Ok, then, 2.6.12

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 138bd332..3828b8f8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -8,7 +8,7 @@ php:
 services:
 - redis
 before_install:
-- sudo apt-get install -y mongodb-org=2.6.* mongodb-org-server=2.6.* mongodb-org-shell=2.6.* mongodb-org-mongos=2.6.* mongodb-org-tools=2.6.*
+- sudo apt-get install -y mongodb-org=2.6.12 mongodb-org-server=2.6.12 mongodb-org-shell=2.6.12 mongodb-org-mongos=2.6.12 mongodb-org-tools=2.6.12
 - echo "extension = mongodb.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini
 - sleep 15
 install:

From ad85dee8bf574074d2864fdea344651a99ebe884 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 2 Nov 2017 13:19:38 -0400
Subject: [PATCH 04/42] Faff about with Docker

---
 .travis.yml        | 5 ++++-
 docker-compose.yml | 4 ++++
 2 files changed, 8 insertions(+), 1 deletion(-)
 create mode 100644 docker-compose.yml

diff --git a/.travis.yml b/.travis.yml
index 3828b8f8..ebd54c8f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,8 +7,11 @@ php:
 - 5.4
 services:
 - redis
+- docker
 before_install:
-- sudo apt-get install -y mongodb-org=2.6.12 mongodb-org-server=2.6.12 mongodb-org-shell=2.6.12 mongodb-org-mongos=2.6.12 mongodb-org-tools=2.6.12
+- wget https://raw.githubusercontent.com/docker-library/mongo/0ac2867637ef5989e4dc051efa0ae296010e58c9/2.6/Dockerfile
+- docker build -t mongo/2.6.12 .
+- docker run -d -p 127.0.0.1:27017:27017 mongo/2.6.12 /bin/sh -c "mongod"
 - echo "extension = mongodb.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini
 - sleep 15
 install:
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 00000000..3cf8dd7f
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,4 @@
+mongo:
+  image: mongo/2.6.12:latest
+  ports:
+      - "27017:27017"
\ No newline at end of file

From 8e4fa22e2e2de956ced455b01233e893df065ea9 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 2 Nov 2017 13:22:17 -0400
Subject: [PATCH 05/42] Need to grab another file

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index ebd54c8f..a1ab458b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,6 +10,7 @@ services:
 - docker
 before_install:
 - wget https://raw.githubusercontent.com/docker-library/mongo/0ac2867637ef5989e4dc051efa0ae296010e58c9/2.6/Dockerfile
+- wget https://raw.githubusercontent.com/docker-library/mongo/0ac2867637ef5989e4dc051efa0ae296010e58c9/2.6/docker-entrypoint.sh
 - docker build -t mongo/2.6.12 .
 - docker run -d -p 127.0.0.1:27017:27017 mongo/2.6.12 /bin/sh -c "mongod"
 - echo "extension = mongodb.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini

From 33b34802d460f574c208bf7589b958fefe5ad0bf Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 2 Nov 2017 13:53:37 -0400
Subject: [PATCH 06/42] fix docker run

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index a1ab458b..6239f985 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,7 +12,7 @@ before_install:
 - wget https://raw.githubusercontent.com/docker-library/mongo/0ac2867637ef5989e4dc051efa0ae296010e58c9/2.6/Dockerfile
 - wget https://raw.githubusercontent.com/docker-library/mongo/0ac2867637ef5989e4dc051efa0ae296010e58c9/2.6/docker-entrypoint.sh
 - docker build -t mongo/2.6.12 .
-- docker run -d -p 127.0.0.1:27017:27017 mongo/2.6.12 /bin/sh -c "mongod"
+- docker run -d -p 27017:27017 -v ~/data:/data/db mongo
 - echo "extension = mongodb.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini
 - sleep 15
 install:

From cb777a776fbdf69d589094a4e1889e7e86b4a04c Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 2 Nov 2017 13:59:22 -0400
Subject: [PATCH 07/42] Add tag

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 6239f985..3bc93ce1 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,7 +12,7 @@ before_install:
 - wget https://raw.githubusercontent.com/docker-library/mongo/0ac2867637ef5989e4dc051efa0ae296010e58c9/2.6/Dockerfile
 - wget https://raw.githubusercontent.com/docker-library/mongo/0ac2867637ef5989e4dc051efa0ae296010e58c9/2.6/docker-entrypoint.sh
 - docker build -t mongo/2.6.12 .
-- docker run -d -p 27017:27017 -v ~/data:/data/db mongo
+- docker run -d -p 27017:27017 -v ~/data:/data/db mongo:2.6.12
 - echo "extension = mongodb.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini
 - sleep 15
 install:

From 0c2f9a1c408371375a21eb3c9532e3d87e34030c Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 2 Nov 2017 14:03:06 -0400
Subject: [PATCH 08/42] References to removed $id variable

---
 src/mongo/delegates/Views.class.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mongo/delegates/Views.class.php b/src/mongo/delegates/Views.class.php
index a96da895..c5635e57 100644
--- a/src/mongo/delegates/Views.class.php
+++ b/src/mongo/delegates/Views.class.php
@@ -483,7 +483,7 @@ public function generateView($viewId,$resource=null,$context=null,$queueName=nul
 
                     $generatedView['value'] = $value;
 
-                    $collection->replaceOne($id, $generatedView, ['upsert' => true]);
+                    $collection->replaceOne($generatedView['_id'], $generatedView, ['upsert' => true]);
                 }
             }
 

From 8f15d91cb008a45c609047571eed5292837ea88b Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 2 Nov 2017 14:08:28 -0400
Subject: [PATCH 09/42] More fumbling with docker

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 3bc93ce1..cc56c6d7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -12,7 +12,7 @@ before_install:
 - wget https://raw.githubusercontent.com/docker-library/mongo/0ac2867637ef5989e4dc051efa0ae296010e58c9/2.6/Dockerfile
 - wget https://raw.githubusercontent.com/docker-library/mongo/0ac2867637ef5989e4dc051efa0ae296010e58c9/2.6/docker-entrypoint.sh
 - docker build -t mongo/2.6.12 .
-- docker run -d -p 27017:27017 -v ~/data:/data/db mongo:2.6.12
+- docker run -d -p 27017:27017 -v ~/data:/data/db mongo/2.6.12:latest
 - echo "extension = mongodb.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini
 - sleep 15
 install:

From 3bc30ab5a464f2cb5409bbfd88f1d89d79e3bcfe Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 2 Nov 2017 14:13:16 -0400
Subject: [PATCH 10/42] Make script executable

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index cc56c6d7..c2046a34 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,6 +11,7 @@ services:
 before_install:
 - wget https://raw.githubusercontent.com/docker-library/mongo/0ac2867637ef5989e4dc051efa0ae296010e58c9/2.6/Dockerfile
 - wget https://raw.githubusercontent.com/docker-library/mongo/0ac2867637ef5989e4dc051efa0ae296010e58c9/2.6/docker-entrypoint.sh
+- chmod 0777 docker-entrypoint.sh
 - docker build -t mongo/2.6.12 .
 - docker run -d -p 27017:27017 -v ~/data:/data/db mongo/2.6.12:latest
 - echo "extension = mongodb.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini

From 3d4ccf343760981b36a202acf055ef6acaf3290b Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 2 Nov 2017 14:29:30 -0400
Subject: [PATCH 11/42] Fix tests

---
 test/unit/mongo/MongoTripodViewsTest.php | 33 ++++++++++++++++++------
 1 file changed, 25 insertions(+), 8 deletions(-)

diff --git a/test/unit/mongo/MongoTripodViewsTest.php b/test/unit/mongo/MongoTripodViewsTest.php
index e26cb60b..216bfa93 100644
--- a/test/unit/mongo/MongoTripodViewsTest.php
+++ b/test/unit/mongo/MongoTripodViewsTest.php
@@ -108,7 +108,9 @@ public function testGenerateView()
             ['typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array']]
         );
         $actualView = $mongo->selectCollection('tripod_php_testing','views')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/3SplCtWGPqEyXcDiyhHQpA',"c"=>'http://talisaspire.com/',"type"=>'v_resource_full')));
-        $this->assertEquals($expectedView,$actualView);
+        $this->assertEquals($expectedView['_id'], $actualView['_id']);
+        $this->assertEquals($expectedView['value'], $actualView['value']);
+        $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']);
     }
 
     /**
@@ -179,7 +181,9 @@ public function testGenerateViewWithFilterRemovesFilteredDataButKeepsResourcesIn
             ['typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array']]
         );
         $actualView = $mongo->selectCollection('tripod_php_testing','views')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_filter1')));
-        $this->assertEquals($expectedView,$actualView);
+        $this->assertEquals($expectedView['_id'], $actualView['_id']);
+        $this->assertEquals($expectedView['value'], $actualView['value']);
+        $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']);
     }
 
     /**
@@ -244,7 +248,9 @@ public function testGenerateViewWithFilterOnLiteralValue()
         $actualView = $mongo->selectCollection('tripod_php_testing','views')->findOne(
             array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_filter2'))
         );
-        $this->assertEquals($expectedView,$actualView);
+        $this->assertEquals($expectedView['_id'], $actualView['_id']);
+        $this->assertEquals($expectedView['value'], $actualView['value']);
+        $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']);
     }
 
     /**
@@ -315,7 +321,9 @@ public function testGenerateViewCorrectlyAfterUpdateAffectsFilter()
             ['typeMap' => ['root' => 'array', 'document' => 'array', 'array' => 'array']]
         );
         $actualView = $mongo->selectCollection('tripod_php_testing','views')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_filter1')));
-        $this->assertEquals($expectedView,$actualView);
+        $this->assertEquals($expectedView['_id'], $actualView['_id']);
+        $this->assertEquals($expectedView['value'], $actualView['value']);
+        $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']);
 
         // Modify http://talisaspire.com/works/filter1 so that it is a Chapter (included in the view) not a Book (excluded from the view)
         $oldGraph = new \Tripod\ExtendedGraph();
@@ -383,6 +391,9 @@ public function testGenerateViewCorrectlyAfterUpdateAffectsFilter()
 
         $updatedView = $mongo->selectCollection('tripod_php_testing','views')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_filter1')));
         $this->assertEquals($expectedUpdatedView,$updatedView);
+        $this->assertEquals($expectedUpdatedView['_id'], $updatedView['_id']);
+        $this->assertEquals($expectedUpdatedView['value'], $updatedView['value']);
+        $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $updatedView['_cts']);
     }
 
     /**
@@ -458,7 +469,9 @@ public function testGenerateViewContainingRdfSequence()
         );
         $actualView = $mongo->selectCollection('tripod_php_testing','views')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_rdfsequence')));
 
-        $this->assertEquals($expectedView,$actualView);
+        $this->assertEquals($expectedView['_id'], $actualView['_id']);
+        $this->assertEquals($expectedView['value'], $actualView['value']);
+        $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']);
     }
 
     public function testGenerateViewWithTTL()
@@ -504,7 +517,9 @@ public function testGenerateViewWithTTL()
         );
         // get the view direct from mongo
         $actualView = \Tripod\Mongo\Config::getInstance()->getCollectionForView('tripod_php_testing', 'v_resource_full_ttl')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/3SplCtWGPqEyXcDiyhHQpA',"c"=>'http://talisaspire.com/',"type"=>'v_resource_full_ttl')));
-        $this->assertEquals($expectedView,$actualView);
+        $this->assertEquals($expectedView['_id'], $actualView['_id']);
+        $this->assertEquals($expectedView['value'], $actualView['value']);
+        $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']);
     }
 
     /**
@@ -609,7 +624,9 @@ public function testGenerateViewWithCountAggregate()
         );
 
         $actualView = \Tripod\Mongo\Config::getInstance()->getCollectionForView('tripod_php_testing', 'v_counts')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/works/4d101f63c10a6',"c"=>"http://talisaspire.com/","type"=>'v_counts')));
-        $this->assertEquals($expectedView,$actualView);
+        $this->assertEquals($expectedView['_id'], $actualView['_id']);
+        $this->assertEquals($expectedView['value'], $actualView['value']);
+        $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $actualView['_cts']);
     }
 
     public function testGetViewWithNamespaces()
@@ -2008,4 +2025,4 @@ public function testCursorNoExceptionThrownWhenCursorThrowsSomeExceptions()
 
         $mockTripodViews->getViewForResources(array($uri1),$viewType,$context);
     }
-}
\ No newline at end of file
+}

From 942b2687ad68f8ee358326321e56e2a97545691b Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 2 Nov 2017 14:44:07 -0400
Subject: [PATCH 12/42] Newer version of php-mongodb; use docker image

---
 .gitignore    | 1 +
 .travis.yml   | 7 ++-----
 composer.json | 2 +-
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index 3ee7cbe0..3f246e37 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 .idea
 composer.phar
+composer.lock
 output
 vendor
 tags
diff --git a/.travis.yml b/.travis.yml
index c2046a34..b1efb761 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -9,11 +9,8 @@ services:
 - redis
 - docker
 before_install:
-- wget https://raw.githubusercontent.com/docker-library/mongo/0ac2867637ef5989e4dc051efa0ae296010e58c9/2.6/Dockerfile
-- wget https://raw.githubusercontent.com/docker-library/mongo/0ac2867637ef5989e4dc051efa0ae296010e58c9/2.6/docker-entrypoint.sh
-- chmod 0777 docker-entrypoint.sh
-- docker build -t mongo/2.6.12 .
-- docker run -d -p 27017:27017 -v ~/data:/data/db mongo/2.6.12:latest
+- docker pull rossfsinger/mongo-2.6.12
+- docker run -d -p 27017:27017 -v ~/data:/data/db rossfsinger/mongo-2.6.12:latest
 - echo "extension = mongodb.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini
 - sleep 15
 install:
diff --git a/composer.json b/composer.json
index 5d941e4a..dab691ac 100644
--- a/composer.json
+++ b/composer.json
@@ -22,7 +22,7 @@
         "semsol/arc2": "v2.2.4",
         "chrisboulton/php-resque": "dev-master#98fde571db008a8b48e73022599d1d1c07d4a7b5",
         "monolog/monolog" : "~1.13",
-        "mongodb/mongodb": "^1.0.0"
+        "mongodb/mongodb": "1.0.4"
     },
     "require-dev": {
         "phpunit/phpunit": "4.1.*"

From f881cefe848a1aa053647dc2fe835c28275164d5 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 2 Nov 2017 14:51:56 -0400
Subject: [PATCH 13/42] Build tweaking

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index b1efb761..d5d86660 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -10,12 +10,12 @@ services:
 - docker
 before_install:
 - docker pull rossfsinger/mongo-2.6.12
-- docker run -d -p 27017:27017 -v ~/data:/data/db rossfsinger/mongo-2.6.12:latest
+- docker run -d -p 127.0.0.1:27017:27017 -v ~/data:/data/db rossfsinger/mongo-2.6.12:latest
 - echo "extension = mongodb.so" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini
 - sleep 15
 install:
 - composer install
-script: ant
+# script: ant
 notifications:
   hipchat:
     rooms:

From 032c7cb7375df2ac9285ac33c5c951ed158fe23b Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 2 Nov 2017 14:55:17 -0400
Subject: [PATCH 14/42] Just use ant

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index d5d86660..3a44cd74 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,7 +15,7 @@ before_install:
 - sleep 15
 install:
 - composer install
-# script: ant
+script: ant
 notifications:
   hipchat:
     rooms:

From 399036d92c64899491ba54f8c8d5135e3911b64c Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 2 Nov 2017 15:33:36 -0400
Subject: [PATCH 15/42] Left in old assertion

---
 test/unit/mongo/MongoTripodViewsTest.php | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/unit/mongo/MongoTripodViewsTest.php b/test/unit/mongo/MongoTripodViewsTest.php
index 216bfa93..e54ae576 100644
--- a/test/unit/mongo/MongoTripodViewsTest.php
+++ b/test/unit/mongo/MongoTripodViewsTest.php
@@ -390,7 +390,6 @@ public function testGenerateViewCorrectlyAfterUpdateAffectsFilter()
         );
 
         $updatedView = $mongo->selectCollection('tripod_php_testing','views')->findOne(array('_id'=>array("r"=>'http://talisaspire.com/resources/filter1',"c"=>'http://talisaspire.com/',"type"=>'v_resource_filter1')));
-        $this->assertEquals($expectedUpdatedView,$updatedView);
         $this->assertEquals($expectedUpdatedView['_id'], $updatedView['_id']);
         $this->assertEquals($expectedUpdatedView['value'], $updatedView['value']);
         $this->assertInstanceOf('\MongoDB\BSON\UTCDateTime', $updatedView['_cts']);

From a54a861b5d21ee33502a4db42458481ae3fe4b08 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 2 Nov 2017 15:52:58 -0400
Subject: [PATCH 16/42] Use helper method

---
 src/mongo/delegates/SearchDocuments.class.php | 2 +-
 src/mongo/delegates/Tables.class.php          | 6 +++---
 src/mongo/delegates/Views.class.php           | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/mongo/delegates/SearchDocuments.class.php b/src/mongo/delegates/SearchDocuments.class.php
index 716142be..fa78c9a3 100644
--- a/src/mongo/delegates/SearchDocuments.class.php
+++ b/src/mongo/delegates/SearchDocuments.class.php
@@ -111,7 +111,7 @@ public function generateSearchDocumentBasedOnSpecId($specId, $resource, $context
         $this->debugLog("Processing {$specId}");
 
         // build the document
-        $generatedDocument = [\_CREATED_TS => new \MongoDB\BSON\UTCDateTime()];
+        $generatedDocument = [\_CREATED_TS => \Tripod\Mongo\DateUtil::getMongoDate()];
         $this->addIdToImpactIndex($_id, $generatedDocument);
 
         $_id['type'] = $specId;
diff --git a/src/mongo/delegates/Tables.class.php b/src/mongo/delegates/Tables.class.php
index d19fd607..eef4e384 100644
--- a/src/mongo/delegates/Tables.class.php
+++ b/src/mongo/delegates/Tables.class.php
@@ -373,7 +373,7 @@ public function deleteTableRowsByTableId($tableId, $timestamp = null) {
         $query = ['_id.type' => $tableId];
         if ($timestamp) {
             if (!($timestamp instanceof \MongoDB\BSON\UTCDateTime)) {
-                $timestamp = new \MongoDB\BSON\UTCDateTime($timestamp);
+                $timestamp = \Tripod\Mongo\DateUtil::getMongoDate($timestamp);
             }
             $query[\_CREATED_TS] = [
                 '$or' => [
@@ -562,10 +562,10 @@ public function generateTableRows($tableType, $resource = null, $context = null,
                         _ID_CONTEXT => $doc['_id'][_ID_CONTEXT],
                         _ID_TYPE=>$tableSpec['_id']
                     ],
-                    \_CREATED_TS => new \MongoDB\BSON\UTCDateTime()
+                    \_CREATED_TS => \Tripod\Mongo\DateUtil::getMongoDate()
                 ];
                 // everything must go in the value object todo: this is a hang over from map reduce days, engineer out once we have stability on new PHP method for M/R
-                $value = array('_id'=>$doc['_id'], '_cts' => new \MongoDB\BSON\UTCDateTime());
+                $value = ['_id' => $doc['_id']];
                 $this->addIdToImpactIndex($doc['_id'], $value); // need to add the doc to the impact index to be consistent with views/search etc. this is needed for discovering impacted operations
                 $this->addFields($doc,$tableSpec,$value);
                 if (isset($tableSpec['joins'])) {
diff --git a/src/mongo/delegates/Views.class.php b/src/mongo/delegates/Views.class.php
index c5635e57..95248985 100644
--- a/src/mongo/delegates/Views.class.php
+++ b/src/mongo/delegates/Views.class.php
@@ -378,7 +378,7 @@ public function deleteViewsByViewId($viewId, $timestamp = null)
         $query = ['_id.type' => $viewId];
         if ($timestamp) {
             if (!($timestamp instanceof \MongoDB\BSON\UTCDateTime)) {
-                $timestamp = new \MongoDB\BSON\UTCDateTime($timestamp);
+                $timestamp = \Tripod\Mongo\DateUtil::getMongoDate($timestamp);
             }
             $query[\_CREATED_TS] = [
                 '$or' => [
@@ -462,7 +462,7 @@ public function generateView($viewId,$resource=null,$context=null,$queueName=nul
                             _ID_CONTEXT => $doc['_id'][_ID_CONTEXT],
                             _ID_TYPE=>$viewSpec['_id']
                         ],
-                        \_CREATED_TS => new \MongoDB\BSON\UTCDateTime()
+                        \_CREATED_TS => \Tripod\Mongo\DateUtil::getMongoDate()
                     ];
                     $value = array(); // everything must go in the value object todo: this is a hang over from map reduce days, engineer out once we have stability on new PHP method for M/R
 

From cdb54fe22149574a8999f4115d8c69ea727ddfc0 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Fri, 3 Nov 2017 13:58:19 -0400
Subject: [PATCH 17/42] Add count methods

---
 src/mongo/delegates/Tables.class.php              | 13 +++++++++++++
 src/mongo/delegates/Views.class.php               | 12 ++++++++++++
 src/mongo/providers/MongoSearchProvider.class.php | 13 +++++++++++++
 3 files changed, 38 insertions(+)

diff --git a/src/mongo/delegates/Tables.class.php b/src/mongo/delegates/Tables.class.php
index eef4e384..b43e9b55 100644
--- a/src/mongo/delegates/Tables.class.php
+++ b/src/mongo/delegates/Tables.class.php
@@ -1441,4 +1441,17 @@ private function applyRegexToValue($regex, $value)
             throw new \Tripod\Exceptions\Exception("Was expecting either VALUE_URI or VALUE_LITERAL when applying regex to value - possible data corruption with: ".var_export($value,true));
         }
     }
+
+    /**
+     * Count the number of documents in the spec that match $filters
+     *
+     * @param string $tableSpec Table spec ID
+     * @param array  $filters   Query filters to get count on
+     * @return integer
+     */
+    public function count($tableSpec, array $filters = [])
+    {
+        $filters['_id.type'] = $tableSpec;
+        return $this->config->getCollectionForTable($this->storeName, $tableSpec)->count($filters);
+    }
 }
diff --git a/src/mongo/delegates/Views.class.php b/src/mongo/delegates/Views.class.php
index 95248985..31d167bf 100644
--- a/src/mongo/delegates/Views.class.php
+++ b/src/mongo/delegates/Views.class.php
@@ -813,4 +813,16 @@ protected function getCollectionForViewSpec($viewSpecId)
         return $this->getConfigInstance()->getCollectionForView($this->storeName, $viewSpecId);
     }
 
+    /**
+     * Count the number of documents in the spec that match $filters
+     *
+     * @param string $viewSpec View spec ID
+     * @param array  $filters  Query filters to get count on
+     * @return integer
+     */
+    public function count($viewSpec, array $filters = [])
+    {
+        $filters['_id.type'] = $viewSpec;
+        return $this->getCollectionForViewSpec($viewSpec)->count($filters);
+    }
 }
diff --git a/src/mongo/providers/MongoSearchProvider.class.php b/src/mongo/providers/MongoSearchProvider.class.php
index f2c6916d..a6c9f32e 100644
--- a/src/mongo/providers/MongoSearchProvider.class.php
+++ b/src/mongo/providers/MongoSearchProvider.class.php
@@ -372,4 +372,17 @@ protected function getSearchDocumentSpecification($typeId)
     {
         return Config::getInstance()->getSearchDocumentSpecification($this->storeName, $typeId);
     }
+
+    /**
+     * Count the number of documents in the spec that match $filters
+     *
+     * @param string $searchSpec Search spec ID
+     * @param array  $filters    Query filters to get count on
+     * @return integer
+     */
+    public function count($searchSpec, array $filters = [])
+    {
+        $filters['_id.type'] = $searchSpec;
+        return $this->config->getCollectionForSearchDocument($this->storeName, $searchSpec)->count($filters);
+    }
 }

From e3ae8a575b7e337d29abc7023b37542d123051e6 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Fri, 3 Nov 2017 14:43:31 -0400
Subject: [PATCH 18/42] Add indexes

---
 src/mongo/util/IndexUtils.class.php | 33 ++++++++++++++++-------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/src/mongo/util/IndexUtils.class.php b/src/mongo/util/IndexUtils.class.php
index 523cfec2..8e9999f2 100644
--- a/src/mongo/util/IndexUtils.class.php
+++ b/src/mongo/util/IndexUtils.class.php
@@ -68,11 +68,12 @@ public function ensureIndexes($reindex=false,$storeName=null,$background=true)
                 $collection = $config->getCollectionForView($storeName, $viewId);
                 if($collection)
                 {
-                    $indexes = array(
-                        array(_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1, _ID_KEY.'.'._ID_TYPE => 1),
-                        array(_ID_KEY.'.'._ID_TYPE => 1),
-                        array('value.'._IMPACT_INDEX => 1)
-                    );
+                    $indexes = [
+                        [_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1, _ID_KEY.'.'._ID_TYPE => 1],
+                        [_ID_KEY.'.'._ID_TYPE => 1],
+                        ['value.'._IMPACT_INDEX => 1],
+                        [\_CREATED_TS => 1]
+                    ];
                     if(isset($spec['ensureIndexes']))
                     {
                         $indexes = array_merge($indexes, $spec['ensureIndexes']);
@@ -99,11 +100,12 @@ public function ensureIndexes($reindex=false,$storeName=null,$background=true)
                 $collection = $config->getCollectionForTable($storeName, $tableId);
                 if($collection)
                 {
-                    $indexes = array(
-                        array(_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1, _ID_KEY.'.'._ID_TYPE => 1),
-                        array(_ID_KEY.'.'._ID_TYPE => 1),
-                        array('value.'._IMPACT_INDEX => 1)
-                    );
+                    $indexes = [
+                        [_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1, _ID_KEY.'.'._ID_TYPE => 1],
+                        [_ID_KEY.'.'._ID_TYPE => 1],
+                        ['value.'._IMPACT_INDEX => 1],
+                        [\_CREATED_TS => 1]
+                    ];
                     if(isset($spec['ensureIndexes']))
                     {
                         $indexes = array_merge($indexes, $spec['ensureIndexes']);
@@ -130,11 +132,12 @@ public function ensureIndexes($reindex=false,$storeName=null,$background=true)
                 $collection = $config->getCollectionForSearchDocument($storeName, $searchId);
                 if($collection)
                 {
-                    $indexes = array(
-                        array(_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1),
-                        array(_ID_KEY.'.'._ID_TYPE => 1),
-                        array(_IMPACT_INDEX => 1)
-                    );
+                    $indexes = [
+                        [_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1],
+                        [_ID_KEY.'.'._ID_TYPE => 1],
+                        [_IMPACT_INDEX => 1],
+                        [\_CREATED_TS => 1]
+                    ];
 
                     if($reindex)
                     {

From 23dd4c747db0a7c64a656dd2de7e68a342185658 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Mon, 20 Nov 2017 14:39:28 -0500
Subject: [PATCH 19/42] _id key was missing from query

---
 src/mongo/delegates/Views.class.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mongo/delegates/Views.class.php b/src/mongo/delegates/Views.class.php
index 31d167bf..8ff4509f 100644
--- a/src/mongo/delegates/Views.class.php
+++ b/src/mongo/delegates/Views.class.php
@@ -483,7 +483,7 @@ public function generateView($viewId,$resource=null,$context=null,$queueName=nul
 
                     $generatedView['value'] = $value;
 
-                    $collection->replaceOne($generatedView['_id'], $generatedView, ['upsert' => true]);
+                    $collection->replaceOne(['_id' => $generatedView['_id']], $generatedView, ['upsert' => true]);
                 }
             }
 

From 3fb4a4a44b51a1d0db38d0d3fd29080b0fdea4a8 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Mon, 20 Nov 2017 14:58:14 -0500
Subject: [PATCH 20/42] Add expectation for _cts on indexes

---
 test/unit/mongo/IndexUtilsTest.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/unit/mongo/IndexUtilsTest.php b/test/unit/mongo/IndexUtilsTest.php
index bbce2432..e8c87cf9 100644
--- a/test/unit/mongo/IndexUtilsTest.php
+++ b/test/unit/mongo/IndexUtilsTest.php
@@ -526,7 +526,7 @@ protected function setConfigForViewIndexes($mockConfig)
                     array(
                         "_id" => "v_testview",
                         "ensureIndexes" => array(
-                            array("rdf:type.u"=>1)
+                            array("rdf:type.u"=>1, '_cts' => 1)
                         ),
                         "from" => "CBD_testing",
                         "type" => "temp:TestType",

From b854685b2e62bdb46bc16f9c25d9419a15c35723 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Mon, 20 Nov 2017 16:09:52 -0500
Subject: [PATCH 21/42] Add index expectations

---
 test/unit/mongo/IndexUtilsTest.php | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/test/unit/mongo/IndexUtilsTest.php b/test/unit/mongo/IndexUtilsTest.php
index e8c87cf9..b26499a0 100644
--- a/test/unit/mongo/IndexUtilsTest.php
+++ b/test/unit/mongo/IndexUtilsTest.php
@@ -410,13 +410,14 @@ protected function oneCustomAndThreeInternalTripodViewIndexesShouldBeCreated($mo
         // params that we know.
         // a) one custom index is created based on the view specification
         // b) three internal indexes are always created
-        $mockCollection->expects($this->exactly(4))
+        $mockCollection->expects($this->exactly(5))
             ->method('createIndex')
             ->withConsecutive(
                 array(array(_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1, _ID_KEY.'.'._ID_TYPE => 1), array('background'=>$background)),
                 array(array(_ID_KEY.'.'._ID_TYPE => 1), array('background' => $background)),
                 array(array('value.'._IMPACT_INDEX => 1), array('background' => $background)),
-                array(array('rdf:type.u' => 1), array('background' => $background))
+                [['_cts' => 1], ['background' => $background]],
+                [['rdf:type.u' => 1, '_cts' => 1], ['background' => $background]]
             );
     }
 
@@ -430,13 +431,14 @@ protected function oneCustomAndThreeInternalTripodTableIndexesShouldBeCreated($m
         // params that we know.
         // a) one custom index is created based on the view specification
         // b) three internal indexes are always created
-        $mockCollection->expects($this->exactly(4))
+        $mockCollection->expects($this->exactly(5))
             ->method('createIndex')
             ->withConsecutive(
                 array(array(_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1, _ID_KEY.'.'._ID_TYPE => 1), array('background'=>$background)),
                 array(array(_ID_KEY.'.'._ID_TYPE => 1), array('background' => $background)),
                 array(array('value.'._IMPACT_INDEX => 1), array('background' => $background)),
-                array(array('rdf:type.u' => 1), array('background'=>$background))
+                [['_cts' => 1], ['background' => $background]],
+                [['rdf:type.u' => 1], ['background' => $background]]
             );
     }
 
@@ -449,12 +451,13 @@ protected function threeInternalTripodSearchDocIndexesShouldBeCreated($mockColle
         // create index is called 3 times, each time with a different set of
         // params that we know.
         // for search docs only internal indexes are created
-        $mockCollection->expects($this->exactly(3))
+        $mockCollection->expects($this->exactly(4))
             ->method('createIndex')
             ->withConsecutive(
                 array(array(_ID_KEY.'.'._ID_RESOURCE => 1, _ID_KEY.'.'._ID_CONTEXT => 1), array('background' => $background)),
                 array(array(_ID_KEY.'.'._ID_TYPE => 1), array('background' => $background)),
-                array(array(_IMPACT_INDEX => 1), array('background' => $background))
+                array(array(_IMPACT_INDEX => 1), array('background' => $background)),
+                [['_cts' => 1], ['background' => $background]]
             );
 
     }

From 5b98fc25d4ab1e8a1a1eb1ee2f8d9d4c56df3439 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Tue, 28 Nov 2017 15:38:00 -0500
Subject: [PATCH 22/42] Flip $or query around

---
 src/mongo/delegates/Tables.class.php              | 8 +++-----
 src/mongo/delegates/Views.class.php               | 8 +++-----
 src/mongo/providers/MongoSearchProvider.class.php | 8 +++-----
 3 files changed, 9 insertions(+), 15 deletions(-)

diff --git a/src/mongo/delegates/Tables.class.php b/src/mongo/delegates/Tables.class.php
index b43e9b55..4016f43a 100644
--- a/src/mongo/delegates/Tables.class.php
+++ b/src/mongo/delegates/Tables.class.php
@@ -375,11 +375,9 @@ public function deleteTableRowsByTableId($tableId, $timestamp = null) {
             if (!($timestamp instanceof \MongoDB\BSON\UTCDateTime)) {
                 $timestamp = \Tripod\Mongo\DateUtil::getMongoDate($timestamp);
             }
-            $query[\_CREATED_TS] = [
-                '$or' => [
-                    ['$lt' => $timestamp],
-                    ['$exists' => false]
-                ]
+            $query['$or'] = [
+                [\_CREATED_TS => ['$lt' => $timestamp]],
+                [\_CREATED_TS => ['$exists' => false]]
             ];
         }
         $this->config->getCollectionForTable($this->storeName, $tableId)
diff --git a/src/mongo/delegates/Views.class.php b/src/mongo/delegates/Views.class.php
index 8ff4509f..bbaf44b5 100644
--- a/src/mongo/delegates/Views.class.php
+++ b/src/mongo/delegates/Views.class.php
@@ -380,11 +380,9 @@ public function deleteViewsByViewId($viewId, $timestamp = null)
             if (!($timestamp instanceof \MongoDB\BSON\UTCDateTime)) {
                 $timestamp = \Tripod\Mongo\DateUtil::getMongoDate($timestamp);
             }
-            $query[\_CREATED_TS] = [
-                '$or' => [
-                    ['$lt' => $timestamp],
-                    ['$exists' => false]
-                ]
+            $query['$or'] = [
+                [\_CREATED_TS => ['$lt' => $timestamp]],
+                [\_CREATED_TS => ['$exists' => false]]
             ];
         }
         $this->config->getCollectionForView($this->storeName, $viewId)
diff --git a/src/mongo/providers/MongoSearchProvider.class.php b/src/mongo/providers/MongoSearchProvider.class.php
index a6c9f32e..d08950b3 100644
--- a/src/mongo/providers/MongoSearchProvider.class.php
+++ b/src/mongo/providers/MongoSearchProvider.class.php
@@ -352,11 +352,9 @@ public function deleteSearchDocumentsByTypeId($typeId, $timestamp = null)
             if (!($timestamp instanceof \MongoDB\BSON\UTCDateTime)) {
                 $timestamp = new \MongoDB\BSON\UTCDateTime($timestamp);
             }
-            $query[\_CREATED_TS] = [
-                '$or' => [
-                    ['$lt' => $timestamp],
-                    ['$exists' => false]
-                ]
+            $query['$or'] = [
+                [\_CREATED_TS => ['$lt' => $timestamp]],
+                [\_CREATED_TS => ['$exists' => false]]
             ];
         }
         return $this->config->getCollectionForSearchDocument($this->storeName, $typeId)

From 2858ef65554306dd7beb9b90d62823512bb682df Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Wed, 29 Nov 2017 15:11:57 -0500
Subject: [PATCH 23/42] Add tracking identifier for batch jobs

---
 src/mongo/delegates/Tables.class.php    |  21 +++-
 src/mongo/delegates/Views.class.php     | 154 +++++++++++++-----------
 src/mongo/jobs/ApplyOperation.class.php |  13 +-
 3 files changed, 107 insertions(+), 81 deletions(-)

diff --git a/src/mongo/delegates/Tables.class.php b/src/mongo/delegates/Tables.class.php
index 4016f43a..b4768026 100644
--- a/src/mongo/delegates/Tables.class.php
+++ b/src/mongo/delegates/Tables.class.php
@@ -495,7 +495,7 @@ public function generateTableRowsForType($rdfType,$subject=null,$context=null, $
      * @param string|null $resource
      * @param string|null $context
      * @param string|null $queueName Queue for background bulk generation
-     * @return null //@todo: this should be a bool
+     * @return array
      */
     public function generateTableRows($tableType, $resource = null, $context = null, $queueName = null)
     {
@@ -535,7 +535,14 @@ public function generateTableRows($tableType, $resource = null, $context = null,
             'maxTimeMS' => 1000000
         ));
 
+        $jobOptions = [];
+        if ($queueName && !$resource && ($this->stat || !empty($this->statsConfig))) {
+            $jobOptions['statsConfig'] = $this->getStatsConfig();
+            $jobOptions[ApplyOperation::TRACKING_KEY] = \uniqid();
+        }
+        $count = 0;
         foreach ($docs as $doc) {
+            $count++;
             if ($queueName && !$resource) {
                 $subject = new ImpactedSubject(
                     $doc['_id'],
@@ -544,10 +551,8 @@ public function generateTableRows($tableType, $resource = null, $context = null,
                     $from,
                     array($tableType)
                 );
-
-                $jobOptions = array();
-
-                if($this->stat || !empty($this->statsConfig)) {
+                if (empty($jobOptions))
+                if ($this->stat || !empty($this->statsConfig)) {
                     $jobOptions['statsConfig'] = $this->getStatsConfig();
                 }
 
@@ -591,6 +596,12 @@ public function generateTableRows($tableType, $resource = null, $context = null,
             'filter'=>$filter,
             'from'=>$from));
         $this->getStat()->timer(MONGO_CREATE_TABLE.".$tableType",$t->result());
+
+        $stat = ['count' => $count];
+        if (isset($jobOptions[ApplyOperation::TRACKING_KEY])) {
+            $stat[ApplyOperation::TRACKING_KEY] = $jobOptions[ApplyOperation::TRACKING_KEY];
+        }
+        return $stat;
     }
 
     /**
diff --git a/src/mongo/delegates/Views.class.php b/src/mongo/delegates/Views.class.php
index bbaf44b5..f49c2ff2 100644
--- a/src/mongo/delegates/Views.class.php
+++ b/src/mongo/delegates/Views.class.php
@@ -9,6 +9,7 @@
 use Tripod\Mongo\Labeller;
 use \MongoDB\Driver\ReadPreference;
 use \MongoDB\Collection;
+use Tripod\Mongo\Jobs\ApplyOperation;
 
 /**
  * Class Views
@@ -405,94 +406,101 @@ public function generateView($viewId,$resource=null,$context=null,$queueName=nul
         if ($viewSpec == null) {
             $this->debugLog("Could not find a view specification for $resource with viewId '$viewId'");
             return null;
-        } else {
-            $t = new \Tripod\Timer();
-            $t->start();
+        }
+        $t = new \Tripod\Timer();
+        $t->start();
 
-            $from = $this->getFromCollectionForViewSpec($viewSpec);
-            $collection = $this->config->getCollectionForView($this->storeName, $viewId);
+        $from = $this->getFromCollectionForViewSpec($viewSpec);
+        $collection = $this->config->getCollectionForView($this->storeName, $viewId);
 
-            if (!isset($viewSpec['joins'])) {
-                throw new \Tripod\Exceptions\ViewException('Could not find any joins in view specification - usecase better served with select()');
-            }
+        if (!isset($viewSpec['joins'])) {
+            throw new \Tripod\Exceptions\ViewException('Could not find any joins in view specification - usecase better served with select()');
+        }
 
-            $types = array(); // this is used to filter the CBD table to speed up the view creation
-            if (is_array($viewSpec["type"])) {
-                foreach ($viewSpec["type"] as $type) {
-                    $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($type));
-                    $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($type));
-                }
-            } else {
-                $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($viewSpec["type"]));
-                $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($viewSpec["type"]));
-            }
-            $filter = array('$or'=> $types);
-            if (isset($resource)) {
-                $resourceAlias = $this->labeller->uri_to_alias($resource);
-                $filter["_id"] = array(_ID_RESOURCE=>$resourceAlias,_ID_CONTEXT=>$contextAlias);
+        $types = array(); // this is used to filter the CBD table to speed up the view creation
+        if (is_array($viewSpec["type"])) {
+            foreach ($viewSpec["type"] as $type) {
+                $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($type));
+                $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($type));
             }
+        } else {
+            $types[] = array("rdf:type.u"=>$this->labeller->qname_to_alias($viewSpec["type"]));
+            $types[] = array("rdf:type.u"=>$this->labeller->uri_to_alias($viewSpec["type"]));
+        }
+        $filter = array('$or'=> $types);
+        if (isset($resource)) {
+            $resourceAlias = $this->labeller->uri_to_alias($resource);
+            $filter["_id"] = array(_ID_RESOURCE=>$resourceAlias,_ID_CONTEXT=>$contextAlias);
+        }
 
-            $docs = $this->config->getCollectionForCBD($this->storeName, $from)->find($filter, array(
-                'maxTimeMS' => \Tripod\Mongo\Config::getInstance()->getMongoCursorTimeout()
-            ));
-
-            foreach ($docs as $doc) {
-                if ($queueName && !$resource) {
-                    $subject = new ImpactedSubject(
-                        $doc['_id'],
-                        OP_VIEWS,
-                        $this->storeName,
-                        $from,
-                        array($viewId)
-                    );
+        $docs = $this->config->getCollectionForCBD($this->storeName, $from)->find($filter, array(
+            'maxTimeMS' => \Tripod\Mongo\Config::getInstance()->getMongoCursorTimeout()
+        ));
 
-                    $jobOptions = array();
-                    if ($this->stat || !empty($this->statsConfig)) {
-                        $jobOptions['statsConfig'] = $this->getStatsConfig();
-                    }
+        $jobOptions = [];
+        if ($queueName && !$resource && ($this->stat || !empty($this->statsConfig))) {
+            $jobOptions['statsConfig'] = $this->getStatsConfig();
+            $jobOptions[ApplyOperation::TRACKING_KEY] = \uniqid();
+        }
+        $count = 0;
+        foreach ($docs as $doc) {
+            $count++;
+            if ($queueName && !$resource) {
+                $subject = new ImpactedSubject(
+                    $doc['_id'],
+                    OP_VIEWS,
+                    $this->storeName,
+                    $from,
+                    array($viewId)
+                );
 
-                    $this->getApplyOperation()->createJob(array($subject), $queueName, $jobOptions);
+                $this->getApplyOperation()->createJob(array($subject), $queueName, $jobOptions);
+            } else {
+                // Set up view meta information
+                $generatedView = [
+                    '_id' => [
+                        _ID_RESOURCE => $doc['_id'][_ID_RESOURCE],
+                        _ID_CONTEXT => $doc['_id'][_ID_CONTEXT],
+                        _ID_TYPE=>$viewSpec['_id']
+                    ],
+                    \_CREATED_TS => \Tripod\Mongo\DateUtil::getMongoDate()
+                ];
+                $value = array(); // everything must go in the value object todo: this is a hang over from map reduce days, engineer out once we have stability on new PHP method for M/R
+
+                $value[_GRAPHS] = array();
+
+                $buildImpactIndex=true;
+                if (isset($viewSpec['ttl'])) {
+                    $buildImpactIndex=false;
+                    $value[_EXPIRES] = \Tripod\Mongo\DateUtil::getMongoDate($this->getExpirySecFromNow($viewSpec['ttl']) * 1000);
                 } else {
-                    // Set up view meta information
-                    $generatedView = [
-                        '_id' => [
-                            _ID_RESOURCE => $doc['_id'][_ID_RESOURCE],
-                            _ID_CONTEXT => $doc['_id'][_ID_CONTEXT],
-                            _ID_TYPE=>$viewSpec['_id']
-                        ],
-                        \_CREATED_TS => \Tripod\Mongo\DateUtil::getMongoDate()
-                    ];
-                    $value = array(); // everything must go in the value object todo: this is a hang over from map reduce days, engineer out once we have stability on new PHP method for M/R
-
-                    $value[_GRAPHS] = array();
-
-                    $buildImpactIndex=true;
-                    if (isset($viewSpec['ttl'])) {
-                        $buildImpactIndex=false;
-                        $value[_EXPIRES] = \Tripod\Mongo\DateUtil::getMongoDate($this->getExpirySecFromNow($viewSpec['ttl']) * 1000);
-                    } else {
-                        $value[_IMPACT_INDEX] = array($doc['_id']);
-                    }
+                    $value[_IMPACT_INDEX] = array($doc['_id']);
+                }
 
-                    $this->doJoins($doc,$viewSpec['joins'],$value,$from,$contextAlias,$buildImpactIndex);
+                $this->doJoins($doc,$viewSpec['joins'],$value,$from,$contextAlias,$buildImpactIndex);
 
-                    // add top level properties
-                    $value[_GRAPHS][] = $this->extractProperties($doc,$viewSpec,$from);
+                // add top level properties
+                $value[_GRAPHS][] = $this->extractProperties($doc,$viewSpec,$from);
 
-                    $generatedView['value'] = $value;
+                $generatedView['value'] = $value;
 
-                    $collection->replaceOne(['_id' => $generatedView['_id']], $generatedView, ['upsert' => true]);
-                }
+                $collection->replaceOne(['_id' => $generatedView['_id']], $generatedView, ['upsert' => true]);
             }
+        }
 
-            $t->stop();
-            $this->timingLog(MONGO_CREATE_VIEW, array(
-                'view'=>$viewSpec['type'],
-                'duration'=>$t->result(),
-                'filter'=>$filter,
-                'from'=>$from));
-            $this->getStat()->timer(MONGO_CREATE_VIEW.".$viewId",$t->result());
+        $t->stop();
+        $this->timingLog(MONGO_CREATE_VIEW, array(
+            'view'=>$viewSpec['type'],
+            'duration'=>$t->result(),
+            'filter'=>$filter,
+            'from'=>$from));
+        $this->getStat()->timer(MONGO_CREATE_VIEW.".$viewId",$t->result());
+
+        $stat = ['count' => $count];
+        if (isset($jobOptions[ApplyOperation::TRACKING_KEY])) {
+            $stat[ApplyOperation::TRACKING_KEY] = $jobOptions[ApplyOperation::TRACKING_KEY];
         }
+        return $stat;
     }
 
     /**
diff --git a/src/mongo/jobs/ApplyOperation.class.php b/src/mongo/jobs/ApplyOperation.class.php
index eee50390..a2eb0f74 100644
--- a/src/mongo/jobs/ApplyOperation.class.php
+++ b/src/mongo/jobs/ApplyOperation.class.php
@@ -9,6 +9,8 @@
 class ApplyOperation extends JobBase {
 
     const SUBJECTS_KEY = 'subjects';
+    const TRACKING_KEY = 'batchId';
+
     /**
      * Run the ApplyOperation job
      * @throws \Exception
@@ -35,8 +37,7 @@ public function perform()
 
             $this->getStat()->increment(MONGO_QUEUE_APPLY_OPERATION_JOB . '.' . SUBJECT_COUNT, count($this->args[self::SUBJECTS_KEY]));
 
-            foreach($this->args[self::SUBJECTS_KEY] as $subject)
-            {
+            foreach ($this->args[self::SUBJECTS_KEY] as $subject) {
                 $opTimer = new \Tripod\Timer();
                 $opTimer->start();
 
@@ -53,6 +54,12 @@ public function perform()
             // stat time taken to process job, from time it was picked up
             $this->getStat()->timer(MONGO_QUEUE_APPLY_OPERATION_SUCCESS,$timer->result());
 
+            if (isset($this->args[self::TRACKING_KEY])) {
+                $this->getStat()->increment(
+                    \MONGO_QUEUE_APPLY_OPERATION . '.' . $subject['operation'] . '.' . $this->args[self::TRACKING_KEY]
+                );
+            }
+
             $this->debugLog("[JOBID " . $this->job->payload['id'] . "] ApplyOperation::perform() done in {$timer->result()}ms");
         }
         catch (\Exception $e)
@@ -100,7 +107,7 @@ protected function createImpactedSubject(array $args)
             $args["storeName"],
             $args["podName"],
             $args["specTypes"]
-        );        
+        );
     }
 
     /**

From 3242f54cfbe3a2c88dab4a85f4f69549cffc198e Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Wed, 29 Nov 2017 15:20:55 -0500
Subject: [PATCH 24/42] use ApplyOperation

---
 src/mongo/delegates/Tables.class.php | 7 ++++---
 src/mongo/delegates/Views.class.php  | 8 ++++----
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/mongo/delegates/Tables.class.php b/src/mongo/delegates/Tables.class.php
index b4768026..aa6ad5d5 100644
--- a/src/mongo/delegates/Tables.class.php
+++ b/src/mongo/delegates/Tables.class.php
@@ -5,9 +5,10 @@
 require_once TRIPOD_DIR . 'mongo/MongoTripodConstants.php';
 require_once TRIPOD_DIR . 'mongo/base/DriverBase.class.php';
 
-use Tripod\Mongo\Config;
-use Tripod\Mongo\ImpactedSubject;
-use Tripod\Mongo\Labeller;
+use \Tripod\Mongo\Jobs\ApplyOperation;
+use \Tripod\Mongo\Config;
+use \Tripod\Mongo\ImpactedSubject;
+use \Tripod\Mongo\Labeller;
 use \MongoDB\Driver\ReadPreference;
 use \MongoDB\Collection;
 
diff --git a/src/mongo/delegates/Views.class.php b/src/mongo/delegates/Views.class.php
index f49c2ff2..5e7cce7f 100644
--- a/src/mongo/delegates/Views.class.php
+++ b/src/mongo/delegates/Views.class.php
@@ -4,12 +4,12 @@
 
 require_once TRIPOD_DIR . 'mongo/base/DriverBase.class.php';
 
-use Tripod\Mongo\Config;
-use Tripod\Mongo\ImpactedSubject;
-use Tripod\Mongo\Labeller;
+use \Tripod\Mongo\Jobs\ApplyOperation;
+use \Tripod\Mongo\Config;
+use \Tripod\Mongo\ImpactedSubject;
+use \Tripod\Mongo\Labeller;
 use \MongoDB\Driver\ReadPreference;
 use \MongoDB\Collection;
-use Tripod\Mongo\Jobs\ApplyOperation;
 
 /**
  * Class Views

From 694c6763d37a199b67e75ed0d0962f1dda720b43 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 30 Nov 2017 12:25:46 -0500
Subject: [PATCH 25/42] Avoid stat class collisions

---
 src/mongo/MongoTripodConstants.php      | 2 ++
 src/mongo/jobs/ApplyOperation.class.php | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/mongo/MongoTripodConstants.php b/src/mongo/MongoTripodConstants.php
index 4725e5aa..3cf026b2 100644
--- a/src/mongo/MongoTripodConstants.php
+++ b/src/mongo/MongoTripodConstants.php
@@ -97,6 +97,8 @@
 define('STAT_CLASS', 'tripod');
 define('STAT_PIVOT_FIELD', 'group_by_db');
 
+define('BATCH_TRACKING_GROUP', 'BATCH_TRACKING_GROUP');
+
 //Audit types, statuses
 define('AUDIT_TYPE_REMOVE_INERT_LOCKS', 'REMOVE_INERT_LOCKS');
 define('AUDIT_STATUS_IN_PROGRESS', 'IN_PROGRESS');
diff --git a/src/mongo/jobs/ApplyOperation.class.php b/src/mongo/jobs/ApplyOperation.class.php
index a2eb0f74..92cce5a0 100644
--- a/src/mongo/jobs/ApplyOperation.class.php
+++ b/src/mongo/jobs/ApplyOperation.class.php
@@ -56,7 +56,7 @@ public function perform()
 
             if (isset($this->args[self::TRACKING_KEY])) {
                 $this->getStat()->increment(
-                    \MONGO_QUEUE_APPLY_OPERATION . '.' . $subject['operation'] . '.' . $this->args[self::TRACKING_KEY]
+                    BATCH_TRACKING_GROUP . '.' . $subject['operation'] . '.' . $this->args[self::TRACKING_KEY]
                 );
             }
 

From 2f380f2c1a025d3aea0e42c77766d96862dd6373 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Mon, 4 Dec 2017 16:51:52 -0500
Subject: [PATCH 26/42] Track bulk job progress in Tripod and cleanup

---
 src/mongo/Config.class.php                    | 13 ++++
 src/mongo/JobGroup.php                        | 63 +++++++++++++++++++
 src/mongo/MongoTripodConstants.php            |  1 +
 src/mongo/delegates/Tables.class.php          | 14 +++--
 src/mongo/delegates/Views.class.php           | 17 +++--
 src/mongo/jobs/ApplyOperation.class.php       | 40 +++++++++---
 .../providers/MongoSearchProvider.class.php   |  5 +-
 7 files changed, 132 insertions(+), 21 deletions(-)
 create mode 100644 src/mongo/JobGroup.php

diff --git a/src/mongo/Config.class.php b/src/mongo/Config.class.php
index fb0975ce..0de41cbb 100644
--- a/src/mongo/Config.class.php
+++ b/src/mongo/Config.class.php
@@ -2054,6 +2054,19 @@ public function getCollectionForManualRollbackAudit($storeName, $readPreference
         );
     }
 
+    /**
+     * @param string $storeName
+     * @param string $readPreference
+     * @return Collection
+     */
+    public function getCollectionForJobGroups($storeName, $readPreference = ReadPreference::RP_PRIMARY_PREFERRED)
+    {
+        return $this->getMongoCollection(
+            $this->getDatabase($storeName, $this->dbConfig[$storeName]['data_source'], $readPreference),
+            OPERATION_GROUPS_COLLECTION
+        );
+    }
+
     /**
      * @param $readPreference
      * @return Database
diff --git a/src/mongo/JobGroup.php b/src/mongo/JobGroup.php
new file mode 100644
index 00000000..5e525bdf
--- /dev/null
+++ b/src/mongo/JobGroup.php
@@ -0,0 +1,63 @@
+<?php
+
+namespace Tripod\Mongo;
+
+class JobGroup
+{
+    private $id;
+    private $collection;
+    private $storeName;
+
+    public function __construct($storeName, $groupId = null)
+    {
+        $this->storeName = $storeName;
+        if (!$groupId) {
+            $groupId = new \MongoDB\BSON\ObjectId();
+        } elseif (!$groupId instanceof \MongoDB\BSON\ObjectId) {
+            $groupId = new \MongoDB\BSON\ObjectId($groupId);
+        }
+        $this->id = $groupId;
+    }
+
+    public function setJobCount($count)
+    {
+        $this->getMongoCollection()->updateOne(
+            ['_id' => $this->id],
+            ['$set' => ['count' => $count]],
+            ['upsert' => true]
+        );
+    }
+
+    public function incrementJobCount($inc = 1)
+    {
+        $updateResult = $this->getMongoCollection()->findOneAndUpdate(
+            ['_id' => $this->id],
+            ['$inc' => ['count' => $inc]],
+            ['upsert' => true, 'returnDocument' => MongoDB\Operation\FindOneAndUpdate::RETURN_DOCUMENT_AFTER]
+        );
+        return $updateResult['count'];
+    }
+
+    /**
+     * @return \MongoDB\BSON\ObjectId
+     */
+    public function getId()
+    {
+        return $this->id;
+    }
+
+    /**
+     * For mocking
+     *
+     * @return \MongoDB\Collection
+     */
+    protected function getMongoCollection()
+    {
+        if (!isset($this->collection)) {
+            $config = Config::getInstance();
+
+            $this->collection = $config->getCollectionForJobGroups($this->storeName);
+        }
+        return $this->collection;
+    }
+}
diff --git a/src/mongo/MongoTripodConstants.php b/src/mongo/MongoTripodConstants.php
index 3cf026b2..54ed42b4 100644
--- a/src/mongo/MongoTripodConstants.php
+++ b/src/mongo/MongoTripodConstants.php
@@ -6,6 +6,7 @@
 define('VIEWS_COLLECTION', 'views');
 define('LOCKS_COLLECTION', 'locks');
 define('AUDIT_MANUAL_ROLLBACKS_COLLECTION','audit_manual_rollbacks');
+define('OPERATION_GROUP_COLLECTION', 'job_groups');
 
 // search
 define('SEARCH_INDEX_COLLECTION', 'search');
diff --git a/src/mongo/delegates/Tables.class.php b/src/mongo/delegates/Tables.class.php
index aa6ad5d5..cc0c1935 100644
--- a/src/mongo/delegates/Tables.class.php
+++ b/src/mongo/delegates/Tables.class.php
@@ -361,6 +361,7 @@ protected function deleteTableRowsForResource($resource, $context=null, $specTyp
      * This method will delete all table rows where the _id.type matches the specified $tableId
      * @param string                         $tableId   Table spec ID
      * @param \MongoDB\BSON\UTCDateTime|null $timestamp Optional timestamp to delete all table rows that are older than
+     * @return integer                                  The number of table rows deleted
      */
     public function deleteTableRowsByTableId($tableId, $timestamp = null) {
         $t = new \Tripod\Timer();
@@ -381,11 +382,12 @@ public function deleteTableRowsByTableId($tableId, $timestamp = null) {
                 [\_CREATED_TS => ['$exists' => false]]
             ];
         }
-        $this->config->getCollectionForTable($this->storeName, $tableId)
+        $deleteResult = $this->config->getCollectionForTable($this->storeName, $tableId)
             ->deleteMany($query);
 
         $t->stop();
         $this->timingLog(MONGO_DELETE_TABLE_ROWS, array('duration'=>$t->result(), 'query'=>$query));
+        return $deleteResult->getDeletedCount();
     }
 
     /**
@@ -531,7 +533,8 @@ public function generateTableRows($tableType, $resource = null, $context = null,
         if (isset($resource)) {
             $filter["_id"] = array(_ID_RESOURCE=>$this->labeller->uri_to_alias($resource),_ID_CONTEXT=>$contextAlias);
         }
-
+        // @todo Change this to a command when we upgrade MongoDB to 1.1+
+        $count = $this->config->getCollectionForCBD($this->storeName, $from)->count($filter);
         $docs = $this->config->getCollectionForCBD($this->storeName, $from)->find($filter, array(
             'maxTimeMS' => 1000000
         ));
@@ -539,11 +542,12 @@ public function generateTableRows($tableType, $resource = null, $context = null,
         $jobOptions = [];
         if ($queueName && !$resource && ($this->stat || !empty($this->statsConfig))) {
             $jobOptions['statsConfig'] = $this->getStatsConfig();
-            $jobOptions[ApplyOperation::TRACKING_KEY] = \uniqid();
+            $jobGroup = new JobGroup($this->storeName);
+            $jobOptions[ApplyOperation::TRACKING_KEY] = $jobGroup->getId()->__toString();
+            $jobGroup->setJobCount($count);;
         }
-        $count = 0;
+
         foreach ($docs as $doc) {
-            $count++;
             if ($queueName && !$resource) {
                 $subject = new ImpactedSubject(
                     $doc['_id'],
diff --git a/src/mongo/delegates/Views.class.php b/src/mongo/delegates/Views.class.php
index 5e7cce7f..b866ae22 100644
--- a/src/mongo/delegates/Views.class.php
+++ b/src/mongo/delegates/Views.class.php
@@ -10,6 +10,7 @@
 use \Tripod\Mongo\Labeller;
 use \MongoDB\Driver\ReadPreference;
 use \MongoDB\Collection;
+use Tripod\Mongo\JobGroup;
 
 /**
  * Class Views
@@ -368,6 +369,7 @@ public function generateViewsForResourcesOfType($rdfType,$resource=null,$context
      * This method will delete all views where the _id.type of the viewmatches the specified $viewId
      * @param string                         $viewId    View spec ID
      * @param \MongoDB\BSON\UTCDateTime|null $timestamp Optional timestamp to delete all views that are older than
+     * @return integer                                  The number of views deleted
      */
     public function deleteViewsByViewId($viewId, $timestamp = null)
     {
@@ -386,8 +388,9 @@ public function deleteViewsByViewId($viewId, $timestamp = null)
                 [\_CREATED_TS => ['$exists' => false]]
             ];
         }
-        $this->config->getCollectionForView($this->storeName, $viewId)
+        $deleteResult = $this->config->getCollectionForView($this->storeName, $viewId)
             ->deleteMany($query);
+        return $deleteResult->getDeletedCount();
     }
 
     /**
@@ -433,18 +436,22 @@ public function generateView($viewId,$resource=null,$context=null,$queueName=nul
             $filter["_id"] = array(_ID_RESOURCE=>$resourceAlias,_ID_CONTEXT=>$contextAlias);
         }
 
+        // @todo Change this to a command when we upgrade MongoDB to 1.1+
+        $count = $this->config->getCollectionForCBD($this->storeName, $from)->count($filter);
         $docs = $this->config->getCollectionForCBD($this->storeName, $from)->find($filter, array(
             'maxTimeMS' => \Tripod\Mongo\Config::getInstance()->getMongoCursorTimeout()
         ));
 
+
+
         $jobOptions = [];
-        if ($queueName && !$resource && ($this->stat || !empty($this->statsConfig))) {
+        if ($queueName && !$resource) {
             $jobOptions['statsConfig'] = $this->getStatsConfig();
-            $jobOptions[ApplyOperation::TRACKING_KEY] = \uniqid();
+            $jobGroup = new JobGroup($this->storeName);
+            $jobOptions[ApplyOperation::TRACKING_KEY] = $jobGroup->getId()->__toString();
+            $jobGroup->setJobCount($count);
         }
-        $count = 0;
         foreach ($docs as $doc) {
-            $count++;
             if ($queueName && !$resource) {
                 $subject = new ImpactedSubject(
                     $doc['_id'],
diff --git a/src/mongo/jobs/ApplyOperation.class.php b/src/mongo/jobs/ApplyOperation.class.php
index 92cce5a0..c5351d95 100644
--- a/src/mongo/jobs/ApplyOperation.class.php
+++ b/src/mongo/jobs/ApplyOperation.class.php
@@ -2,6 +2,9 @@
 
 namespace Tripod\Mongo\Jobs;
 
+use Tripod\Mongo\JobGroup;
+
+
 /**
  * Class ApplyOperation
  * @package Tripod\Mongo\Jobs
@@ -17,8 +20,7 @@ class ApplyOperation extends JobBase {
      */
     public function perform()
     {
-        try
-        {
+        try {
             $this->debugLog("[JOBID " . $this->job->payload['id'] . "] ApplyOperation::perform() start");
 
             $timer = new \Tripod\Timer();
@@ -55,15 +57,35 @@ public function perform()
             $this->getStat()->timer(MONGO_QUEUE_APPLY_OPERATION_SUCCESS,$timer->result());
 
             if (isset($this->args[self::TRACKING_KEY])) {
-                $this->getStat()->increment(
-                    BATCH_TRACKING_GROUP . '.' . $subject['operation'] . '.' . $this->args[self::TRACKING_KEY]
-                );
+                $jobGroup = new JobGroup($this->args[self::TRACKING_KEY]);
+                $jobCount = $jobGroup->incrementJobCount(-1);
+                if ($jobCount <= 0) {
+                    // @todo Replace this with ObjectId->getTimestamp() if we upgrade Mongo driver to 1.2
+                    $timestamp = new \MongoDB\BSON\UTCDateTime(hexdec(substr($jobGroup->getId(), 0, 8)) * 1000);
+                    $tripod = $this->getTripod($this->args['storeName'], $this->args['podName']);
+                    $count = 0;
+                    foreach ($this->args['specId'] as $specId) {
+                        switch ($this->args['operation']) {
+                            case \OP_VIEWS:
+                                $count += $tripod->getTripodViews()->deleteViewsByViewId($specId, $timestamp);
+                                break;
+                            case \OP_TABLES:
+                                $count += $tripod->getTripodTables()->deleteTableRowsByTableId($specId, $timestamp);
+                                break;
+                            case \OP_SEARCH:
+                                $searchProvider = new \Tripod\Mongo\MongoSearchProvider($tripod);
+                                $count += $searchProvider->deleteSearchDocumentsByTypeId($specId, $timestamp);
+                                break;
+                        }
+                    }
+                    $this->infoLog(
+                        '[JobGroupId ' . $jobGroup->getId()->__toString() . '] composite cleanup for ' .
+                        $this->args['operation'] . ' removed ' . $count . ' stale composite documents'
+                    );
+                }
             }
-
             $this->debugLog("[JOBID " . $this->job->payload['id'] . "] ApplyOperation::perform() done in {$timer->result()}ms");
-        }
-        catch (\Exception $e)
-        {
+        } catch (\Exception $e) {
             $this->getStat()->increment(MONGO_QUEUE_APPLY_OPERATION_FAIL);
             $this->errorLog("Caught exception in ".get_class($this).": ".$e->getMessage());
             throw $e;
diff --git a/src/mongo/providers/MongoSearchProvider.class.php b/src/mongo/providers/MongoSearchProvider.class.php
index d08950b3..d379d172 100644
--- a/src/mongo/providers/MongoSearchProvider.class.php
+++ b/src/mongo/providers/MongoSearchProvider.class.php
@@ -338,7 +338,7 @@ public function getSearchCollectionName()
      * If type id is not specified this method will throw an exception.
      * @param string                         $typeId    Search type id
      * @param \MongoDB\BSON\UTCDateTime|null $timestamp Optional timestamp to delete all search docs that are older than
-     * @return bool|array  response returned by mongo
+     * @return integer                                  The number of search documnts deleted
      * @throws \Tripod\Exceptions\Exception if there was an error performing the operation
      */
     public function deleteSearchDocumentsByTypeId($typeId, $timestamp = null)
@@ -357,8 +357,9 @@ public function deleteSearchDocumentsByTypeId($typeId, $timestamp = null)
                 [\_CREATED_TS => ['$exists' => false]]
             ];
         }
-        return $this->config->getCollectionForSearchDocument($this->storeName, $typeId)
+        $deleteResponse = $this->config->getCollectionForSearchDocument($this->storeName, $typeId)
             ->deleteMany($query);
+        return $deleteResponse->getDeletedCount();
     }
 
     /**

From 13b5f6ba2615fc9d9d6e28101542a7ec776cf53d Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Tue, 5 Dec 2017 11:08:56 -0500
Subject: [PATCH 27/42] Add job group to searchindexer

---
 src/mongo/delegates/SearchIndexer.class.php | 40 ++++++++++++---------
 1 file changed, 23 insertions(+), 17 deletions(-)

diff --git a/src/mongo/delegates/SearchIndexer.class.php b/src/mongo/delegates/SearchIndexer.class.php
index 8d9eef92..232ac37a 100644
--- a/src/mongo/delegates/SearchIndexer.class.php
+++ b/src/mongo/delegates/SearchIndexer.class.php
@@ -10,6 +10,7 @@
 use Tripod\Mongo\Config;
 use Tripod\Mongo\ImpactedSubject;
 use Tripod\Mongo\Labeller;
+use Tripod\Mongo\Jobs\ApplyOperation;
 use \MongoDB\Driver\ReadPreference;
 use \MongoDB\Collection;
 
@@ -174,9 +175,9 @@ public function generateSearchDocuments($searchDocumentType, $resourceUri=null,
         // default the context
         $contextAlias = $this->getContextAlias($context);
         $spec = \Tripod\Mongo\Config::getInstance()->getSearchDocumentSpecification($this->getStoreName(), $searchDocumentType);
-        
+
         if($resourceUri)
-        {            
+        {
             $this->generateAndIndexSearchDocuments($resourceUri, $contextAlias, $spec['from'], $searchDocumentType);
             return;
         }
@@ -205,13 +206,20 @@ public function generateSearchDocuments($searchDocumentType, $resourceUri=null,
             $filter["_id"] = array(_ID_RESOURCE=>$this->labeller->uri_to_alias($resource),_ID_CONTEXT=>$contextAlias);
         }
 
+        $count = $this->config->getCollectionForCBD($this->getStoreName(), $from)->count($filter);
         $docs = $this->config->getCollectionForCBD($this->getStoreName(), $from)->find($filter, array(
             'maxTimeMS' => $this->config->getMongoCursorTimeout()
         ));
-        foreach ($docs as $doc)
-        {
-            if($queueName && !$resourceUri)
-            {
+
+        $jobOptions = [];
+        if ($queueName && !$resource) {
+            $jobOptions['statsConfig'] = $this->getStatsConfig();
+            $jobGroup = new JobGroup($this->storeName);
+            $jobOptions[ApplyOperation::TRACKING_KEY] = $jobGroup->getId()->__toString();
+            $jobGroup->setJobCount($count);
+        }
+        foreach ($docs as $doc) {
+            if ($queueName && !$resourceUri) {
                 $subject = new ImpactedSubject(
                     $doc['_id'],
                     OP_SEARCH,
@@ -219,17 +227,9 @@ public function generateSearchDocuments($searchDocumentType, $resourceUri=null,
                     $from,
                     array($searchDocumentType)
                 );
-                $jobOptions = array();
-
-                if($this->stat || !empty($this->statsConfig))
-                {
-                    $jobOptions['statsConfig'] = $this->getStatsConfig();
-                }
 
                 $this->getApplyOperation()->createJob(array($subject), $queueName, $jobOptions);
-            }
-            else
-            {
+            } else {
                 $this->generateAndIndexSearchDocuments(
                     $doc[_ID_KEY][_ID_RESOURCE],
                     $doc[_ID_KEY][_ID_CONTEXT],
@@ -246,6 +246,12 @@ public function generateSearchDocuments($searchDocumentType, $resourceUri=null,
             'filter'=>$filter,
             'from'=>$from));
         $this->getStat()->timer(MONGO_CREATE_SEARCH_DOC.".$searchDocumentType",$t->result());
+
+        $stat = ['count' => $count];
+        if (isset($jobOptions[ApplyOperation::TRACKING_KEY])) {
+            $stat[ApplyOperation::TRACKING_KEY] = $jobOptions[ApplyOperation::TRACKING_KEY];
+        }
+        return $stat;
     }
 
     /**
@@ -266,7 +272,7 @@ public function deleteSearchDocumentsByTypeId($typeId)
     {
     	return $this->getSearchProvider()->deleteSearchDocumentsByTypeId($typeId);
     }
-    
+
 
     /**
      * @return \Tripod\ISearchProvider
@@ -300,4 +306,4 @@ protected function deDupe(Array $input)
         }
         return $output;
     }
-}
\ No newline at end of file
+}

From 4eaecc8691fc1a83a5ba305e42ffd6a1bbb7e6cb Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Wed, 6 Dec 2017 08:43:20 -0500
Subject: [PATCH 28/42] Left out a use

---
 src/mongo/delegates/SearchIndexer.class.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/mongo/delegates/SearchIndexer.class.php b/src/mongo/delegates/SearchIndexer.class.php
index 232ac37a..73218a11 100644
--- a/src/mongo/delegates/SearchIndexer.class.php
+++ b/src/mongo/delegates/SearchIndexer.class.php
@@ -11,6 +11,7 @@
 use Tripod\Mongo\ImpactedSubject;
 use Tripod\Mongo\Labeller;
 use Tripod\Mongo\Jobs\ApplyOperation;
+use Tripod\Mongo\JobGroup;
 use \MongoDB\Driver\ReadPreference;
 use \MongoDB\Collection;
 

From ff43de117c9d4ce6dda34f12aab4fdff89dd3ebb Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Wed, 6 Dec 2017 08:49:07 -0500
Subject: [PATCH 29/42] We really need an autoloader

---
 src/mongo/delegates/Tables.class.php | 1 +
 src/tripod.inc.php                   | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/mongo/delegates/Tables.class.php b/src/mongo/delegates/Tables.class.php
index cc0c1935..8a2e83bf 100644
--- a/src/mongo/delegates/Tables.class.php
+++ b/src/mongo/delegates/Tables.class.php
@@ -9,6 +9,7 @@
 use \Tripod\Mongo\Config;
 use \Tripod\Mongo\ImpactedSubject;
 use \Tripod\Mongo\Labeller;
+use \Tripod\Mongo\JobGroup;
 use \MongoDB\Driver\ReadPreference;
 use \MongoDB\Collection;
 
diff --git a/src/tripod.inc.php b/src/tripod.inc.php
index adb7654b..6326c486 100644
--- a/src/tripod.inc.php
+++ b/src/tripod.inc.php
@@ -31,6 +31,7 @@
 require_once TRIPOD_DIR.'classes/ChangeSet.class.php';
 require_once TRIPOD_DIR.'classes/Labeller.class.php';
 require_once TRIPOD_DIR . '/mongo/Driver.class.php';
+require_once TRIPOD_DIR . '/mongo/JobGroup.php';
 
 require_once TRIPOD_DIR.'/mongo/base/JobBase.class.php';
 require_once TRIPOD_DIR . '/mongo/jobs/DiscoverImpactedSubjects.class.php';

From 2167907d24b0eff3c1821a761393cf596994680b Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Wed, 6 Dec 2017 09:21:47 -0500
Subject: [PATCH 30/42] Signature changed

---
 src/mongo/jobs/ApplyOperation.class.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mongo/jobs/ApplyOperation.class.php b/src/mongo/jobs/ApplyOperation.class.php
index c5351d95..ac846938 100644
--- a/src/mongo/jobs/ApplyOperation.class.php
+++ b/src/mongo/jobs/ApplyOperation.class.php
@@ -57,7 +57,7 @@ public function perform()
             $this->getStat()->timer(MONGO_QUEUE_APPLY_OPERATION_SUCCESS,$timer->result());
 
             if (isset($this->args[self::TRACKING_KEY])) {
-                $jobGroup = new JobGroup($this->args[self::TRACKING_KEY]);
+                $jobGroup = new JobGroup($this->args['storeName'], $this->args[self::TRACKING_KEY]);
                 $jobCount = $jobGroup->incrementJobCount(-1);
                 if ($jobCount <= 0) {
                     // @todo Replace this with ObjectId->getTimestamp() if we upgrade Mongo driver to 1.2

From a36d1be81589755d640bc6efbe8f28750b1fd8ce Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Wed, 6 Dec 2017 10:57:39 -0500
Subject: [PATCH 31/42] Logic was in the wrong place

---
 src/mongo/jobs/ApplyOperation.class.php | 58 ++++++++++++-------------
 1 file changed, 29 insertions(+), 29 deletions(-)

diff --git a/src/mongo/jobs/ApplyOperation.class.php b/src/mongo/jobs/ApplyOperation.class.php
index ac846938..b9789317 100644
--- a/src/mongo/jobs/ApplyOperation.class.php
+++ b/src/mongo/jobs/ApplyOperation.class.php
@@ -49,41 +49,41 @@ public function perform()
                 $opTimer->stop();
                 // stat time taken to perform operation for the given subject
                 $this->getStat()->timer(MONGO_QUEUE_APPLY_OPERATION.'.'.$subject['operation'], $opTimer->result());
+
+                if (isset($this->args[self::TRACKING_KEY])) {
+                    $jobGroup = new JobGroup($subject['storeName'], $this->args[self::TRACKING_KEY]);
+                    $jobCount = $jobGroup->incrementJobCount(-1);
+                    if ($jobCount <= 0) {
+                        // @todo Replace this with ObjectId->getTimestamp() if we upgrade Mongo driver to 1.2
+                        $timestamp = new \MongoDB\BSON\UTCDateTime(hexdec(substr($jobGroup->getId(), 0, 8)) * 1000);
+                        $tripod = $this->getTripod($subject['storeName'], $subject['podName']);
+                        $count = 0;
+                        foreach ($subject['specTypes'] as $specId) {
+                            switch ($subject['operation']) {
+                                case \OP_VIEWS:
+                                    $count += $tripod->getTripodViews()->deleteViewsByViewId($specId, $timestamp);
+                                    break;
+                                case \OP_TABLES:
+                                    $count += $tripod->getTripodTables()->deleteTableRowsByTableId($specId, $timestamp);
+                                    break;
+                                case \OP_SEARCH:
+                                    $searchProvider = new \Tripod\Mongo\MongoSearchProvider($tripod);
+                                    $count += $searchProvider->deleteSearchDocumentsByTypeId($specId, $timestamp);
+                                    break;
+                            }
+                        }
+                        $this->infoLog(
+                            '[JobGroupId ' . $jobGroup->getId()->__toString() . '] composite cleanup for ' .
+                            $this->args['operation'] . ' removed ' . $count . ' stale composite documents'
+                        );
+                    }
+                }
             }
 
 
             $timer->stop();
             // stat time taken to process job, from time it was picked up
             $this->getStat()->timer(MONGO_QUEUE_APPLY_OPERATION_SUCCESS,$timer->result());
-
-            if (isset($this->args[self::TRACKING_KEY])) {
-                $jobGroup = new JobGroup($this->args['storeName'], $this->args[self::TRACKING_KEY]);
-                $jobCount = $jobGroup->incrementJobCount(-1);
-                if ($jobCount <= 0) {
-                    // @todo Replace this with ObjectId->getTimestamp() if we upgrade Mongo driver to 1.2
-                    $timestamp = new \MongoDB\BSON\UTCDateTime(hexdec(substr($jobGroup->getId(), 0, 8)) * 1000);
-                    $tripod = $this->getTripod($this->args['storeName'], $this->args['podName']);
-                    $count = 0;
-                    foreach ($this->args['specId'] as $specId) {
-                        switch ($this->args['operation']) {
-                            case \OP_VIEWS:
-                                $count += $tripod->getTripodViews()->deleteViewsByViewId($specId, $timestamp);
-                                break;
-                            case \OP_TABLES:
-                                $count += $tripod->getTripodTables()->deleteTableRowsByTableId($specId, $timestamp);
-                                break;
-                            case \OP_SEARCH:
-                                $searchProvider = new \Tripod\Mongo\MongoSearchProvider($tripod);
-                                $count += $searchProvider->deleteSearchDocumentsByTypeId($specId, $timestamp);
-                                break;
-                        }
-                    }
-                    $this->infoLog(
-                        '[JobGroupId ' . $jobGroup->getId()->__toString() . '] composite cleanup for ' .
-                        $this->args['operation'] . ' removed ' . $count . ' stale composite documents'
-                    );
-                }
-            }
             $this->debugLog("[JOBID " . $this->job->payload['id'] . "] ApplyOperation::perform() done in {$timer->result()}ms");
         } catch (\Exception $e) {
             $this->getStat()->increment(MONGO_QUEUE_APPLY_OPERATION_FAIL);

From 2afe0dc975c15f1841a3beba0d0d707baeb5f915 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Wed, 6 Dec 2017 11:13:50 -0500
Subject: [PATCH 32/42] Leading NS slash

---
 src/mongo/JobGroup.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mongo/JobGroup.php b/src/mongo/JobGroup.php
index 5e525bdf..1d0f82e1 100644
--- a/src/mongo/JobGroup.php
+++ b/src/mongo/JobGroup.php
@@ -33,7 +33,7 @@ public function incrementJobCount($inc = 1)
         $updateResult = $this->getMongoCollection()->findOneAndUpdate(
             ['_id' => $this->id],
             ['$inc' => ['count' => $inc]],
-            ['upsert' => true, 'returnDocument' => MongoDB\Operation\FindOneAndUpdate::RETURN_DOCUMENT_AFTER]
+            ['upsert' => true, 'returnDocument' => \MongoDB\Operation\FindOneAndUpdate::RETURN_DOCUMENT_AFTER]
         );
         return $updateResult['count'];
     }

From 1f0925b5cec3d1dbbe69c8c11d10b96164f5d011 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Wed, 6 Dec 2017 11:46:15 -0500
Subject: [PATCH 33/42] Typo in constant

---
 src/mongo/MongoTripodConstants.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mongo/MongoTripodConstants.php b/src/mongo/MongoTripodConstants.php
index 54ed42b4..342d6353 100644
--- a/src/mongo/MongoTripodConstants.php
+++ b/src/mongo/MongoTripodConstants.php
@@ -6,7 +6,7 @@
 define('VIEWS_COLLECTION', 'views');
 define('LOCKS_COLLECTION', 'locks');
 define('AUDIT_MANUAL_ROLLBACKS_COLLECTION','audit_manual_rollbacks');
-define('OPERATION_GROUP_COLLECTION', 'job_groups');
+define('OPERATION_GROUPS_COLLECTION', 'job_groups');
 
 // search
 define('SEARCH_INDEX_COLLECTION', 'search');

From d5eeca26d786c3f22ad0e7ef14d8ffa4dfce547f Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Wed, 6 Dec 2017 11:48:50 -0500
Subject: [PATCH 34/42] Return count only if array

---
 src/mongo/JobGroup.php | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/mongo/JobGroup.php b/src/mongo/JobGroup.php
index 1d0f82e1..5b1ec939 100644
--- a/src/mongo/JobGroup.php
+++ b/src/mongo/JobGroup.php
@@ -35,7 +35,9 @@ public function incrementJobCount($inc = 1)
             ['$inc' => ['count' => $inc]],
             ['upsert' => true, 'returnDocument' => \MongoDB\Operation\FindOneAndUpdate::RETURN_DOCUMENT_AFTER]
         );
-        return $updateResult['count'];
+        if (\is_array($updateResult)) {
+            return $updateResult['count'];
+        }
     }
 
     /**

From 99dc7aa43c8c8e0bbc12b954d6b5325ecd8cd207 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Wed, 6 Dec 2017 12:12:22 -0500
Subject: [PATCH 35/42] Again, argument in the wrong place

---
 src/mongo/jobs/ApplyOperation.class.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/mongo/jobs/ApplyOperation.class.php b/src/mongo/jobs/ApplyOperation.class.php
index b9789317..e0725c95 100644
--- a/src/mongo/jobs/ApplyOperation.class.php
+++ b/src/mongo/jobs/ApplyOperation.class.php
@@ -74,7 +74,7 @@ public function perform()
                         }
                         $this->infoLog(
                             '[JobGroupId ' . $jobGroup->getId()->__toString() . '] composite cleanup for ' .
-                            $this->args['operation'] . ' removed ' . $count . ' stale composite documents'
+                            $subject['operation'] . ' removed ' . $count . ' stale composite documents'
                         );
                     }
                 }

From 9e02be625fa3c2f390f7657d29d7c0d0fd5cee02 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Wed, 6 Dec 2017 14:17:48 -0500
Subject: [PATCH 36/42] result could be an array or object

---
 src/mongo/JobGroup.php               | 2 ++
 src/mongo/delegates/Tables.class.php | 6 +-----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/mongo/JobGroup.php b/src/mongo/JobGroup.php
index 5b1ec939..bae5cc3d 100644
--- a/src/mongo/JobGroup.php
+++ b/src/mongo/JobGroup.php
@@ -37,6 +37,8 @@ public function incrementJobCount($inc = 1)
         );
         if (\is_array($updateResult)) {
             return $updateResult['count'];
+        } elseif (isset($updateResult->count)) {
+            return $updateResult->count;
         }
     }
 
diff --git a/src/mongo/delegates/Tables.class.php b/src/mongo/delegates/Tables.class.php
index 8a2e83bf..be97aca8 100644
--- a/src/mongo/delegates/Tables.class.php
+++ b/src/mongo/delegates/Tables.class.php
@@ -545,7 +545,7 @@ public function generateTableRows($tableType, $resource = null, $context = null,
             $jobOptions['statsConfig'] = $this->getStatsConfig();
             $jobGroup = new JobGroup($this->storeName);
             $jobOptions[ApplyOperation::TRACKING_KEY] = $jobGroup->getId()->__toString();
-            $jobGroup->setJobCount($count);;
+            $jobGroup->setJobCount($count);
         }
 
         foreach ($docs as $doc) {
@@ -557,10 +557,6 @@ public function generateTableRows($tableType, $resource = null, $context = null,
                     $from,
                     array($tableType)
                 );
-                if (empty($jobOptions))
-                if ($this->stat || !empty($this->statsConfig)) {
-                    $jobOptions['statsConfig'] = $this->getStatsConfig();
-                }
 
                 $this->getApplyOperation()->createJob(array($subject), $queueName, $jobOptions);
             } else {

From b396e3df4009f47b87c9753d80b2b501e2ce3af2 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Wed, 6 Dec 2017 14:29:16 -0500
Subject: [PATCH 37/42] Call method for id

---
 src/mongo/JobGroup.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/mongo/JobGroup.php b/src/mongo/JobGroup.php
index bae5cc3d..1e6d4bfc 100644
--- a/src/mongo/JobGroup.php
+++ b/src/mongo/JobGroup.php
@@ -22,7 +22,7 @@ public function __construct($storeName, $groupId = null)
     public function setJobCount($count)
     {
         $this->getMongoCollection()->updateOne(
-            ['_id' => $this->id],
+            ['_id' => $this->getId()],
             ['$set' => ['count' => $count]],
             ['upsert' => true]
         );
@@ -31,7 +31,7 @@ public function setJobCount($count)
     public function incrementJobCount($inc = 1)
     {
         $updateResult = $this->getMongoCollection()->findOneAndUpdate(
-            ['_id' => $this->id],
+            ['_id' => $this->getId()],
             ['$inc' => ['count' => $inc]],
             ['upsert' => true, 'returnDocument' => \MongoDB\Operation\FindOneAndUpdate::RETURN_DOCUMENT_AFTER]
         );

From 46998bc4cca570a01cdd898a41a9b0096d94d4c6 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Mon, 11 Dec 2017 12:00:22 -0500
Subject: [PATCH 38/42] Docblocks

---
 src/mongo/JobGroup.php | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/mongo/JobGroup.php b/src/mongo/JobGroup.php
index 1e6d4bfc..b4f25f4c 100644
--- a/src/mongo/JobGroup.php
+++ b/src/mongo/JobGroup.php
@@ -8,6 +8,11 @@ class JobGroup
     private $collection;
     private $storeName;
 
+    /**
+     * Constructor method
+     * @param string                        $storeName Tripod store (database) name
+     * @param string|\MongoDB\BSON\ObjectId $groupId   Optional tracking ID, will assign a new one if omitted
+     */
     public function __construct($storeName, $groupId = null)
     {
         $this->storeName = $storeName;
@@ -19,6 +24,12 @@ public function __construct($storeName, $groupId = null)
         $this->id = $groupId;
     }
 
+    /**
+     * Update the number of jobs
+     *
+     * @param integer $count Number of jobs in group
+     * @return void
+     */
     public function setJobCount($count)
     {
         $this->getMongoCollection()->updateOne(
@@ -28,6 +39,12 @@ public function setJobCount($count)
         );
     }
 
+    /**
+     * Update the number of jobs by $inc.  To decrement, use a negative integer
+     *
+     * @param integer $inc Number to increment or decrement by
+     * @return integer Updated job count
+     */
     public function incrementJobCount($inc = 1)
     {
         $updateResult = $this->getMongoCollection()->findOneAndUpdate(

From 043bd06050b6a0b659f00354d17af3baac9a0944 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Mon, 11 Dec 2017 14:19:40 -0500
Subject: [PATCH 39/42] Unit tests for batch ApplyOperation jobs

---
 src/mongo/jobs/ApplyOperation.class.php |  32 +-
 test/unit/mongo/ApplyOperationTest.php  | 669 +++++++++++++++++++++++-
 2 files changed, 676 insertions(+), 25 deletions(-)

diff --git a/src/mongo/jobs/ApplyOperation.class.php b/src/mongo/jobs/ApplyOperation.class.php
index e0725c95..95c9b8ef 100644
--- a/src/mongo/jobs/ApplyOperation.class.php
+++ b/src/mongo/jobs/ApplyOperation.class.php
@@ -3,6 +3,7 @@
 namespace Tripod\Mongo\Jobs;
 
 use Tripod\Mongo\JobGroup;
+use Tripod\Mongo\Driver;
 
 
 /**
@@ -51,7 +52,7 @@ public function perform()
                 $this->getStat()->timer(MONGO_QUEUE_APPLY_OPERATION.'.'.$subject['operation'], $opTimer->result());
 
                 if (isset($this->args[self::TRACKING_KEY])) {
-                    $jobGroup = new JobGroup($subject['storeName'], $this->args[self::TRACKING_KEY]);
+                    $jobGroup = $this->getJobGroup($subject['storeName'], $this->args[self::TRACKING_KEY]);
                     $jobCount = $jobGroup->incrementJobCount(-1);
                     if ($jobCount <= 0) {
                         // @todo Replace this with ObjectId->getTimestamp() if we upgrade Mongo driver to 1.2
@@ -61,13 +62,13 @@ public function perform()
                         foreach ($subject['specTypes'] as $specId) {
                             switch ($subject['operation']) {
                                 case \OP_VIEWS:
-                                    $count += $tripod->getTripodViews()->deleteViewsByViewId($specId, $timestamp);
+                                    $count += $tripod->getComposite(\OP_VIEWS)->deleteViewsByViewId($specId, $timestamp);
                                     break;
                                 case \OP_TABLES:
-                                    $count += $tripod->getTripodTables()->deleteTableRowsByTableId($specId, $timestamp);
+                                    $count += $tripod->getComposite(\OP_TABLES)->deleteTableRowsByTableId($specId, $timestamp);
                                     break;
                                 case \OP_SEARCH:
-                                    $searchProvider = new \Tripod\Mongo\MongoSearchProvider($tripod);
+                                    $searchProvider = $this->getSearchProvider($tripod);
                                     $count += $searchProvider->deleteSearchDocumentsByTypeId($specId, $timestamp);
                                     break;
                             }
@@ -140,4 +141,27 @@ protected function getMandatoryArgs()
     {
         return array(self::TRIPOD_CONFIG_KEY,self::SUBJECTS_KEY);
     }
+
+    /**
+     * For mocking
+     *
+     * @param string                        $storeName   Tripod store (database) name
+     * @param string|\MongoDB\BSON\ObjectId $trackingKey JobGroup ID
+     * @return JobGroup
+     */
+    protected function getJobGroup($storeName, $trackingKey)
+    {
+        return new JobGroup($storeName, $trackingKey);
+    }
+
+    /**
+     * For mocking
+     *
+     * @param Driver $tripod
+     * @return \Tripod\Mongo\MongoSearchProvider
+     */
+    protected function getSearchProvider(Driver $tripod)
+    {
+        return new \Tripod\Mongo\MongoSearchProvider($tripod);
+    }
 }
diff --git a/test/unit/mongo/ApplyOperationTest.php b/test/unit/mongo/ApplyOperationTest.php
index 007746b3..6829a666 100644
--- a/test/unit/mongo/ApplyOperationTest.php
+++ b/test/unit/mongo/ApplyOperationTest.php
@@ -1,5 +1,7 @@
 <?php
 
+use Tripod\Mongo\Jobs\ApplyOperation;
+
 require_once 'MongoTripodTestBase.php';
 
 /**
@@ -109,6 +111,216 @@ public function testApplyViewOperation()
         $applyOperation->perform();
     }
 
+    public function testApplyViewOperationDecrementsJobGroupForBatchOperations()
+    {
+        $this->setArgs();
+        $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation')
+            ->setMethods(['createImpactedSubject', 'getStat', 'getJobGroup', 'getTripod'])
+            ->getMock();
+
+        $applyOperation->args = $this->args;
+        $applyOperation->job->payload['id'] = uniqid();
+        $jobTrackerId = new \MongoDB\BSON\ObjectId();
+        $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString();
+
+        $jobGroup = $this->getMockBuilder('\Tripod\Mongo\JobGroup')
+            ->setMethods(['incrementJobCount'])
+            ->setConstructorArgs(['tripod_php_testing', $jobTrackerId])
+            ->getMock();
+
+        $jobGroup->expects($this->once())
+            ->method('incrementJobCount')
+            ->with(-1)
+            ->will($this->returnValue(2));
+
+        $statMock = $this->getMockStat(
+            $this->args['statsConfig']['config']['host'],
+            $this->args['statsConfig']['config']['port'],
+            $this->args['statsConfig']['config']['prefix'],
+            ['timer','increment']
+        );
+
+        $subject = $this->getMockBuilder('\Tripod\Mongo\ImpactedSubject')
+            ->setMethods(['getTripod'])
+            ->setConstructorArgs(
+                [
+                    [
+                        _ID_RESOURCE=>'http://example.com/resources/foo',
+                        _ID_CONTEXT=>'http://talisaspire.com'
+                    ],
+                    OP_VIEWS,
+                    'tripod_php_testing',
+                    'CBD_testing'
+                ]
+            )->getMock();
+
+        $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver')
+            ->setMethods(['getComposite'])
+            ->setConstructorArgs(['CBD_testing', 'tripod_php_testing'])
+            ->getMock();
+
+        $views = $this->getMockBuilder('\Tripod\Mongo\Composites\Views')
+            ->setMethods(['update', 'deleteViewsByViewId'])
+            ->setConstructorArgs(
+                [
+                    'tripod_php_testing',
+                    \Tripod\Mongo\Config::getInstance()->getCollectionForCBD('tripod_php_testing', 'CBD_testing'),
+                    'http://talisapire.com/'
+                ]
+            )->getMock();
+
+        $applyOperation->expects($this->once())
+            ->method('createImpactedSubject')
+            ->will($this->returnValue($subject));
+
+        $applyOperation->expects($this->exactly(3))
+            ->method('getStat')
+            ->will($this->returnValue($statMock));
+
+        $applyOperation->expects($this->once())
+            ->method('getJobGroup')
+            ->with('tripod_php_testing', $jobTrackerId->__toString())
+            ->will($this->returnValue($jobGroup));
+
+        $applyOperation->expects($this->never())
+            ->method('getTripod');
+
+        $statMock->expects($this->once())
+            ->method('increment')
+            ->with(MONGO_QUEUE_APPLY_OPERATION_JOB . '.' . SUBJECT_COUNT, 1);
+        $statMock->expects($this->exactly(2))
+            ->method('timer')
+            ->withConsecutive(
+                [MONGO_QUEUE_APPLY_OPERATION.'.'.OP_VIEWS, $this->anything()],
+                [MONGO_QUEUE_APPLY_OPERATION_SUCCESS, $this->anything()]
+            );
+
+        $subject->expects($this->once())
+            ->method('getTripod')
+            ->will($this->returnValue($tripod));
+
+        $tripod->expects($this->once())
+            ->method('getComposite')
+            ->with(OP_VIEWS)
+            ->will($this->returnValue($views));
+
+        $views->expects($this->once())
+            ->method('update')
+            ->with($subject);
+
+        $views->expects($this->never())->method('deleteViewsByViewId');
+        $applyOperation->perform();
+    }
+
+    public function testApplyViewOperationCleanupIfAllGroupJobsComplete()
+    {
+        $this->setArgs(OP_VIEWS, ['v_foo_bar']);
+        $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation')
+            ->setMethods(['createImpactedSubject', 'getStat', 'getJobGroup', 'getTripod'])
+            ->getMock();
+
+        $applyOperation->args = $this->args;
+        $applyOperation->job->payload['id'] = uniqid();
+        $jobTrackerId = new \MongoDB\BSON\ObjectId();
+        $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString();
+        $timestamp = new \MongoDB\BSON\UTCDateTime(hexdec(substr($jobTrackerId, 0, 8)) * 1000);
+
+        $jobGroup = $this->getMockBuilder('\Tripod\Mongo\JobGroup')
+            ->setMethods(['incrementJobCount'])
+            ->setConstructorArgs(['tripod_php_testing', $jobTrackerId])
+            ->getMock();
+
+        $jobGroup->expects($this->once())
+            ->method('incrementJobCount')
+            ->with(-1)
+            ->will($this->returnValue(0));
+
+        $statMock = $this->getMockStat(
+            $this->args['statsConfig']['config']['host'],
+            $this->args['statsConfig']['config']['port'],
+            $this->args['statsConfig']['config']['prefix'],
+            array('timer','increment')
+        );
+
+        $subject = $this->getMockBuilder('\Tripod\Mongo\ImpactedSubject')
+            ->setMethods(['getTripod'])
+            ->setConstructorArgs(
+                [
+                    [
+                        _ID_RESOURCE=>'http://example.com/resources/foo',
+                        _ID_CONTEXT=>'http://talisaspire.com'
+                    ],
+                    OP_VIEWS,
+                    'tripod_php_testing',
+                    'CBD_testing'
+                ]
+            )->getMock();
+
+        $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver')
+            ->setMethods(['getComposite'])
+            ->setConstructorArgs(['CBD_testing', 'tripod_php_testing'])
+            ->getMock();
+
+        $views = $this->getMockBuilder('\Tripod\Mongo\Composites\Views')
+            ->setMethods(['update', 'deleteViewsByViewId'])
+            ->setConstructorArgs(
+                [
+                    'tripod_php_testing',
+                    \Tripod\Mongo\Config::getInstance()->getCollectionForCBD('tripod_php_testing', 'CBD_testing'),
+                    'http://talisapire.com/'
+                ]
+            )->getMock();
+
+        $applyOperation->expects($this->once())
+            ->method('createImpactedSubject')
+            ->will($this->returnValue($subject));
+
+        // $applyOperation->expects($this->exactly(3))
+        $applyOperation->expects($this->any())
+            ->method('getStat')
+            ->will($this->returnValue($statMock));
+
+        $applyOperation->expects($this->once())
+            ->method('getJobGroup')
+            ->with('tripod_php_testing', $jobTrackerId->__toString())
+            ->will($this->returnValue($jobGroup));
+
+        $applyOperation->expects($this->once())
+            ->method('getTripod')
+            ->with('tripod_php_testing', 'CBD_testing')
+            ->will($this->returnValue($tripod));
+
+        $statMock->expects($this->once())
+            ->method('increment')
+            ->with(MONGO_QUEUE_APPLY_OPERATION_JOB . '.' . SUBJECT_COUNT, 1);
+        $statMock->expects($this->exactly(2))
+            ->method('timer')
+            ->withConsecutive(
+                [MONGO_QUEUE_APPLY_OPERATION.'.'.OP_VIEWS, $this->anything()],
+                [MONGO_QUEUE_APPLY_OPERATION_SUCCESS, $this->anything()]
+            );
+
+        $subject->expects($this->once())
+            ->method('getTripod')
+            ->will($this->returnValue($tripod));
+
+        $tripod->expects($this->exactly(2))
+            ->method('getComposite')
+            ->with(OP_VIEWS)
+            ->will($this->returnValue($views));
+
+        $views->expects($this->once())
+            ->method('update')
+            ->with($subject);
+
+        $views->expects($this->once())
+            ->method('deleteViewsByViewId')
+            ->with('v_foo_bar', $timestamp)
+            ->will($this->returnValue(3));
+
+        $applyOperation->perform();
+    }
+
     public function testApplyTableOperation()
     {
         $this->setArgs();
@@ -200,9 +412,222 @@ public function testApplyTableOperation()
         $applyOperation->perform();
     }
 
+    public function testApplyTableOperationDecrementsJobGroupForBatchOperations()
+    {
+        $this->setArgs(OP_TABLES, ['t_resource']);
+        $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation')
+            ->setMethods(['createImpactedSubject', 'getStat', 'getJobGroup', 'getTripod'])
+            ->getMock();
+
+        $statMock = $this->getMockStat(
+            $this->args['statsConfig']['config']['host'],
+            $this->args['statsConfig']['config']['port'],
+            $this->args['statsConfig']['config']['prefix'],
+            ['timer', 'increment']
+        );
+
+        $applyOperation->args = $this->args;
+        $applyOperation->job->payload['id'] = uniqid();
+
+        $jobTrackerId = new \MongoDB\BSON\ObjectId();
+        $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString();
+
+        $jobGroup = $this->getMockBuilder('\Tripod\Mongo\JobGroup')
+            ->setMethods(['incrementJobCount'])
+            ->setConstructorArgs(['tripod_php_testing', $jobTrackerId])
+            ->getMock();
+
+        $jobGroup->expects($this->once())
+            ->method('incrementJobCount')
+            ->with(-1)
+            ->will($this->returnValue(2));
+
+        $subject = $this->getMockBuilder('\Tripod\Mongo\ImpactedSubject')
+            ->setMethods(['getTripod'])
+            ->setConstructorArgs(
+                [
+                    [
+                        _ID_RESOURCE=>'http://example.com/resources/foo',
+                        _ID_CONTEXT=>'http://talisaspire.com'
+                    ],
+                    OP_TABLES,
+                    'tripod_php_testing',
+                    'CBD_testing'
+                ]
+            )->getMock();
+
+        $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver')
+            ->setMethods(['getComposite'])
+            ->setConstructorArgs(['CBD_testing', 'tripod_php_testing'])
+            ->getMock();
+
+        $tables = $this->getMockBuilder('\Tripod\Mongo\Composites\Tables')
+            ->setMethods(['update', 'deleteTableRowsByTableId'])
+            ->setConstructorArgs(
+                [
+                    'tripod_php_testing',
+                    \Tripod\Mongo\Config::getInstance()->getCollectionForCBD('tripod_php_testing', 'CBD_testing'),
+                    'http://talisapire.com/'
+                ]
+            )->getMock();
+
+        $applyOperation->expects($this->once())
+            ->method('createImpactedSubject')
+            ->will($this->returnValue($subject));
+
+        $applyOperation->expects($this->exactly(3))
+            ->method('getStat')
+            ->will($this->returnValue($statMock));
+
+        $applyOperation->expects($this->once())
+            ->method('getJobGroup')
+            ->with('tripod_php_testing', $jobTrackerId->__toString())
+            ->will($this->returnValue($jobGroup));
+
+        $applyOperation->expects($this->never())
+            ->method('getTripod');
+
+        $statMock->expects($this->once())
+            ->method('increment')
+            ->with(MONGO_QUEUE_APPLY_OPERATION_JOB . '.' . SUBJECT_COUNT, 1);
+        $statMock->expects($this->exactly(2))
+            ->method('timer')
+            ->withConsecutive(
+                array(MONGO_QUEUE_APPLY_OPERATION.'.'.OP_TABLES, $this->anything()),
+                array(MONGO_QUEUE_APPLY_OPERATION_SUCCESS, $this->anything())
+            );
+
+
+        $subject->expects($this->once())
+            ->method('getTripod')
+            ->will($this->returnValue($tripod));
+
+        $tripod->expects($this->once())
+            ->method('getComposite')
+            ->with(OP_TABLES)
+            ->will($this->returnValue($tables));
+
+        $tables->expects($this->once())
+            ->method('update')
+            ->with($subject);
+        $tables->expects($this->never())
+            ->method('deleteTableRowsByTableId');
+
+        $applyOperation->perform();
+    }
+
+    public function testApplyTableOperationCleanupIfAllGroupJobsComplete()
+    {
+        $this->setArgs(OP_TABLES, ['t_resource']);
+        $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation')
+            ->setMethods(['createImpactedSubject', 'getStat', 'getJobGroup', 'getTripod'])
+            ->getMock();
+
+        $statMock = $this->getMockStat(
+            $this->args['statsConfig']['config']['host'],
+            $this->args['statsConfig']['config']['port'],
+            $this->args['statsConfig']['config']['prefix'],
+            ['timer','increment']
+        );
+
+        $applyOperation->args = $this->args;
+        $applyOperation->job->payload['id'] = uniqid();
+
+        $jobTrackerId = new \MongoDB\BSON\ObjectId();
+        $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString();
+        $timestamp = new \MongoDB\BSON\UTCDateTime(hexdec(substr($jobTrackerId, 0, 8)) * 1000);
+
+        $jobGroup = $this->getMockBuilder('\Tripod\Mongo\JobGroup')
+            ->setMethods(['incrementJobCount'])
+            ->setConstructorArgs(['tripod_php_testing', $jobTrackerId])
+            ->getMock();
+
+        $jobGroup->expects($this->once())
+            ->method('incrementJobCount')
+            ->with(-1)
+            ->will($this->returnValue(0));
+
+        $subject = $this->getMockBuilder('\Tripod\Mongo\ImpactedSubject')
+            ->setMethods(['getTripod'])
+            ->setConstructorArgs(
+                [
+                    [
+                        _ID_RESOURCE=>'http://example.com/resources/foo',
+                        _ID_CONTEXT=>'http://talisaspire.com'
+                    ],
+                    OP_TABLES,
+                    'tripod_php_testing',
+                    'CBD_testing'
+                ]
+            )->getMock();
+
+        $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver')
+            ->setMethods(['getComposite'])
+            ->setConstructorArgs(['CBD_testing', 'tripod_php_testing'])
+            ->getMock();
+
+        $tables = $this->getMockBuilder('\Tripod\Mongo\Composites\Tables')
+            ->setMethods(['update', 'deleteTableRowsByTableId'])
+            ->setConstructorArgs(
+                [
+                    'tripod_php_testing',
+                    \Tripod\Mongo\Config::getInstance()->getCollectionForCBD('tripod_php_testing', 'CBD_testing'),
+                    'http://talisapire.com/'
+                ]
+            )->getMock();
+
+        $applyOperation->expects($this->once())
+            ->method('createImpactedSubject')
+            ->will($this->returnValue($subject));
+
+        $applyOperation->expects($this->exactly(3))
+            ->method('getStat')
+            ->will($this->returnValue($statMock));
+
+        $applyOperation->expects($this->once())
+            ->method('getJobGroup')
+            ->with('tripod_php_testing', $jobTrackerId->__toString())
+            ->will($this->returnValue($jobGroup));
+
+        $applyOperation->expects($this->once())
+            ->method('getTripod')
+            ->with('tripod_php_testing', 'CBD_testing')
+            ->will($this->returnValue($tripod));
+
+        $statMock->expects($this->once())
+            ->method('increment')
+            ->with(MONGO_QUEUE_APPLY_OPERATION_JOB . '.' . SUBJECT_COUNT, 1);
+        $statMock->expects($this->exactly(2))
+            ->method('timer')
+            ->withConsecutive(
+                array(MONGO_QUEUE_APPLY_OPERATION.'.'.OP_TABLES, $this->anything()),
+                array(MONGO_QUEUE_APPLY_OPERATION_SUCCESS, $this->anything())
+            );
+
+
+        $subject->expects($this->once())
+            ->method('getTripod')
+            ->will($this->returnValue($tripod));
+
+        $tripod->expects($this->exactly(2))
+            ->method('getComposite')
+            ->with(OP_TABLES)
+            ->will($this->returnValue($tables));
+
+        $tables->expects($this->once())
+            ->method('update')
+            ->with($subject);
+        $tables->expects($this->once())
+            ->method('deleteTableRowsByTableId')
+            ->with('t_resource', $timestamp)
+            ->will($this->returnValue(4));
+
+        $applyOperation->perform();
+    }
+
     public function testApplySearchOperation()
     {
-        $this->setArgs();
+        $this->setArgs(OP_SEARCH, ['i_search_resource']);
         $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation')
             ->setMethods(array('createImpactedSubject', 'getStat'))
             ->getMock();
@@ -214,18 +639,6 @@ public function testApplySearchOperation()
             array('timer','increment')
         );
 
-        $impactedSubject = new \Tripod\Mongo\ImpactedSubject(
-            array(
-                _ID_RESOURCE=>'http://example.com/resources/foo',
-                _ID_CONTEXT=>'http://talisaspire.com/'
-            ),
-            OP_SEARCH,
-            'tripod_php_testing',
-            'CBD_testing',
-            array('t_resource')
-        );
-        $this->args['subjects'] = array($impactedSubject->toArray());
-
         $applyOperation->args = $this->args;
         $applyOperation->job->payload['id'] = uniqid();
 
@@ -288,6 +701,219 @@ public function testApplySearchOperation()
         $applyOperation->perform();
     }
 
+    public function testApplySearchOperationDecrementsJobGroupForBatchOperations()
+    {
+        $this->setArgs(OP_SEARCH, ['i_search_resource']);
+        $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation')
+            ->setMethods(['createImpactedSubject', 'getStat', 'getJobGroup', 'getTripod'])
+            ->getMock();
+
+        $statMock = $this->getMockStat(
+            $this->args['statsConfig']['config']['host'],
+            $this->args['statsConfig']['config']['port'],
+            $this->args['statsConfig']['config']['prefix'],
+            ['timer', 'increment']
+        );
+
+        $applyOperation->args = $this->args;
+        $applyOperation->job->payload['id'] = uniqid();
+
+        $jobTrackerId = new \MongoDB\BSON\ObjectId();
+        $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString();
+
+        $jobGroup = $this->getMockBuilder('\Tripod\Mongo\JobGroup')
+            ->setMethods(['incrementJobCount'])
+            ->setConstructorArgs(['tripod_php_testing', $jobTrackerId])
+            ->getMock();
+
+        $jobGroup->expects($this->once())
+            ->method('incrementJobCount')
+            ->with(-1)
+            ->will($this->returnValue(2));
+
+        $subject = $this->getMockBuilder('\Tripod\Mongo\ImpactedSubject')
+            ->setMethods(['getTripod'])
+            ->setConstructorArgs(
+                [
+                    [
+                        _ID_RESOURCE=>'http://example.com/resources/foo',
+                        _ID_CONTEXT=>'http://talisaspire.com'
+                    ],
+                    OP_SEARCH,
+                    'tripod_php_testing',
+                    'CBD_testing'
+                ]
+            )->getMock();
+
+        $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver')
+            ->setMethods(['getComposite'])
+            ->setConstructorArgs(['CBD_testing', 'tripod_php_testing'])
+            ->getMock();
+
+        $search = $this->getMockBuilder('\Tripod\Mongo\Composites\SearchIndexer')
+            ->setMethods(['update'])
+            ->setConstructorArgs([$tripod])
+            ->getMock();
+
+        $applyOperation->expects($this->once())
+            ->method('createImpactedSubject')
+            ->will($this->returnValue($subject));
+
+        $applyOperation->expects($this->exactly(3))
+            ->method('getStat')
+            ->will($this->returnValue($statMock));
+
+        $applyOperation->expects($this->once())
+            ->method('getJobGroup')
+            ->with('tripod_php_testing', $jobTrackerId->__toString())
+            ->will($this->returnValue($jobGroup));
+
+        $applyOperation->expects($this->never())
+            ->method('getTripod');
+
+        $statMock->expects($this->once())
+            ->method('increment')
+            ->with(MONGO_QUEUE_APPLY_OPERATION_JOB . '.' . SUBJECT_COUNT, 1);
+        $statMock->expects($this->exactly(2))
+            ->method('timer')
+            ->withConsecutive(
+                array(MONGO_QUEUE_APPLY_OPERATION.'.'.OP_SEARCH, $this->anything()),
+                array(MONGO_QUEUE_APPLY_OPERATION_SUCCESS, $this->anything())
+            );
+
+
+        $subject->expects($this->once())
+            ->method('getTripod')
+            ->will($this->returnValue($tripod));
+
+        $tripod->expects($this->once())
+            ->method('getComposite')
+            ->with(OP_SEARCH)
+            ->will($this->returnValue($search));
+
+        $search->expects($this->once())
+            ->method('update')
+            ->with($subject);
+
+        $applyOperation->perform();
+    }
+
+    public function testApplySearchOperationCleanupIfAllGroupJobsComplete()
+    {
+        $this->setArgs(OP_SEARCH, ['i_search_resource']);
+        $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation')
+            ->setMethods(['createImpactedSubject', 'getStat', 'getJobGroup', 'getTripod', 'getSearchProvider'])
+            ->getMock();
+
+        $statMock = $this->getMockStat(
+            $this->args['statsConfig']['config']['host'],
+            $this->args['statsConfig']['config']['port'],
+            $this->args['statsConfig']['config']['prefix'],
+            ['timer', 'increment']
+        );
+
+        $applyOperation->args = $this->args;
+        $applyOperation->job->payload['id'] = uniqid();
+
+        $jobTrackerId = new \MongoDB\BSON\ObjectId();
+        $applyOperation->args[ApplyOperation::TRACKING_KEY] = $jobTrackerId->__toString();
+        $timestamp = new \MongoDB\BSON\UTCDateTime(hexdec(substr($jobTrackerId, 0, 8)) * 1000);
+
+        $jobGroup = $this->getMockBuilder('\Tripod\Mongo\JobGroup')
+            ->setMethods(['incrementJobCount'])
+            ->setConstructorArgs(['tripod_php_testing', $jobTrackerId])
+            ->getMock();
+
+        $jobGroup->expects($this->once())
+            ->method('incrementJobCount')
+            ->with(-1)
+            ->will($this->returnValue(0));
+
+        $subject = $this->getMockBuilder('\Tripod\Mongo\ImpactedSubject')
+            ->setMethods(['getTripod'])
+            ->setConstructorArgs(
+                [
+                    [
+                        _ID_RESOURCE=>'http://example.com/resources/foo',
+                        _ID_CONTEXT=>'http://talisaspire.com'
+                    ],
+                    OP_SEARCH,
+                    'tripod_php_testing',
+                    'CBD_testing'
+                ]
+            )->getMock();
+
+        $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver')
+            ->setMethods(['getComposite'])
+            ->setConstructorArgs(['CBD_testing', 'tripod_php_testing'])
+            ->getMock();
+
+        $searchProvider = $this->getMockBuilder('\Tripod\Mongo\MongoSearchProvider')
+            ->setMethods(['deleteSearchDocumentsByTypeId'])
+            ->setConstructorArgs([$tripod])
+            ->getMock();
+
+        $search = $this->getMockBuilder('\Tripod\Mongo\Composites\SearchIndexer')
+            ->setMethods(['update'])
+            ->setConstructorArgs([$tripod])
+            ->getMock();
+
+        $applyOperation->expects($this->once())
+            ->method('createImpactedSubject')
+            ->will($this->returnValue($subject));
+
+        $applyOperation->expects($this->exactly(3))
+            ->method('getStat')
+            ->will($this->returnValue($statMock));
+
+        $applyOperation->expects($this->once())
+            ->method('getJobGroup')
+            ->with('tripod_php_testing', $jobTrackerId->__toString())
+            ->will($this->returnValue($jobGroup));
+
+        $applyOperation->expects($this->once())
+            ->method('getTripod')
+            ->with('tripod_php_testing', 'CBD_testing')
+            ->will($this->returnValue($tripod));
+
+        $applyOperation->expects($this->once())
+            ->method('getSearchProvider')
+            ->with($tripod)
+            ->will($this->returnValue($searchProvider));
+
+        $statMock->expects($this->once())
+            ->method('increment')
+            ->with(MONGO_QUEUE_APPLY_OPERATION_JOB . '.' . SUBJECT_COUNT, 1);
+
+        $statMock->expects($this->exactly(2))
+            ->method('timer')
+            ->withConsecutive(
+                array(MONGO_QUEUE_APPLY_OPERATION.'.'.OP_SEARCH, $this->anything()),
+                array(MONGO_QUEUE_APPLY_OPERATION_SUCCESS, $this->anything())
+            );
+
+
+        $subject->expects($this->once())
+            ->method('getTripod')
+            ->will($this->returnValue($tripod));
+
+        $tripod->expects($this->once())
+            ->method('getComposite')
+            ->with(OP_SEARCH)
+            ->will($this->returnValue($search));
+
+        $search->expects($this->once())
+            ->method('update')
+            ->with($subject);
+
+        $searchProvider->expects($this->once())
+            ->method('deleteSearchDocumentsByTypeId')
+            ->with('i_search_resource', $timestamp)
+            ->will($this->returnValue(8));
+
+        $applyOperation->perform();
+    }
+
     public function testCreateJobDefaultQueue()
     {
         $impactedSubject = new \Tripod\Mongo\ImpactedSubject(
@@ -429,22 +1055,23 @@ public function testCreateJobSpecifyQueue()
     /**
      * Sets job arguments
      */
-    protected function setArgs()
+    protected function setArgs($operation = OP_VIEWS, array $specTypes = [])
     {
         $subject = new \Tripod\Mongo\ImpactedSubject(
-            array(
+            [
                 _ID_RESOURCE=>'http://example.com/resources/foo',
                 _ID_CONTEXT=>'http://talisaspire.com/'
-            ),
-            OP_VIEWS,
+            ],
+            $operation,
             'tripod_php_testing',
-            'CBD_testing'
+            'CBD_testing',
+            $specTypes
         );
 
         $this->args = array(
-            'tripodConfig'=>\Tripod\Mongo\Config::getConfig(),
-            'subjects'=>array($subject->toArray()),
-            'statsConfig'=>$this->getStatsDConfig()
+            'tripodConfig' => \Tripod\Mongo\Config::getConfig(),
+            'subjects'=> [$subject->toArray()],
+            'statsConfig' => $this->getStatsDConfig()
         );
     }
 }

From c15f352d36c3589584ff1b49c070e1ce35c26c23 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Mon, 11 Dec 2017 15:29:07 -0500
Subject: [PATCH 40/42] Composite tests for count & delete

---
 src/mongo/delegates/Tables.class.php          |  15 +-
 src/mongo/delegates/Views.class.php           |   2 +-
 .../providers/MongoSearchProvider.class.php   |  15 +-
 test/unit/mongo/MongoSearchProviderTest.php   | 156 +++++++++++++++++-
 test/unit/mongo/MongoTripodTablesTest.php     | 126 ++++++++++++++
 test/unit/mongo/MongoTripodViewsTest.php      | 127 ++++++++++++++
 6 files changed, 435 insertions(+), 6 deletions(-)

diff --git a/src/mongo/delegates/Tables.class.php b/src/mongo/delegates/Tables.class.php
index be97aca8..a20992fe 100644
--- a/src/mongo/delegates/Tables.class.php
+++ b/src/mongo/delegates/Tables.class.php
@@ -383,7 +383,7 @@ public function deleteTableRowsByTableId($tableId, $timestamp = null) {
                 [\_CREATED_TS => ['$exists' => false]]
             ];
         }
-        $deleteResult = $this->config->getCollectionForTable($this->storeName, $tableId)
+        $deleteResult = $this->getCollectionForTableSpec($tableId)
             ->deleteMany($query);
 
         $t->stop();
@@ -1463,6 +1463,17 @@ private function applyRegexToValue($regex, $value)
     public function count($tableSpec, array $filters = [])
     {
         $filters['_id.type'] = $tableSpec;
-        return $this->config->getCollectionForTable($this->storeName, $tableSpec)->count($filters);
+        return $this->getCollectionForTableSpec($tableSpec)->count($filters);
+    }
+
+    /**
+     * For mocking
+     *
+     * @param string $tableSpecId Table spec ID
+     * @return \MongoDB\Collection
+     */
+    protected function getCollectionForTableSpec($tableSpecId)
+    {
+        return $this->getConfigInstance()->getCollectionForTable($this->storeName, $tableSpecId);
     }
 }
diff --git a/src/mongo/delegates/Views.class.php b/src/mongo/delegates/Views.class.php
index b866ae22..ce810417 100644
--- a/src/mongo/delegates/Views.class.php
+++ b/src/mongo/delegates/Views.class.php
@@ -388,7 +388,7 @@ public function deleteViewsByViewId($viewId, $timestamp = null)
                 [\_CREATED_TS => ['$exists' => false]]
             ];
         }
-        $deleteResult = $this->config->getCollectionForView($this->storeName, $viewId)
+        $deleteResult = $this->getCollectionForViewSpec($viewId)
             ->deleteMany($query);
         return $deleteResult->getDeletedCount();
     }
diff --git a/src/mongo/providers/MongoSearchProvider.class.php b/src/mongo/providers/MongoSearchProvider.class.php
index d379d172..b26e1d06 100644
--- a/src/mongo/providers/MongoSearchProvider.class.php
+++ b/src/mongo/providers/MongoSearchProvider.class.php
@@ -357,7 +357,7 @@ public function deleteSearchDocumentsByTypeId($typeId, $timestamp = null)
                 [\_CREATED_TS => ['$exists' => false]]
             ];
         }
-        $deleteResponse = $this->config->getCollectionForSearchDocument($this->storeName, $typeId)
+        $deleteResponse = $this->getCollectionForSearchSpec($typeId)
             ->deleteMany($query);
         return $deleteResponse->getDeletedCount();
     }
@@ -382,6 +382,17 @@ protected function getSearchDocumentSpecification($typeId)
     public function count($searchSpec, array $filters = [])
     {
         $filters['_id.type'] = $searchSpec;
-        return $this->config->getCollectionForSearchDocument($this->storeName, $searchSpec)->count($filters);
+        return $this->getCollectionForSearchSpec($searchSpec)->count($filters);
+    }
+
+    /**
+     * For mocking
+     *
+     * @param string $searchSpecId Search spec ID
+     * @return \MongoDB\Collection
+     */
+    protected function getCollectionForSearchSpec($searchSpecId)
+    {
+        return $this->config->getCollectionForSearchDocument($this->storeName, $searchSpecId);
     }
 }
diff --git a/test/unit/mongo/MongoSearchProviderTest.php b/test/unit/mongo/MongoSearchProviderTest.php
index ba58509a..d2c3a87b 100644
--- a/test/unit/mongo/MongoSearchProviderTest.php
+++ b/test/unit/mongo/MongoSearchProviderTest.php
@@ -598,6 +598,160 @@ public function testDeleteSearchDocumentsByTypeIdDoNotDeleteNonMatchingDocuments
     	$this->assertEquals(1, $newSearchDocumentCount, "Should have 1 search documents since there is one search document with 'i_search_list' type that does not match delete type.");
     }
 
+    public function testCountSearchDocuments()
+    {
+        $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver')
+            ->setConstructorArgs(['CBD_testing', 'tripod_php_testing'])
+            ->getMock();
+
+        $collection = $this->getMockBuilder('\MongoDB\Collection')
+            ->disableOriginalConstructor()
+            ->setMethods(['count'])
+            ->getMock();
+        $search = $this->getMockBuilder('\Tripod\Mongo\MongoSearchProvider')
+            ->setMethods(['getCollectionForSearchSpec'])
+            ->setConstructorArgs([$tripod])
+            ->getMock();
+
+        $search->expects($this->once())
+            ->method('getCollectionForSearchSpec')
+            ->with('i_search_list')
+            ->will($this->returnValue($collection));
+
+        $collection->expects($this->once())
+            ->method('count')
+            ->with(['_id.type' => 'i_search_list'])
+            ->will($this->returnValue(21));
+
+        $this->assertEquals(21, $search->count('i_search_list'));
+    }
+
+    public function testCountSearchDocumentsWithFilters()
+    {
+        $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver')
+           ->setConstructorArgs(['CBD_testing', 'tripod_php_testing'])
+            ->getMock();
+
+        $filters = ['_cts' => ['$lte' => new \MongoDB\BSON\UTCDateTime()]];
+        $query = array_merge(['_id.type' => 'i_search_list'], $filters);
+        $collection = $this->getMockBuilder('\MongoDB\Collection')
+            ->disableOriginalConstructor()
+            ->setMethods(['count'])
+            ->getMock();
+        $search = $this->getMockBuilder('\Tripod\Mongo\MongoSearchProvider')
+            ->setMethods(['getCollectionForSearchSpec'])
+            ->setConstructorArgs([$tripod])
+            ->getMock();
+
+        $search->expects($this->once())
+            ->method('getCollectionForSearchSpec')
+            ->with('i_search_list')
+            ->will($this->returnValue($collection));
+
+        $collection->expects($this->once())
+            ->method('count')
+            ->with($query)
+            ->will($this->returnValue(89));
+
+        $this->assertEquals(89, $search->count('i_search_list', $filters));
+    }
+
+    public function testDeleteSearchDocumentsBySearchId()
+    {
+        $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver')
+            ->setConstructorArgs(['CBD_testing', 'tripod_php_testing'])
+            ->getMock();
+
+        $collection = $this->getMockBuilder('\MongoDB\Collection')
+            ->disableOriginalConstructor()
+            ->setMethods(['deleteMany'])
+            ->getMock();
+
+        $deleteResult = $this->getMockBuilder('MongoDB\DeleteResult')
+            ->setMethods(['getDeletedCount'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $deleteResult->expects($this->once())
+            ->method('getDeletedCount')
+            ->will($this->returnValue(9));
+
+        $search = $this->getMockBuilder('\Tripod\Mongo\MongoSearchProvider')
+            ->setMethods(['getCollectionForSearchSpec', 'getSearchDocumentSpecification'])
+            ->setConstructorArgs([$tripod])
+            ->getMock();
+
+        $search->expects($this->once())
+            ->method('getSearchDocumentSpecification')
+            ->with('i_search_list')
+            ->will($this->returnValue(['_id' => 'i_search_list']));
+
+        $search->expects($this->once())
+            ->method('getCollectionForSearchSpec')
+            ->with('i_search_list')
+            ->will($this->returnValue($collection));
+
+        $collection->expects($this->once())
+            ->method('deleteMany')
+            ->with(['_id.type' => 'i_search_list'])
+            ->will($this->returnValue($deleteResult));
+
+        $this->assertEquals(9, $search->deleteSearchDocumentsByTypeId('i_search_list'));
+    }
+
+    public function testDeleteSearchDocumentsBySearchIdWithTimestamp()
+    {
+        $timestamp = new \MongoDB\BSON\UTCDateTime();
+
+        $query = [
+            '_id.type' => 'i_search_list',
+            '$or' => [
+                [\_CREATED_TS => ['$lt' => $timestamp]],
+                [\_CREATED_TS => ['$exists' => false]]
+            ]
+        ];
+
+        $tripod = $this->getMockBuilder('\Tripod\Mongo\Driver')
+            ->setConstructorArgs(['CBD_testing', 'tripod_php_testing'])
+            ->getMock();
+
+        $collection = $this->getMockBuilder('\MongoDB\Collection')
+            ->disableOriginalConstructor()
+            ->setMethods(['deleteMany'])
+            ->getMock();
+
+        $deleteResult = $this->getMockBuilder('MongoDB\DeleteResult')
+            ->setMethods(['getDeletedCount'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $deleteResult->expects($this->once())
+            ->method('getDeletedCount')
+            ->will($this->returnValue(9));
+
+        $search = $this->getMockBuilder('\Tripod\Mongo\MongoSearchProvider')
+            ->setMethods(['getCollectionForSearchSpec', 'getSearchDocumentSpecification'])
+            ->setConstructorArgs([$tripod])
+            ->getMock();
+
+        $search->expects($this->once())
+            ->method('getSearchDocumentSpecification')
+            ->with('i_search_list')
+            ->will($this->returnValue(['_id' => 'i_search_list']));
+
+        $search->expects($this->once())
+            ->method('getCollectionForSearchSpec')
+            ->with('i_search_list')
+            ->will($this->returnValue($collection));
+
+        $collection->expects($this->once())
+            ->method('deleteMany')
+            ->with($query)
+            ->will($this->returnValue($deleteResult));
+
+        $this->assertEquals(9, $search->deleteSearchDocumentsByTypeId('i_search_list', $timestamp));
+    }
+
     /**
      * @param \Tripod\Mongo\Driver $tripod
      * @param array $specs
@@ -617,4 +771,4 @@ protected function getCountForSearchSpecs(\Tripod\Mongo\Driver $tripod, $specs =
         }
         return $count;
     }
-}
\ No newline at end of file
+}
diff --git a/test/unit/mongo/MongoTripodTablesTest.php b/test/unit/mongo/MongoTripodTablesTest.php
index c6e71e95..195c0612 100644
--- a/test/unit/mongo/MongoTripodTablesTest.php
+++ b/test/unit/mongo/MongoTripodTablesTest.php
@@ -1631,6 +1631,132 @@ public function testRemoveTableSpecDoesNotAffectInvalidation()
 
         // The table row should still be there, even if the tablespec no longer exists
         $this->assertGreaterThan(0, $collection->count(array('_id.type'=>'t_resource', 'value._impactIndex'=>array(_ID_RESOURCE=>$uri, _ID_CONTEXT=>$context))));
+    }
+
+    public function testCountTables()
+    {
+        $collection = $this->getMockBuilder('\MongoDB\Collection')
+            ->disableOriginalConstructor()
+            ->setMethods(['count'])
+            ->getMock();
+        $tables = $this->getMockBuilder('\Tripod\Mongo\Composites\Tables')
+            ->setMethods(['getCollectionForTableSpec'])
+            ->setConstructorArgs(['tripod_php_testing', $collection, 'http://example.com/'])
+            ->getMock();
+
+        $tables->expects($this->once())
+            ->method('getCollectionForTableSpec')
+            ->with('t_source_count')
+            ->will($this->returnValue($collection));
+
+        $collection->expects($this->once())
+            ->method('count')
+            ->with(['_id.type' => 't_source_count'])
+            ->will($this->returnValue(50));
+
+        $this->assertEquals(50, $tables->count('t_source_count'));
+    }
+
+    public function testCountTablesWithFilters()
+    {
+        $filters = ['_cts' => ['$lte' => new \MongoDB\BSON\UTCDateTime()]];
+        $query = array_merge(['_id.type' => 't_source_count'], $filters);
+        $collection = $this->getMockBuilder('\MongoDB\Collection')
+            ->disableOriginalConstructor()
+            ->setMethods(['count'])
+            ->getMock();
+        $tables = $this->getMockBuilder('\Tripod\Mongo\Composites\Tables')
+            ->setMethods(['getCollectionForTableSpec'])
+            ->setConstructorArgs(['tripod_php_testing', $collection, 'http://example.com/'])
+            ->getMock();
+
+        $tables->expects($this->once())
+            ->method('getCollectionForTableSpec')
+            ->with('t_source_count')
+            ->will($this->returnValue($collection));
+
+        $collection->expects($this->once())
+            ->method('count')
+            ->with($query)
+            ->will($this->returnValue(37));
+
+        $this->assertEquals(37, $tables->count('t_source_count', $filters));
+    }
+
+    public function testDeleteTableRowsByTableId()
+    {
+        $collection = $this->getMockBuilder('\MongoDB\Collection')
+            ->disableOriginalConstructor()
+            ->setMethods(['deleteMany'])
+            ->getMock();
+
+        $deleteResult = $this->getMockBuilder('MongoDB\DeleteResult')
+            ->setMethods(['getDeletedCount'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $deleteResult->expects($this->once())
+            ->method('getDeletedCount')
+            ->will($this->returnValue(2));
+
+        $tables = $this->getMockBuilder('\Tripod\Mongo\Composites\Tables')
+            ->setMethods(['getCollectionForTableSpec'])
+            ->setConstructorArgs(['tripod_php_testing', $collection, 'http://example.com/'])
+            ->getMock();
+
+        $tables->expects($this->once())
+            ->method('getCollectionForTableSpec')
+            ->with('t_source_count')
+            ->will($this->returnValue($collection));
+
+        $collection->expects($this->once())
+            ->method('deleteMany')
+            ->with(['_id.type' => 't_source_count'])
+            ->will($this->returnValue($deleteResult));
+
+        $this->assertEquals(2, $tables->deleteTableRowsByTableId('t_source_count'));
+    }
+
+    public function testDeleteTableRowsByTableIdWithTimestamp()
+    {
+        $timestamp = new \MongoDB\BSON\UTCDateTime();
+
+        $query = [
+            '_id.type' => 't_source_count',
+            '$or' => [
+                [\_CREATED_TS => ['$lt' => $timestamp]],
+                [\_CREATED_TS => ['$exists' => false]]
+            ]
+        ];
+        $collection = $this->getMockBuilder('\MongoDB\Collection')
+            ->disableOriginalConstructor()
+            ->setMethods(['deleteMany'])
+            ->getMock();
+
+        $deleteResult = $this->getMockBuilder('MongoDB\DeleteResult')
+            ->setMethods(['getDeletedCount'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $deleteResult->expects($this->once())
+            ->method('getDeletedCount')
+            ->will($this->returnValue(11));
+
+        $tables = $this->getMockBuilder('\Tripod\Mongo\Composites\Tables')
+            ->setMethods(['getCollectionForTableSpec'])
+            ->setConstructorArgs(['tripod_php_testing', $collection, 'http://example.com/'])
+            ->getMock();
+
+        $tables->expects($this->once())
+            ->method('getCollectionForTableSpec')
+            ->with('t_source_count')
+            ->will($this->returnValue($collection));
+
+        $collection->expects($this->once())
+            ->method('deleteMany')
+            ->with($query)
+            ->will($this->returnValue($deleteResult));
 
+        $this->assertEquals(11, $tables->deleteTableRowsByTableId('t_source_count', $timestamp));
     }
 }
diff --git a/test/unit/mongo/MongoTripodViewsTest.php b/test/unit/mongo/MongoTripodViewsTest.php
index e54ae576..65928de2 100644
--- a/test/unit/mongo/MongoTripodViewsTest.php
+++ b/test/unit/mongo/MongoTripodViewsTest.php
@@ -2024,4 +2024,131 @@ public function testCursorNoExceptionThrownWhenCursorThrowsSomeExceptions()
 
         $mockTripodViews->getViewForResources(array($uri1),$viewType,$context);
     }
+
+    public function testCountViews()
+    {
+        $collection = $this->getMockBuilder('\MongoDB\Collection')
+            ->disableOriginalConstructor()
+            ->setMethods(['count'])
+            ->getMock();
+        $views = $this->getMockBuilder('\Tripod\Mongo\Composites\Views')
+            ->setMethods(['getCollectionForViewSpec'])
+            ->setConstructorArgs(['tripod_php_testing', $collection, 'http://example.com/'])
+            ->getMock();
+
+        $views->expects($this->once())
+            ->method('getCollectionForViewSpec')
+            ->with('v_some_spec')
+            ->will($this->returnValue($collection));
+
+        $collection->expects($this->once())
+            ->method('count')
+            ->with(['_id.type' => 'v_some_spec'])
+            ->will($this->returnValue(101));
+
+        $this->assertEquals(101, $views->count('v_some_spec'));
+    }
+
+    public function testCountViewsWithFilters()
+    {
+        $filters = ['_cts' => ['$lte' => new \MongoDB\BSON\UTCDateTime()]];
+        $query = array_merge(['_id.type' => 'v_some_spec'], $filters);
+        $collection = $this->getMockBuilder('\MongoDB\Collection')
+            ->disableOriginalConstructor()
+            ->setMethods(['count'])
+            ->getMock();
+        $views = $this->getMockBuilder('\Tripod\Mongo\Composites\Views')
+            ->setMethods(['getCollectionForViewSpec'])
+            ->setConstructorArgs(['tripod_php_testing', $collection, 'http://example.com/'])
+            ->getMock();
+
+        $views->expects($this->once())
+            ->method('getCollectionForViewSpec')
+            ->with('v_some_spec')
+            ->will($this->returnValue($collection));
+
+        $collection->expects($this->once())
+            ->method('count')
+            ->with($query)
+            ->will($this->returnValue(101));
+
+        $this->assertEquals(101, $views->count('v_some_spec', $filters));
+    }
+
+    public function testDeleteViewsByViewId()
+    {
+        $collection = $this->getMockBuilder('\MongoDB\Collection')
+            ->disableOriginalConstructor()
+            ->setMethods(['deleteMany'])
+            ->getMock();
+
+        $deleteResult = $this->getMockBuilder('MongoDB\DeleteResult')
+            ->setMethods(['getDeletedCount'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $deleteResult->expects($this->once())
+            ->method('getDeletedCount')
+            ->will($this->returnValue(30));
+
+        $views = $this->getMockBuilder('\Tripod\Mongo\Composites\Views')
+            ->setMethods(['getCollectionForViewSpec'])
+            ->setConstructorArgs(['tripod_php_testing', $collection, 'http://example.com/'])
+            ->getMock();
+
+        $views->expects($this->once())
+            ->method('getCollectionForViewSpec')
+            ->with('v_resource_full')
+            ->will($this->returnValue($collection));
+
+        $collection->expects($this->once())
+            ->method('deleteMany')
+            ->with(['_id.type' => 'v_resource_full'])
+            ->will($this->returnValue($deleteResult));
+
+        $this->assertEquals(30, $views->deleteViewsByViewId('v_resource_full'));
+    }
+
+    public function testDeleteViewsByViewIdWithTimestamp()
+    {
+        $timestamp = new \MongoDB\BSON\UTCDateTime();
+
+        $query = [
+            '_id.type' => 'v_resource_full',
+            '$or' => [
+                [\_CREATED_TS => ['$lt' => $timestamp]],
+                [\_CREATED_TS => ['$exists' => false]]
+            ]
+        ];
+        $collection = $this->getMockBuilder('\MongoDB\Collection')
+            ->disableOriginalConstructor()
+            ->setMethods(['deleteMany'])
+            ->getMock();
+
+        $deleteResult = $this->getMockBuilder('MongoDB\DeleteResult')
+            ->setMethods(['getDeletedCount'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $deleteResult->expects($this->once())
+            ->method('getDeletedCount')
+            ->will($this->returnValue(30));
+
+        $views = $this->getMockBuilder('\Tripod\Mongo\Composites\Views')
+            ->setMethods(['getCollectionForViewSpec'])
+            ->setConstructorArgs(['tripod_php_testing', $collection, 'http://example.com/'])
+            ->getMock();
+
+        $views->expects($this->once())
+            ->method('getCollectionForViewSpec')
+            ->with('v_resource_full')
+            ->will($this->returnValue($collection));
+
+        $collection->expects($this->once())
+            ->method('deleteMany')
+            ->with($query)
+            ->will($this->returnValue($deleteResult));
+
+        $this->assertEquals(30, $views->deleteViewsByViewId('v_resource_full', $timestamp));
+    }
 }

From 6767c75776a59d9a52b1a35f61244e7bafd38eb1 Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Mon, 11 Dec 2017 15:41:59 -0500
Subject: [PATCH 41/42] UTCDateTime needs an argument

---
 test/unit/mongo/MongoSearchProviderTest.php | 4 ++--
 test/unit/mongo/MongoTripodTablesTest.php   | 4 ++--
 test/unit/mongo/MongoTripodViewsTest.php    | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/test/unit/mongo/MongoSearchProviderTest.php b/test/unit/mongo/MongoSearchProviderTest.php
index d2c3a87b..fba4959e 100644
--- a/test/unit/mongo/MongoSearchProviderTest.php
+++ b/test/unit/mongo/MongoSearchProviderTest.php
@@ -632,7 +632,7 @@ public function testCountSearchDocumentsWithFilters()
            ->setConstructorArgs(['CBD_testing', 'tripod_php_testing'])
             ->getMock();
 
-        $filters = ['_cts' => ['$lte' => new \MongoDB\BSON\UTCDateTime()]];
+        $filters = ['_cts' => ['$lte' => new \MongoDB\BSON\UTCDateTime(null)]];
         $query = array_merge(['_id.type' => 'i_search_list'], $filters);
         $collection = $this->getMockBuilder('\MongoDB\Collection')
             ->disableOriginalConstructor()
@@ -701,7 +701,7 @@ public function testDeleteSearchDocumentsBySearchId()
 
     public function testDeleteSearchDocumentsBySearchIdWithTimestamp()
     {
-        $timestamp = new \MongoDB\BSON\UTCDateTime();
+        $timestamp = new \MongoDB\BSON\UTCDateTime(null);
 
         $query = [
             '_id.type' => 'i_search_list',
diff --git a/test/unit/mongo/MongoTripodTablesTest.php b/test/unit/mongo/MongoTripodTablesTest.php
index 195c0612..85553575 100644
--- a/test/unit/mongo/MongoTripodTablesTest.php
+++ b/test/unit/mongo/MongoTripodTablesTest.php
@@ -1659,7 +1659,7 @@ public function testCountTables()
 
     public function testCountTablesWithFilters()
     {
-        $filters = ['_cts' => ['$lte' => new \MongoDB\BSON\UTCDateTime()]];
+        $filters = ['_cts' => ['$lte' => new \MongoDB\BSON\UTCDateTime(null)]];
         $query = array_merge(['_id.type' => 't_source_count'], $filters);
         $collection = $this->getMockBuilder('\MongoDB\Collection')
             ->disableOriginalConstructor()
@@ -1719,7 +1719,7 @@ public function testDeleteTableRowsByTableId()
 
     public function testDeleteTableRowsByTableIdWithTimestamp()
     {
-        $timestamp = new \MongoDB\BSON\UTCDateTime();
+        $timestamp = new \MongoDB\BSON\UTCDateTime(null);
 
         $query = [
             '_id.type' => 't_source_count',
diff --git a/test/unit/mongo/MongoTripodViewsTest.php b/test/unit/mongo/MongoTripodViewsTest.php
index 65928de2..918bf484 100644
--- a/test/unit/mongo/MongoTripodViewsTest.php
+++ b/test/unit/mongo/MongoTripodViewsTest.php
@@ -2051,7 +2051,7 @@ public function testCountViews()
 
     public function testCountViewsWithFilters()
     {
-        $filters = ['_cts' => ['$lte' => new \MongoDB\BSON\UTCDateTime()]];
+        $filters = ['_cts' => ['$lte' => new \MongoDB\BSON\UTCDateTime(null)]];
         $query = array_merge(['_id.type' => 'v_some_spec'], $filters);
         $collection = $this->getMockBuilder('\MongoDB\Collection')
             ->disableOriginalConstructor()
@@ -2111,7 +2111,7 @@ public function testDeleteViewsByViewId()
 
     public function testDeleteViewsByViewIdWithTimestamp()
     {
-        $timestamp = new \MongoDB\BSON\UTCDateTime();
+        $timestamp = new \MongoDB\BSON\UTCDateTime(null);
 
         $query = [
             '_id.type' => 'v_resource_full',

From 4ca73cd9058e375cbf280285acb469a7073c6f9f Mon Sep 17 00:00:00 2001
From: Ross Singer <rossfsinger@gmail.com>
Date: Thu, 14 Dec 2017 11:36:47 -0500
Subject: [PATCH 42/42] PR comments

---
 src/mongo/JobGroup.php                            | 12 +++++++-----
 src/mongo/MongoTripodConstants.php                |  2 --
 src/mongo/delegates/SearchDocuments.class.php     |  2 +-
 src/mongo/delegates/SearchIndexer.class.php       |  3 ++-
 src/mongo/jobs/ApplyOperation.class.php           |  6 ++++++
 src/mongo/providers/MongoSearchProvider.class.php |  2 +-
 test/unit/mongo/ApplyOperationTest.php            |  3 +--
 7 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/src/mongo/JobGroup.php b/src/mongo/JobGroup.php
index b4f25f4c..6b5d223d 100644
--- a/src/mongo/JobGroup.php
+++ b/src/mongo/JobGroup.php
@@ -2,6 +2,8 @@
 
 namespace Tripod\Mongo;
 
+use \MongoDB\BSON\ObjectId;
+
 class JobGroup
 {
     private $id;
@@ -11,15 +13,15 @@ class JobGroup
     /**
      * Constructor method
      * @param string                        $storeName Tripod store (database) name
-     * @param string|\MongoDB\BSON\ObjectId $groupId   Optional tracking ID, will assign a new one if omitted
+     * @param string|ObjectId $groupId   Optional tracking ID, will assign a new one if omitted
      */
     public function __construct($storeName, $groupId = null)
     {
         $this->storeName = $storeName;
         if (!$groupId) {
-            $groupId = new \MongoDB\BSON\ObjectId();
-        } elseif (!$groupId instanceof \MongoDB\BSON\ObjectId) {
-            $groupId = new \MongoDB\BSON\ObjectId($groupId);
+            $groupId = new ObjectId();
+        } elseif (!$groupId instanceof ObjectId) {
+            $groupId = new ObjectId($groupId);
         }
         $this->id = $groupId;
     }
@@ -60,7 +62,7 @@ public function incrementJobCount($inc = 1)
     }
 
     /**
-     * @return \MongoDB\BSON\ObjectId
+     * @return ObjectId
      */
     public function getId()
     {
diff --git a/src/mongo/MongoTripodConstants.php b/src/mongo/MongoTripodConstants.php
index 342d6353..81909fd0 100644
--- a/src/mongo/MongoTripodConstants.php
+++ b/src/mongo/MongoTripodConstants.php
@@ -98,8 +98,6 @@
 define('STAT_CLASS', 'tripod');
 define('STAT_PIVOT_FIELD', 'group_by_db');
 
-define('BATCH_TRACKING_GROUP', 'BATCH_TRACKING_GROUP');
-
 //Audit types, statuses
 define('AUDIT_TYPE_REMOVE_INERT_LOCKS', 'REMOVE_INERT_LOCKS');
 define('AUDIT_STATUS_IN_PROGRESS', 'IN_PROGRESS');
diff --git a/src/mongo/delegates/SearchDocuments.class.php b/src/mongo/delegates/SearchDocuments.class.php
index fa78c9a3..302a4e91 100644
--- a/src/mongo/delegates/SearchDocuments.class.php
+++ b/src/mongo/delegates/SearchDocuments.class.php
@@ -111,7 +111,7 @@ public function generateSearchDocumentBasedOnSpecId($specId, $resource, $context
         $this->debugLog("Processing {$specId}");
 
         // build the document
-        $generatedDocument = [\_CREATED_TS => \Tripod\Mongo\DateUtil::getMongoDate()];
+        $generatedDocument = [\_CREATED_TS => DateUtil::getMongoDate()];
         $this->addIdToImpactIndex($_id, $generatedDocument);
 
         $_id['type'] = $specId;
diff --git a/src/mongo/delegates/SearchIndexer.class.php b/src/mongo/delegates/SearchIndexer.class.php
index 73218a11..eeb3231c 100644
--- a/src/mongo/delegates/SearchIndexer.class.php
+++ b/src/mongo/delegates/SearchIndexer.class.php
@@ -168,6 +168,7 @@ public function generateAndIndexSearchDocuments($resourceUri, $context, $podName
      * @param string|null $resourceUri
      * @param string|null $context
      * @param string|null $queueName
+     * @return array|null Will return an array with a count and group id, if $queueName is sent and $resourceUri is null
      */
     public function generateSearchDocuments($searchDocumentType, $resourceUri=null, $context=null, $queueName=null)
     {
@@ -213,7 +214,7 @@ public function generateSearchDocuments($searchDocumentType, $resourceUri=null,
         ));
 
         $jobOptions = [];
-        if ($queueName && !$resource) {
+        if ($queueName && !$resourceUri) {
             $jobOptions['statsConfig'] = $this->getStatsConfig();
             $jobGroup = new JobGroup($this->storeName);
             $jobOptions[ApplyOperation::TRACKING_KEY] = $jobGroup->getId()->__toString();
diff --git a/src/mongo/jobs/ApplyOperation.class.php b/src/mongo/jobs/ApplyOperation.class.php
index 95c9b8ef..318cfec1 100644
--- a/src/mongo/jobs/ApplyOperation.class.php
+++ b/src/mongo/jobs/ApplyOperation.class.php
@@ -51,6 +51,12 @@ public function perform()
                 // stat time taken to perform operation for the given subject
                 $this->getStat()->timer(MONGO_QUEUE_APPLY_OPERATION.'.'.$subject['operation'], $opTimer->result());
 
+                /**
+                 * ApplyOperation jobs can either apply to a single resource (e.g. 'create composite for the given
+                 * resource uri) or for a specification id (i.e. regenerate all of the composites defined by the
+                 * specification).  For the latter, we need to keep track of how many jobs have run so we can clean
+                 * up any stale composite documents when completed.  The TRACKING_KEY value will be the JobGroup id.
+                 */
                 if (isset($this->args[self::TRACKING_KEY])) {
                     $jobGroup = $this->getJobGroup($subject['storeName'], $this->args[self::TRACKING_KEY]);
                     $jobCount = $jobGroup->incrementJobCount(-1);
diff --git a/src/mongo/providers/MongoSearchProvider.class.php b/src/mongo/providers/MongoSearchProvider.class.php
index b26e1d06..d4ffef47 100644
--- a/src/mongo/providers/MongoSearchProvider.class.php
+++ b/src/mongo/providers/MongoSearchProvider.class.php
@@ -338,7 +338,7 @@ public function getSearchCollectionName()
      * If type id is not specified this method will throw an exception.
      * @param string                         $typeId    Search type id
      * @param \MongoDB\BSON\UTCDateTime|null $timestamp Optional timestamp to delete all search docs that are older than
-     * @return integer                                  The number of search documnts deleted
+     * @return integer                                  The number of search documents deleted
      * @throws \Tripod\Exceptions\Exception if there was an error performing the operation
      */
     public function deleteSearchDocumentsByTypeId($typeId, $timestamp = null)
diff --git a/test/unit/mongo/ApplyOperationTest.php b/test/unit/mongo/ApplyOperationTest.php
index 6829a666..a5741a8b 100644
--- a/test/unit/mongo/ApplyOperationTest.php
+++ b/test/unit/mongo/ApplyOperationTest.php
@@ -275,8 +275,7 @@ public function testApplyViewOperationCleanupIfAllGroupJobsComplete()
             ->method('createImpactedSubject')
             ->will($this->returnValue($subject));
 
-        // $applyOperation->expects($this->exactly(3))
-        $applyOperation->expects($this->any())
+        $applyOperation->expects($this->exactly(3))
             ->method('getStat')
             ->will($this->returnValue($statMock));