Merge pull request #79 from talis/fix-resque-retry

90% - Detects/Retries a failed request to resque

src/exceptions/JobException.class.php

@@ -0,0 +1,13 @@
+<?php
+
+namespace Tripod\Exceptions;
+
+// @codeCoverageIgnoreStart
+
+/**
+ * Class SearchException
+ * @package Tripod
+ */
+class JobException extends Exception {
+
+} // @codeCoverageIgnoreEnd

src/mongo/Config.class.php

@@ -2061,10 +2061,14 @@ public static function getResqueServer()
         if (empty($resqueServer))
         {
             $resqueServer = self::getenv(MONGO_TRIPOD_RESQUE_SERVER,'');
-            self::getLogger()->addNotice("Use of MONGO_TRIPOD_RESQUE_SERVER is deprecated - use RESQUE_SERVER instead");
+            if (!empty($resqueServer))
+            {
+                self::getLogger()->addNotice("Use of MONGO_TRIPOD_RESQUE_SERVER is deprecated - use RESQUE_SERVER instead");
+            }
         }
         if (empty($resqueServer))
         {
+            self::getLogger()->addWarning("RESQUE_SERVER is missing from environment - using localhost:6379 instead");
             $resqueServer = "localhost:6379";
         }
         return $resqueServer;

src/mongo/base/JobBase.class.php

@@ -1,6 +1,9 @@
 <?php
 
 namespace Tripod\Mongo\Jobs;
+use Tripod\Exceptions\Exception;
+use Tripod\Exceptions\JobException;
+
 /**
  * Todo: How to inject correct stat class... :-S
  */
@@ -76,10 +79,64 @@ public function errorLog($message, $params = null)
      * @param string $queueName
      * @param string $class
      * @param array $data
+     * @param int $retryAttempts if queue fails, retry x times before throwing an exception
+     * @return a tracking token for the submitted job
+     * @throws JobException if there is a problem queuing the job
+     */
+    protected function submitJob($queueName, $class, Array $data, $retryAttempts=5)
+    {
+        // @see https://github.com/chrisboulton/php-resque/issues/228, when this PR is merged we can stop tracking the status in this way
+        try
+        {
+            $token = $this->enqueue($queueName, $class, $data);
+            if(!$this->getJobStatus($token))
+            {
+                $this->errorLog("Could not retrieve status for queued $class job - job $token failed to $queueName");
+                throw new \Exception("Could not retrieve status for queued job - job $token failed to $queueName");
+            }
+            else
+            {
+                $this->debugLog("Queued $class job with $token to $queueName");
+                return $token;
+            }
+        }
+        catch (\Exception $e)
+        {
+            if ($retryAttempts>0)
+            {
+                sleep(1); // back off for 1 sec
+                $this->warningLog("Exception queuing $class job - {$e->getMessage()}, retrying $retryAttempts times");
+                return $this->submitJob($queueName,$class,$data,--$retryAttempts);
+            }
+            else
+            {
+                $this->errorLog("Exception queuing $class job - {$e->getMessage()}");
+                throw new JobException("Exception queuing job  - {$e->getMessage()}",$e->getCode(),$e);
+            }
+        }
+    }
+
+    /**
+     * Actually enqueues the job with Resque. Returns a tracking token. For mocking.
+     * @param $queueName
+     * @param $class
+     * @param $data
+     * @param bool|false $tracking
+     * @return string
+     */
+    protected function enqueue($queueName, $class, $data)
+    {
+        return \Resque::enqueue($queueName, $class, $data, true);
+    }
+
+    /**
+     * Given a token, return the job status. For mocking
+     * @param $token
      */
-    protected function submitJob($queueName, $class, Array $data)
+    protected function getJobStatus($token)
     {
-        \Resque::enqueue($queueName, $class, $data);
+        $status = new \Resque_Job_Status($token);
+        return $status->get();
     }
 }
 

src/tripod.inc.php

@@ -11,6 +11,7 @@
 require_once TRIPOD_DIR . 'exceptions/ConfigException.class.php';
 require_once TRIPOD_DIR . 'exceptions/LabellerException.class.php';
 require_once TRIPOD_DIR . 'exceptions/ViewException.class.php';
+require_once TRIPOD_DIR . 'exceptions/JobException.class.php';
 require_once TRIPOD_DIR.'mongo/MongoTripodConstants.php';
 require_once TRIPOD_DIR.'mongo/MongoGraph.class.php';
 require_once TRIPOD_DIR.'mongo/ImpactedSubject.class.php';

test/unit/mongo/ApplyOperationTest.php

@@ -252,6 +252,78 @@ public function testCreateJobDefaultQueue()
         $applyOperation->createJob(array($impactedSubject));
     }
 
+    public function testCreateJobUnreachableRedis()
+    {
+        $impactedSubject = new \Tripod\Mongo\ImpactedSubject(
+            array(_ID_RESOURCE=>"http://example.com/1",_ID_CONTEXT=>"http://talisaspire.com/"),
+            OP_TABLES,
+            'tripod_php_testing',
+            'CBD_testing',
+            array('t_resource','t_resource_count')
+        );
+
+        /** @var \Tripod\Mongo\Jobs\ApplyOperation|PHPUnit_Framework_MockObject_MockObject $applyOperation */
+        $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation')
+            ->setMethods(array('warningLog','enqueue'))
+            ->getMock();
+
+        $e = new Exception("Connection to Redis failed after 1 failures.Last Error : (0) php_network_getaddresses: getaddrinfo failed: nodename nor servname provided, or not known");
+        $applyOperation->expects($this->any())->method("enqueue")->will($this->throwException($e));
+
+        // expect 5 retries. Catch this with call to warning log
+        $applyOperation->expects($this->exactly(5))->method("warningLog");
+
+        $exceptionThrown = false;
+        try
+        {
+            $applyOperation->createJob(array($impactedSubject));
+        }
+        catch (\Tripod\Exceptions\JobException $e)
+        {
+            $this->assertEquals('Exception queuing job  - Connection to Redis failed after 1 failures.Last Error : (0) php_network_getaddresses: getaddrinfo failed: nodename nor servname provided, or not known',$e->getMessage());
+            $exceptionThrown = true;
+        }
+        if (!$exceptionThrown) {
+            $this->fail("Did not throw JobException");
+        }
+    }
+
+    public function testCreateJobStatusFalse()
+    {
+        $impactedSubject = new \Tripod\Mongo\ImpactedSubject(
+            array(_ID_RESOURCE=>"http://example.com/1",_ID_CONTEXT=>"http://talisaspire.com/"),
+            OP_TABLES,
+            'tripod_php_testing',
+            'CBD_testing',
+            array('t_resource','t_resource_count')
+        );
+
+        /** @var \Tripod\Mongo\Jobs\ApplyOperation|PHPUnit_Framework_MockObject_MockObject $applyOperation */
+        $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation')
+            ->setMethods(array('enqueue','getJobStatus','warningLog'))
+            ->getMock();
+
+        $applyOperation->expects($this->any())->method("enqueue")->will($this->returnValue("sometoken"));
+        $applyOperation->expects($this->any())->method("getJobStatus")->will($this->returnValue(false));
+
+        // expect 5 retries. Catch this with call to warning log
+        $applyOperation->expects($this->exactly(5))->method("warningLog");
+
+        $exceptionThrown = false;
+        try
+        {
+            $applyOperation->createJob(array($impactedSubject));
+        }
+        catch (\Tripod\Exceptions\JobException $e)
+        {
+            $this->assertEquals('Exception queuing job  - Could not retrieve status for queued job - job sometoken failed to tripod::apply',$e->getMessage());
+            $exceptionThrown = true;
+        }
+        if (!$exceptionThrown) {
+            $this->fail("Did not throw JobException");
+        }
+    }
+
     public function testCreateJobSpecifyQueue()
     {
         $impactedSubject = new \Tripod\Mongo\ImpactedSubject(