diff --git a/src/exceptions/JobException.class.php b/src/exceptions/JobException.class.php new file mode 100644 index 00000000..cadfca85 --- /dev/null +++ b/src/exceptions/JobException.class.php @@ -0,0 +1,13 @@ +addNotice("Use of MONGO_TRIPOD_RESQUE_SERVER is deprecated - use RESQUE_SERVER instead"); + if (!empty($resqueServer)) + { + self::getLogger()->addNotice("Use of MONGO_TRIPOD_RESQUE_SERVER is deprecated - use RESQUE_SERVER instead"); + } } if (empty($resqueServer)) { + self::getLogger()->addWarning("RESQUE_SERVER is missing from environment - using localhost:6379 instead"); $resqueServer = "localhost:6379"; } return $resqueServer; diff --git a/src/mongo/base/JobBase.class.php b/src/mongo/base/JobBase.class.php index d8b9f51f..0def636e 100644 --- a/src/mongo/base/JobBase.class.php +++ b/src/mongo/base/JobBase.class.php @@ -1,6 +1,9 @@ enqueue($queueName, $class, $data); + if(!$this->getJobStatus($token)) + { + $this->errorLog("Could not retrieve status for queued $class job - job $token failed to $queueName"); + throw new \Exception("Could not retrieve status for queued job - job $token failed to $queueName"); + } + else + { + $this->debugLog("Queued $class job with $token to $queueName"); + return $token; + } + } + catch (\Exception $e) + { + if ($retryAttempts>0) + { + sleep(1); // back off for 1 sec + $this->warningLog("Exception queuing $class job - {$e->getMessage()}, retrying $retryAttempts times"); + return $this->submitJob($queueName,$class,$data,--$retryAttempts); + } + else + { + $this->errorLog("Exception queuing $class job - {$e->getMessage()}"); + throw new JobException("Exception queuing job - {$e->getMessage()}",$e->getCode(),$e); + } + } + } + + /** + * Actually enqueues the job with Resque. Returns a tracking token. For mocking. + * @param $queueName + * @param $class + * @param $data + * @param bool|false $tracking + * @return string + */ + protected function enqueue($queueName, $class, $data) + { + return \Resque::enqueue($queueName, $class, $data, true); + } + + /** + * Given a token, return the job status. For mocking + * @param $token */ - protected function submitJob($queueName, $class, Array $data) + protected function getJobStatus($token) { - \Resque::enqueue($queueName, $class, $data); + $status = new \Resque_Job_Status($token); + return $status->get(); } } diff --git a/src/tripod.inc.php b/src/tripod.inc.php index 25bd9e06..a63e7b12 100644 --- a/src/tripod.inc.php +++ b/src/tripod.inc.php @@ -11,6 +11,7 @@ require_once TRIPOD_DIR . 'exceptions/ConfigException.class.php'; require_once TRIPOD_DIR . 'exceptions/LabellerException.class.php'; require_once TRIPOD_DIR . 'exceptions/ViewException.class.php'; +require_once TRIPOD_DIR . 'exceptions/JobException.class.php'; require_once TRIPOD_DIR.'mongo/MongoTripodConstants.php'; require_once TRIPOD_DIR.'mongo/MongoGraph.class.php'; require_once TRIPOD_DIR.'mongo/ImpactedSubject.class.php'; diff --git a/test/unit/mongo/ApplyOperationTest.php b/test/unit/mongo/ApplyOperationTest.php index d8ebe834..09624e82 100644 --- a/test/unit/mongo/ApplyOperationTest.php +++ b/test/unit/mongo/ApplyOperationTest.php @@ -252,6 +252,78 @@ public function testCreateJobDefaultQueue() $applyOperation->createJob(array($impactedSubject)); } + public function testCreateJobUnreachableRedis() + { + $impactedSubject = new \Tripod\Mongo\ImpactedSubject( + array(_ID_RESOURCE=>"http://example.com/1",_ID_CONTEXT=>"http://talisaspire.com/"), + OP_TABLES, + 'tripod_php_testing', + 'CBD_testing', + array('t_resource','t_resource_count') + ); + + /** @var \Tripod\Mongo\Jobs\ApplyOperation|PHPUnit_Framework_MockObject_MockObject $applyOperation */ + $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation') + ->setMethods(array('warningLog','enqueue')) + ->getMock(); + + $e = new Exception("Connection to Redis failed after 1 failures.Last Error : (0) php_network_getaddresses: getaddrinfo failed: nodename nor servname provided, or not known"); + $applyOperation->expects($this->any())->method("enqueue")->will($this->throwException($e)); + + // expect 5 retries. Catch this with call to warning log + $applyOperation->expects($this->exactly(5))->method("warningLog"); + + $exceptionThrown = false; + try + { + $applyOperation->createJob(array($impactedSubject)); + } + catch (\Tripod\Exceptions\JobException $e) + { + $this->assertEquals('Exception queuing job - Connection to Redis failed after 1 failures.Last Error : (0) php_network_getaddresses: getaddrinfo failed: nodename nor servname provided, or not known',$e->getMessage()); + $exceptionThrown = true; + } + if (!$exceptionThrown) { + $this->fail("Did not throw JobException"); + } + } + + public function testCreateJobStatusFalse() + { + $impactedSubject = new \Tripod\Mongo\ImpactedSubject( + array(_ID_RESOURCE=>"http://example.com/1",_ID_CONTEXT=>"http://talisaspire.com/"), + OP_TABLES, + 'tripod_php_testing', + 'CBD_testing', + array('t_resource','t_resource_count') + ); + + /** @var \Tripod\Mongo\Jobs\ApplyOperation|PHPUnit_Framework_MockObject_MockObject $applyOperation */ + $applyOperation = $this->getMockBuilder('\Tripod\Mongo\Jobs\ApplyOperation') + ->setMethods(array('enqueue','getJobStatus','warningLog')) + ->getMock(); + + $applyOperation->expects($this->any())->method("enqueue")->will($this->returnValue("sometoken")); + $applyOperation->expects($this->any())->method("getJobStatus")->will($this->returnValue(false)); + + // expect 5 retries. Catch this with call to warning log + $applyOperation->expects($this->exactly(5))->method("warningLog"); + + $exceptionThrown = false; + try + { + $applyOperation->createJob(array($impactedSubject)); + } + catch (\Tripod\Exceptions\JobException $e) + { + $this->assertEquals('Exception queuing job - Could not retrieve status for queued job - job sometoken failed to tripod::apply',$e->getMessage()); + $exceptionThrown = true; + } + if (!$exceptionThrown) { + $this->fail("Did not throw JobException"); + } + } + public function testCreateJobSpecifyQueue() { $impactedSubject = new \Tripod\Mongo\ImpactedSubject(