Merge pull request #67 from edx/yro/refine_heal_add_logging

Refine HEAL process, make logging louder

Author: yro

bin/deliver

@@ -61,7 +61,7 @@ class DeliverCli:
                     'worker',
                     '--loglevel=info',
                     '--concurrency=' + str(auth_dict['celery_threads']),
-                    '-Q ' + auth_dict['celery_deliver_queue'],
+                    '-Q ' + auth_dict['celery_deliver_queue'] + ',' + auth_dict['celery_heal_queue'],
                     '-n deliver.%h'
                 ))
             )

bin/heal

@@ -34,7 +34,7 @@ class HealCli:
                       .replace(hour=0, minute=0, second=0, microsecond=0) \
                       .astimezone(pytz.utc) + timedelta(days=1)
 
-        maintainer_healer.apply_async((self.binscript,), queue='transcode_stat', eta=go_time)
+        maintainer_healer.apply_async((self.binscript,), queue=auth_dict['celery_heal_queue'], eta=go_time)
 
 
 def main():

control/veda_deliver.py

@@ -8,6 +8,7 @@
 import logging
 import shutil
 from os.path import expanduser
+import sys
 
 import boto
 import boto.s3
@@ -35,6 +36,9 @@
     from veda_deliver_3play import ThreePlayMediaClient
 
 LOGGER = logging.getLogger(__name__)
+# TODO: Remove this temporary logging to stdout
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
 try:
     boto.config.add_section('Boto')
 except:
@@ -66,6 +70,7 @@ def run(self):
         Check the destination, route via available methods,
         throw error if method is not extant
         """
+        LOGGER.info('[VIDEO_DELIVER] {video_id} : {encode}'.format(video_id=self.veda_id, encode=self.encode_profile))
         if self.encode_profile == 'hls':
             # HLS encodes are a pass through
             self.hls_run()

control/veda_heal.py

@@ -9,6 +9,7 @@
 """
 import datetime
 from datetime import timedelta
+import logging
 import os
 import sys
 import uuid
@@ -27,6 +28,10 @@
 
 time_safetygap = datetime.datetime.utcnow().replace(tzinfo=utc) - timedelta(days=1)
 
+LOGGER = logging.getLogger(__name__)
+# TODO: Remove this temporary logging to stdout
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
 
 class VedaHeal(object):
     """
@@ -87,6 +92,7 @@ def determine_fault(self, video_object):
         """
         Determine expected and completed encodes
         """
+        LOGGER.info('[HEAL] : {id}'.format(id=video_object.edx_id))
         if self.freezing_bug is True:
             if video_object.video_trans_status == 'Corrupt File':
                 return []
@@ -116,8 +122,11 @@ def determine_fault(self, video_object):
         except KeyError:
             pass
 
-        # list comparison
-        return self.differentiate_encodes(uncompleted_encodes, expected_encodes, video_object)
+        requeued_encodes = self.differentiate_encodes(uncompleted_encodes, expected_encodes, video_object)
+        LOGGER.info(
+            '[HEAL] : Status: {status}, Encodes: {encodes}'.format(status=self.val_status, encodes=requeued_encodes)
+        )
+        return requeued_encodes
 
     def differentiate_encodes(self, uncompleted_encodes, expected_encodes, video_object):
         """

instance_config.yaml

@@ -105,7 +105,7 @@ val_username:
 val_transcript_create_url:
 val_video_transcript_status_url:
 
-
+# Celery Worker Config Information
 rabbitmq_broker:
 rabbitmq_pass:
 rabbitmq_user:

static_config.yaml

@@ -2,11 +2,12 @@
 # This configuration should only have static settings.
 
 # Celery Info
+onsite_worker: False
+celery_threads: 1
 celery_app_name: veda_production
-# can do multiple queues like so: foo,bar,baz
-celery_worker_queue: encode_worker
+celery_worker_queue: encode_worker,large_encode_worker
 celery_deliver_queue: deliver_worker
-celery_threads: 1
+celery_heal_queue: heal_queue
 
 # S3 upload settings
 multi_upload_barrier: 2000000000