Merge pull request #95 from edx/yro/update_logging

Yro/update logging

Author: yro

VEDA/urls.py

@@ -4,9 +4,9 @@
 sys.path.append(os.path.abspath(__file__))
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'VEDA.settings.local')
 
+# pep8: disable=E402
 from django.conf import settings
 from rest_framework import routers
-# from rest_framework.routers import DefaultRouter
 from django.conf.urls import patterns, include, url
 from django.contrib import admin
 

VEDA_OS01/fixtures/destinations.json

@@ -56,15 +56,15 @@
     "fields": {
       "course_name": "Veda Sandbox Test Course",
       "course_hold": true,
-      "institution": "XXX",
-      "edx_classid": "XXXXX",
+      "institution": "EDX",
+      "edx_classid": "DEMOX",
       "semesterid": "2017",
       "yt_proc": false,
       "tp_proc": false,
       "c24_proc": false,
       "s3_proc": true,
       "local_storedir": "course-v1:VEDA+VEDA201+2015_T1",
-      "studio_hex": "xxxx"
+      "studio_hex": "shared_course_token"
     }
   }
 ]

VEDA_OS01/fixtures/encodes.json

@@ -160,5 +160,18 @@
             "encode_resolution": "720",
             "product_spec": "desktop_webm"
         }
+    },
+    {
+        "model": "VEDA_OS01.course",
+        "pk": 1,
+        "fields": {
+            "course_name": "Demo Course",
+            "institution": "EDX",
+            "edx_classid": "DEMOX",
+            "semesterid": 2018,
+            "yt_proc": false,
+            "local_storedir": "this/is/an/exemplar",
+            "studio_hex": "shared_course_token"
+        }
     }
 ]

bin/deliver

@@ -9,7 +9,7 @@ if project_path not in sys.path:
     sys.path.append(project_path)
 
 
-class DeliverCli:
+class DeliverCli(object):
     """
     Deliver
 

bin/heal

@@ -4,11 +4,12 @@ Deliver
 
 Command Line Interface
 """
-import os
-import sys
 import argparse
 import datetime
 from datetime import timedelta
+import logging
+import os
+import sys
 import pytz
 
 project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -21,8 +22,12 @@ from VEDA_OS01.models import Course, Video
 from VEDA_OS01.transcripts import retrieve_three_play_translations
 from VEDA.utils import get_config
 
+LOGGER = logging.getLogger(__name__)
+# TODO: Remove this temporary logging to stdout
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
 
-class HealCli:
+class HealCli(object):
 
     def __init__(self, **kwargs):
         self.logging = kwargs.get('logging', True)
@@ -93,8 +98,8 @@ def main():
         retrieve_three_play_translations()
         return
 
-    print '%s - %s: %s' % ('Healing', 'VEDA ID', veda_id)
-    print '%s - %s: %s' % ('Healing', 'Course', course_id)
+    LOGGER.info('%s - %s: %s' % ('Healing', 'VEDA ID', veda_id))
+    LOGGER.info('%s - %s: %s' % ('Healing', 'Course', course_id))
 
     if veda_id is None and course_id is None and schedule is False:
         VH = VedaHeal()

bin/ingest

@@ -1,4 +1,10 @@
 #!/usr/bin/env python
+"""
+Ingest
+
+Command Line Interface
+"""
+
 import os
 import sys
 import argparse
@@ -9,13 +15,8 @@ if project_path not in sys.path:
 
 from control.veda_utils import EmailAlert
 
-"""
-Ingest
-
-Command Line Interface
-"""
 
-class IngestCli():
+class IngestCli(object):
 
     def __init__(self, **kwargs):
         self.args = None

bin/loop.py

@@ -1,8 +1,13 @@
 #!/usr/bin/env python
+"""
+This is a cheapo way to get a pager (using SES)
+
+"""
 
 import os
 import sys
 import argparse
+import logging
 from django.db import reset_queries
 import resource
 import time
@@ -13,20 +18,19 @@
 if project_path not in sys.path:
     sys.path.append(project_path)
 
-"""
-This is a cheapo way to get a pager (using SES)
-
-"""
-
 import django
 django.setup()
 
 from control.veda_file_discovery import FileDiscovery
 from youtube_callback.daemon import generate_course_list
 from youtube_callback.sftp_id_retrieve import callfunction
 
+LOGGER = logging.getLogger(__name__)
+# TODO: Remove this temporary logging to stdout
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+
 
-class DaemonCli:
+class DaemonCli(object):
 
     def __init__(self):
         self.args = None
@@ -80,20 +84,20 @@ def ingest_daemon(self):
             reset_queries()
             x += 1
             if x >= 100:
-                print 'Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+                LOGGER.info('Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
                 x = 0
 
     def youtube_daemon(self):
         x = 0
         while True:
             self.course_list = generate_course_list()
             for course in self.course_list:
-                print "%s%s: Callback" % (course.institution, course.edx_classid)
+                LOGGER.info('%s%s: Callback' % (course.institution, course.edx_classid))
                 callfunction(course)
 
             x += 1
             if x >= 100:
-                print 'Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
+                LOGGER.info('Memory usage: %s (kb)' % resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
                 x = 0
 
             reset_queries()

bin/youtubecallback

@@ -18,7 +18,7 @@ Command Line Interface
 """
 
 
-class YoutubeCallbackCli():
+class YoutubeCallbackCli(object):
 
     def __init__(self, **kwargs):
         self.args = None
@@ -54,14 +54,10 @@ class YoutubeCallbackCli():
 
     def _parse_args(self):
         self.course_id = self.args.courseid
-        self.list = self.args.list
 
     def run(self):
 
-        if self.list is True:
-            self.listcourses()
-        else:
-            self.loop()
+        self.loop()
 
     def loop(self):
         """
@@ -82,16 +78,6 @@ class YoutubeCallbackCli():
         E1 = EmailAlert(message='Youtube Callback Daemon Crash', subject='Youtube Callback Daemon')
         E1.email()
 
-    def listcourses(self):
-        """
-        list and exit
-        :return:
-        """
-        self.course_list = generate_course_list()
-        for course in self.course_list:
-            print course.institution
-            print course.edx_classid
-
 
 def main():
     YTCC = YoutubeCallbackCli()

control/celeryapp.py

@@ -4,15 +4,20 @@
 
 from __future__ import absolute_import
 
-import os
 from celery import Celery
-from VEDA.utils import get_config
+import logging
+import os
+import sys
 
+from VEDA.utils import get_config
 try:
     from control.veda_deliver import VedaDelivery
 except ImportError:
     from veda_deliver import VedaDelivery
 
+LOGGER = logging.getLogger(__name__)
+# TODO: Remove this temporary logging to stdout
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
 
 auth_dict = get_config()
 
@@ -42,7 +47,7 @@
 
 @app.task(name='worker_encode')
 def worker_task_fire(veda_id, encode_profile, jobid):
-    print '[ENCODE] Misfire : {id} : {encode}'.format(id=veda_id, encode=encode_profile)
+    LOGGER.info('[ENCODE] Misfire : {id} : {encode}'.format(id=veda_id, encode=encode_profile))
     return 1
 
 

control/old_veda_deliver_cielo.py

@@ -1,7 +1,3 @@
-import requests
-import ast
-import urllib
-
 """
 Cielo24 API Job Start and Download
 Options (reflected in Course.models):
@@ -14,10 +10,17 @@
   priority (48h)
 turnaround_hours = number, overrides 'priority' call, will change a standard to a priority silently
 """
+import logging
+import requests
+import ast
+import urllib
+
 from control_env import *
-from veda_utils import ErrorObject
 
 requests.packages.urllib3.disable_warnings()
+LOGGER = logging.getLogger(__name__)
+# TODO: Remove this temporary logging to stdout
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
 
 
 class Cielo24TranscriptOld(object):
@@ -77,11 +80,9 @@ def retrieve_defaults(self):
             videoID=video_query,
             encode_url__icontains='_DTH.mp4',
         ).latest()
-        
+
         if video_query.inst_class.c24_username is None:
-            ErrorObject.print_error(
-                message='Cielo24 Record Incomplete',
-            )
+            LOGGER.error('[VIDEO_PIPELINE] {id} : Cielo API : Course record incomplete'.format(id=self.veda_id))
             return None
 
         c24_defaults = {
@@ -102,10 +103,8 @@ def tokengenerator(self):
         # Generate Token
         r1 = requests.get(token_url)
         if r1.status_code > 299:
-            ErrorObject.print_error(
-                message='Cielo24 API Access Error',
-            )
-            return None
+            LOGGER.error('[VIDEO_PIPELINE] {id} : Cielo API access'.format(id=self.veda_id))
+            return
         api_token = ast.literal_eval(r1.text)["ApiToken"]
         return api_token
 
@@ -161,7 +160,6 @@ def embed_url(self, api_token, job_id):
                 urllib.quote_plus(self.c24_defaults['url'])
             ))
         )
-        print str(r4.status_code) + ' : Cielo24 Status Code'
         return ast.literal_eval(r4.text)['TaskId']
 
 

control/tests/test_encode.py

@@ -81,30 +81,3 @@ def test_encode_url(self):
         ).delete()
         encode_list = self.E.determine_encodes()
         self.assertTrue(len(encode_list) == baseline)
-
-
-def main():
-    unittest.main()
-
-
-if __name__ == '__main__':
-    sys.exit(main())
-
-    '''
-    Save for poss future test
-
-    # import celeryapp
-
-    # co = Course.objects.get(institution='XXX', edx_classid='C93BC')
-    # vid = 'XXXC93BC2016-V003500'
-    # v = VedaEncode(course_object=co, veda_id=vid)
-    # encode_list = v.determine_encodes()
-    # for e in encode_list:
-    #     veda_id = vid
-    #     encode_profile = e
-    #     jobid = uuid.uuid1().hex[0:10]
-    #     # celeryapp.worker_task_fire.apply_async(
-    #     #     (veda_id, encode_profile, jobid),
-    #     #     queue='encode_worker'
-    #     #     )
-    '''

control/tests/test_file_discovery.py

@@ -169,22 +169,6 @@ def test_discover_studio_ingested_videos(self, mock_validate_and_feed_to_ingest)
             file_discovery_instance.discover_studio_ingested_videos()
             self.assertTrue(mock_validate_and_feed_to_ingest.called)
 
-    @ddt.data(
-        ('veda/working', '[File Ingest] S3 Ingest Connection Failure'),
-        (None, '[File Ingest] No Working Node directory')
-    )
-    @ddt.unpack
-    @patch('control.veda_file_discovery.ErrorObject.print_error')
-    @patch('boto.s3.connection.S3Connection')
-    def test_discover_studio_ingested_video_exceptions(self, work_dir, error_message, mocked_s3_conn, mock_error):
-        """
-        Tests 'FileDiscovery.discover_studio_ingested_videos' exception cases.
-        """
-        mocked_s3_conn.side_effect = S3ResponseError('Error', 'Timeout')
-        file_discovery_instance = FileDiscovery(node_work_directory=work_dir)
-        file_discovery_instance.discover_studio_ingested_videos()
-        mock_error.assert_called_with(message=error_message)
-
     @ddt.data(
         (None, 'invalid_course_key'),
         ('non-existent-hex', None)