Merge pull request #70 from edx/yro/release_bugfix

yro · web-flow · commit f579279ec988 · 2017-12-06T07:23:01.000-05:00
1.2.0 Release Debug
diff --git a/bin/heal b/bin/heal
@@ -1,4 +1,9 @@
 #!/usr/bin/env python
+"""
+Deliver
+
+Command Line Interface
+"""
 import os
 import sys
 import argparse
@@ -14,12 +19,7 @@ from control.celeryapp import maintainer_healer
 from control.veda_heal import VedaHeal
 from VEDA_OS01.models import Course, Video
 from VEDA_OS01.transcripts import retrieve_three_play_translations
-
-"""
-Deliver
-
-Command Line Interface
-"""
+from VEDA.utils import get_config
 
 
 class HealCli:
@@ -30,6 +30,10 @@ class HealCli:
         self.binscript = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'heal')
 
     def schedule(self):
+        """
+        Reschedule Heal process via Celery ETA
+        """
+        auth_dict = get_config()
         go_time = datetime.datetime.now(pytz.timezone("America/New_York")) \
                       .replace(hour=0, minute=0, second=0, microsecond=0) \
                       .astimezone(pytz.utc) + timedelta(days=1)
diff --git a/control/tests/test_heal.py b/control/tests/test_heal.py
@@ -125,7 +125,7 @@ def test_heal(self):
             'video_active': True,
         },
         {
-            'edx_id': '1',
+            'edx_id': '2',
             'video_trans_status': 'Complete',
             'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),
             'video_active': False,
diff --git a/control/veda_file_discovery.py b/control/veda_file_discovery.py
@@ -1,69 +1,59 @@
+"""
+multi-point videofile discovery
+Currently:
+    Amazon S3 (studio-ingest as well as about/marketing
+        video ingest
+        )
+    Local (watchfolder w/o edit priv.)
+
+"""
 
 import json
 import logging
+import os.path
+
 import boto
 import boto.s3
-from boto.exception import S3ResponseError, S3DataError
-
+from boto.exception import NoAuthHandlerFound, S3DataError, S3ResponseError
 from opaque_keys import InvalidKeyError
 from opaque_keys.edx.keys import CourseKey
 
+from control_env import *
 from VEDA.utils import extract_course_org, get_config
+from veda_file_ingest import VedaIngest, VideoProto
 from VEDA_OS01.models import TranscriptCredentials
+from veda_utils import ErrorObject
+from veda_val import VALAPICall
 
 try:
     boto.config.add_section('Boto')
 except:
     pass
 boto.config.set('Boto', 'http_socket_timeout', '100')
 
-"""
-multi-point videofile discovery
-Currently:
-    FTP
-    Amazon S3 (studio-ingest as well as about/marketing
-        video ingest
-        )
-    Local (watchfolder w/o edit priv.)
-
-"""
-from control_env import *
-from veda_utils import ErrorObject
-from veda_file_ingest import VideoProto, VedaIngest
-from veda_val import VALAPICall
-
 LOGGER = logging.getLogger(__name__)
 
 
 class FileDiscovery(object):
 
     def __init__(self, **kwargs):
         self.video_info = {}
-
         self.auth_dict = get_config()
-
         self.bucket = None
-        """
-        FTP Server Vars
-        """
-        self.ftp_key = None
-        self.ftp_follow_delay = str(5000)
-        self.ftp_log = "/Users/Shared/edX1/LG/Transfers.log"
-        self.wfm_log = "/Users/Shared/edX1/LG/WFM.log"
-        self.ftp_faillog = "/Users/Shared/edX1/LG/FailedTransfers.log"
         self.node_work_directory = kwargs.get('node_work_directory', WORK_DIRECTORY)
 
     def about_video_ingest(self):
         """
         Crawl VEDA Upload bucket
         """
         if self.node_work_directory is None:
-            ErrorObject().print_error(
-                message='No Workdir'
-            )
+            print '[Discovery Error] No Workdir'
+            return
+        try:
+            conn = boto.connect_s3()
+        except NoAuthHandlerFound:
+            print '[Discovery Error] BOTO Auth Handler'
             return
-        conn = boto.connect_s3()
-
         try:
             self.bucket = conn.get_bucket(self.auth_dict['veda_s3_upload_bucket'])
         except S3ResponseError:
@@ -217,7 +207,6 @@ def download_video_to_working_directory(self, key, file_name, file_extension):
         """
         if len(file_extension) == 3:
             file_name = u'{file_name}.{ext}'.format(file_name=file_name, ext=file_extension)
-
         try:
             key.get_contents_to_filename(os.path.join(self.node_work_directory, file_name))
             file_ingested = True
@@ -263,6 +252,8 @@ def discover_studio_ingested_videos(self):
                     self.validate_metadata_and_feed_to_ingest(video_s3_key=self.bucket.get_key(video_s3_key.name))
             except S3ResponseError:
                 ErrorObject.print_error(message='[File Ingest] S3 Ingest Connection Failure')
+            except NoAuthHandlerFound:
+                ErrorObject.print_error(message='[Discovery Error] BOTO Auth Handler')
         else:
             ErrorObject.print_error(message='[File Ingest] No Working Node directory')
 
diff --git a/control/veda_file_ingest.py b/control/veda_file_ingest.py
@@ -162,7 +162,7 @@ def queue_job(self):
             jobid = uuid.uuid1().hex[0:10]
             celeryapp.worker_task_fire.apply_async(
                 (veda_id, encode_profile, jobid),
-                queue=self.auth_dict['celery_worker_queue']
+                queue=self.auth_dict['celery_worker_queue'].split(',')[0]
             )
 
         """
diff --git a/control/veda_heal.py b/control/veda_heal.py
@@ -85,7 +85,7 @@ def send_encodes(self):
                     jobid = uuid.uuid1().hex[0:10]
                     celeryapp.worker_task_fire.apply_async(
                         (veda_id, encode_profile, jobid),
-                        queue=self.auth_dict['celery_worker_queue']
+                        queue=self.auth_dict['celery_worker_queue'].split(',')[0]
                     )
 
     def determine_fault(self, video_object):
@@ -95,15 +95,15 @@ def determine_fault(self, video_object):
         LOGGER.info('[HEAL] : {id}'.format(id=video_object.edx_id))
         if self.freezing_bug is True:
             if video_object.video_trans_status == 'Corrupt File':
+                self.val_status = 'file_corrupt'
                 return []
 
-        if video_object.video_trans_status == 'Review Reject':
+        if video_object.video_trans_status == 'Review Reject' or video_object.video_trans_status == 'Review Hold' or \
+            video_object.video_trans_status == 'Review Hold':
             return []
 
-        if video_object.video_trans_status == 'Review Hold':
-            return []
-
-        if video_object.video_active is False:
+        if video_object.video_trans_status == 'Youtube Duplicate':
+            self.val_status = 'duplicate'
             return []
 
         """
diff --git a/youtube_callback/sftp_id_retrieve.py b/youtube_callback/sftp_id_retrieve.py
@@ -3,16 +3,23 @@
 
 """
 import datetime
-from datetime import timedelta
-import django
-from django.utils.timezone import utc
 import fnmatch
 import os
-from os.path import expanduser
-import pysftp
 import shutil
 import sys
 import xml.etree.ElementTree as ET
+from datetime import timedelta
+from os.path import expanduser
+
+import django
+import pysftp
+from django.utils.timezone import utc
+
+from control.veda_utils import ErrorObject, Metadata, VideoProto
+from control.veda_val import VALAPICall
+from frontend.abvid_reporting import report_status
+from VEDA_OS01.models import URL, Encode, Video
+from youtube_callback.daemon import get_course
 
 project_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 if project_path not in sys.path:
@@ -21,11 +28,6 @@
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'VEDA.settings.local')
 django.setup()
 
-from VEDA_OS01.models import Video, Encode, URL
-from frontend.abvid_reporting import report_status
-from control.veda_val import VALAPICall
-from control.veda_utils import ErrorObject, Metadata, VideoProto
-from youtube_callback.daemon import get_course
 
 """
 Defaults:
@@ -72,19 +74,16 @@ def xml_downloader(course):
     cnopts = pysftp.CnOpts()
     cnopts.hostkeys = None
 
-    try:
-        with pysftp.Connection(
-            'partnerupload.google.com',
-            username=course.yt_logon,
-            private_key=private_key,
-            port=19321,
-            cnopts=cnopts
-        ) as s1:
-            for d in s1.listdir_attr():
-                crawl_sftp(d=d, s1=s1)
-    except:
-        ErrorObject.print_error("Failed Auth: Youtube SFTP")
-        return None
+    with pysftp.Connection(
+        'partnerupload.google.com',
+        username=course.yt_logon,
+        private_key=private_key,
+        port=19321,
+        cnopts=cnopts
+    ) as s1:
+        s1.timeout = 60.0
+        for d in s1.listdir_attr():
+            crawl_sftp(d=d, s1=s1)
 
 
 def crawl_sftp(d, s1):
@@ -284,30 +283,25 @@ def urlpatch(upload_data):
             return None
 
         if len(encode_list) == 0:
-            Video.objects.filter(
-                edx_id=upload_data['edx_id']
-            ).update(
-                video_trans_status='Complete'
-            )
+            Video.objects.filter(edx_id=upload_data['edx_id']).update(video_trans_status='Complete')
             val_status = 'file_complete'
-
         else:
             val_status = 'transcode_active'
 
-        VAC = VALAPICall(
+        ApiConn = VALAPICall(
             video_proto=video_proto,
             val_status=val_status,
             endpoint_url=upload_data['youtube_id'],
             encode_profile='youtube'
         )
-        VAC.call()
+        ApiConn.call()
 
     elif upload_data['status'] == 'Duplicate' and \
             upload_data['file_suffix'] == '100':
 
         url_query = URL.objects.filter(
             videoID=Video.objects.filter(
-                edx_id=test_id.edx_id
+                edx_id=upload_data['edx_id']
             ).latest(),
             encode_profile=Encode.objects.get(
                 encode_suffix=upload_data['file_suffix']
@@ -322,12 +316,7 @@ def urlpatch(upload_data):
                     youtube_id=''
                 )
 
-            Video.objects.filter(
-                edx_id=upload_data['edx_id']
-            ).update(
-                video_trans_status='Youtube Duplicate'
-            )
-
+            Video.objects.filter(edx_id=upload_data['edx_id']).update(video_trans_status='Youtube Duplicate')
             video_proto = VideoProto(
                 veda_id=test_id.edx_id,
                 val_id=test_id.studio_id,
@@ -336,11 +325,10 @@ def urlpatch(upload_data):
                 bitrate='0',
                 s3_filename=test_id.studio_id
             )
-
-            VAC = VALAPICall(
+            ApiConn = VALAPICall(
                 video_proto=video_proto,
                 val_status="duplicate",
                 endpoint_url="DUPLICATE",
                 encode_profile='youtube'
             )
-            VAC.call()
+            ApiConn.call()

Original file line number	Diff line number	Diff line change
`@@ -125,7 +125,7 @@ def test_heal(self):`
`125`	`125`	`'video_active': True,`
`126`	`126`	`},`
`127`	`127`	`{`
`128`		`- 'edx_id': '1',`
	`128`	`+ 'edx_id': '2',`
`129`	`129`	`'video_trans_status': 'Complete',`
`130`	`130`	`'video_trans_start': datetime.datetime.utcnow().replace(tzinfo=utc),`
`131`	`131`	`'video_active': False,`
Original file line number	Diff line number	Diff line change
`@@ -162,7 +162,7 @@ def queue_job(self):`
`162`	`162`	`jobid = uuid.uuid1().hex[0:10]`
`163`	`163`	`celeryapp.worker_task_fire.apply_async(`
`164`	`164`	`(veda_id, encode_profile, jobid),`
`165`		`- queue=self.auth_dict['celery_worker_queue']`
	`165`	`+ queue=self.auth_dict['celery_worker_queue'].split(',')[0]`
`166`	`166`	`)`
`167`	`167`
`168`	`168`	`"""`