Merge pull request #76 from edx/release_061217

Deployment Bugfix

Author: yro

VEDA_OS01/admin.py

@@ -151,6 +151,20 @@ class TranscriptProcessMetadataAdmin(admin.ModelAdmin):
     """
     Admin for TranscriptProcessMetadata model.
     """
+    raw_id_fields = ('video', )
+    list_display = ('get_video', 'provider', 'process_id', 'translation_id', 'lang_code', 'status')
+
+    def get_video(self, obj):
+        return u'"{studio_id}" - "{edx_id}"'.format(
+            studio_id=obj.video.studio_id,
+            edx_id=obj.video.edx_id
+        )
+
+    get_video.admin_order_field = 'video'
+    get_video.short_description = 'Transcript Video'
+
+    search_fields = ['video__edx_id', 'video__studio_id', 'process_id', 'translation_id']
+
     model = TranscriptProcessMetadata
 
 

VEDA_OS01/models.py

@@ -733,9 +733,8 @@ def update(self, **fields):
         self.save()
 
     def __unicode__(self):
-        return u'{video} - {provider} - {lang} - {status}'.format(
+        return u'{video} - {provider} - {lang}'.format(
             video=self.video.edx_id,
             provider=self.provider,
-            lang=self.lang_code,
-            status=self.status,
+            lang=self.lang_code
         )

control/veda_file_discovery.py

@@ -186,6 +186,7 @@ def get_or_create_course(self, course_id, course_hex=None):
                     institution=course_key.org,
                     edx_classid=course_key.course,
                     local_storedir=course_id,
+                    yt_proc=False,
                 )
         else:
             try:
@@ -249,7 +250,8 @@ def discover_studio_ingested_videos(self):
                 connection = boto.connect_s3()
                 self.bucket = connection.get_bucket(self.auth_dict['edx_s3_ingest_bucket'])
                 for video_s3_key in self.bucket.list(self.auth_dict['edx_s3_ingest_prefix'], '/'):
-                    self.validate_metadata_and_feed_to_ingest(video_s3_key=self.bucket.get_key(video_s3_key.name))
+                    if video_s3_key.name != 'prod-edx/unprocessed/':
+                        self.validate_metadata_and_feed_to_ingest(video_s3_key=self.bucket.get_key(video_s3_key.name))
             except S3ResponseError:
                 ErrorObject.print_error(message='[File Ingest] S3 Ingest Connection Failure')
             except NoAuthHandlerFound:

static_config.yaml

@@ -29,10 +29,8 @@ encode_dict:
     mobile_override:
         - override
     s3_proc:
-        - mobile_high
         - mobile_low
         - audio_mp3
-        - desktop_webm
         - desktop_mp4
         - hls
 
@@ -51,17 +49,10 @@ val_profile_dict:
     override:
         - desktop_mp4
         - mobile_low
-        - mobile_high
-
-    mobile_high:
-        - mobile_high
 
     audio_mp3:
         - audio_mp3
 
-    desktop_webm:
-        - desktop_webm
-
     youtube:
         - youtube
     review:

youtube_callback/sftp_id_retrieve.py

@@ -4,12 +4,14 @@
 """
 import datetime
 import fnmatch
+import logging
 import os
 import shutil
 import sys
 import xml.etree.ElementTree as ET
 from datetime import timedelta
 from os.path import expanduser
+from paramiko.ssh_exception import AuthenticationException
 
 import django
 import pysftp
@@ -28,6 +30,9 @@
 os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'VEDA.settings.local')
 django.setup()
 
+LOGGER = logging.getLogger(__name__)
+# TODO: Remove this temporary logging to stdout
+logging.basicConfig(stream=sys.stdout, level=logging.INFO)
 
 """
 Defaults:
@@ -73,17 +78,19 @@ def xml_downloader(course):
 
     cnopts = pysftp.CnOpts()
     cnopts.hostkeys = None
-
-    with pysftp.Connection(
-        'partnerupload.google.com',
-        username=course.yt_logon,
-        private_key=private_key,
-        port=19321,
-        cnopts=cnopts
-    ) as s1:
-        s1.timeout = 60.0
-        for d in s1.listdir_attr():
-            crawl_sftp(d=d, s1=s1)
+    try:
+        with pysftp.Connection(
+            'partnerupload.google.com',
+            username=course.yt_logon,
+            private_key=private_key,
+            port=19321,
+            cnopts=cnopts
+        ) as s1:
+            s1.timeout = 60.0
+            for d in s1.listdir_attr():
+                crawl_sftp(d=d, s1=s1)
+    except AuthenticationException:
+        LOGGER.info("{inst}{clss} : Authentication Failed".format(inst=course.institution, clss=course.edx_classid))
 
 
 def crawl_sftp(d, s1):