From 05762f90ce2d554e2961adfd28f3a1c5d47cd433 Mon Sep 17 00:00:00 2001 From: Qubad786 Date: Wed, 28 Feb 2018 16:10:25 +0500 Subject: [PATCH] restore old transcription workflow --- control/old_veda_deliver_cielo.py | 173 ++++++++++++++++++++++++++++++ control/veda_deliver.py | 74 ++++++++++++- static_config.yaml | 3 + 3 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 control/old_veda_deliver_cielo.py diff --git a/control/old_veda_deliver_cielo.py b/control/old_veda_deliver_cielo.py new file mode 100644 index 0000000..855a8c2 --- /dev/null +++ b/control/old_veda_deliver_cielo.py @@ -0,0 +1,173 @@ +import requests +import ast +import urllib + +""" +Cielo24 API Job Start and Download +Options (reflected in Course.models): +transcription_fidelity = + Mechanical (75%), + Premium (95%)(3-72h), + Professional (99+%)(3-72h) +priority = + standard (24h), + priority (48h) +turnaround_hours = number, overrides 'priority' call, will change a standard to a priority silently +""" +from control_env import * +from veda_utils import ErrorObject + +requests.packages.urllib3.disable_warnings() + + +class Cielo24TranscriptOld(object): + + def __init__(self, veda_id): + self.veda_id = veda_id + '''Defaults''' + self.c24_site = 'https://api.cielo24.com/api' + self.c24_login = '/account/login' + self.c24_joblist = '/job/list' + self.c24_newjob = '/job/new' + self.add_media = '/job/add_media' + self.transcribe = '/job/perform_transcription' + + '''Retreive C24 Course-based defaults''' + self.c24_defaults = self.retrieve_defaults() + + def perform_transcription(self): + if self.c24_defaults['c24_user'] is None: + return None + ''' + GET /api/job/perform_transcription?v=1 HTTP/1.1 + &api_token=xxxx + &job_id=xxxx + &transcription_fidelity=PREMIUM&priority=STANDARD + Host: api.cielo24.com + ''' + api_token = self.tokengenerator() + if api_token is None: + return None + + job_id = self.generate_jobs(api_token) + task_id = self.embed_url(api_token, job_id) + + r5 = requests.get( + ''.join(( + self.c24_site, + self.transcribe, + '?v=1&api_token=', + api_token, + '&job_id=', + job_id, + '&transcription_fidelity=', + self.c24_defaults['c24_fidelity'], + '&priority=', + self.c24_defaults['c24_speed'] + )) + ) + return ast.literal_eval(r5.text)['TaskId'] + + def retrieve_defaults(self): + video_query = Video.objects.filter( + edx_id=self.veda_id + ).latest() + + url_query = URL.objects.filter( + videoID=video_query, + encode_url__icontains='_DTH.mp4', + ).latest() + + if video_query.inst_class.c24_username is None: + ErrorObject.print_error( + message='Cielo24 Record Incomplete', + ) + return None + + c24_defaults = { + 'c24_user': video_query.inst_class.c24_username, + 'c24_pass': video_query.inst_class.c24_password, + 'c24_speed': video_query.inst_class.c24_speed, + 'c24_fidelity': video_query.inst_class.c24_fidelity, + 'edx_id': self.veda_id, + 'url': url_query.encode_url + } + return c24_defaults + + def tokengenerator(self): + token_url = self.c24_site + self.c24_login + \ + '?v=1&username=' + self.c24_defaults['c24_user'] + \ + '&password=' + self.c24_defaults['c24_pass'] + + # Generate Token + r1 = requests.get(token_url) + if r1.status_code > 299: + ErrorObject.print_error( + message='Cielo24 API Access Error', + ) + return None + api_token = ast.literal_eval(r1.text)["ApiToken"] + return api_token + + def listjobs(self): + """List Jobs""" + api_token = self.tokengenerator() + r2 = requests.get( + ''.join(( + self.c24_site, + self.c24_joblist, + '?v=1&api_token=', + api_token + )) + ) + job_list = r2.text + return job_list + + def generate_jobs(self, api_token): + """ + 'https://api.cielo24.com/job/new?v=1&\ + api_token=xxx&job_name=xxx&language=en' + """ + r3 = requests.get( + ''.join(( + self.c24_site, + self.c24_newjob, + '?v=1&api_token=', + api_token, + '&job_name=', + self.c24_defaults['edx_id'], + '&language=en' + )) + ) + job_id = ast.literal_eval(r3.text)['JobId'] + return job_id + + def embed_url(self, api_token, job_id): + """ + GET /api/job/add_media?v=1&api_token=xxxx + &job_id=xxxxx + &media_url=http%3A%2F%2Fwww.domain.com%2Fvideo.mp4 HTTP/1.1 + Host: api.cielo24.com + """ + r4 = requests.get( + ''.join(( + self.c24_site, + self.add_media, + '?v=1&api_token=', + api_token, + '&job_id=', + job_id, + '&media_url=', + urllib.quote_plus(self.c24_defaults['url']) + )) + ) + print str(r4.status_code) + ' : Cielo24 Status Code' + return ast.literal_eval(r4.text)['TaskId'] + + +def main(): + pass + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/control/veda_deliver.py b/control/veda_deliver.py index e5b81f1..d3f343d 100644 --- a/control/veda_deliver.py +++ b/control/veda_deliver.py @@ -5,10 +5,10 @@ """ import datetime +import ftplib import logging import shutil from os.path import expanduser -import sys import boto import boto.s3 @@ -19,6 +19,7 @@ from boto.s3.key import Key from django.core.urlresolvers import reverse +from control.old_veda_deliver_cielo import Cielo24TranscriptOld from control_env import * from veda_deliver_cielo import Cielo24Transcript from veda_deliver_youtube import DeliverYoutube @@ -128,6 +129,11 @@ def run(self): u1.encode_size = self.video_proto.filesize u1.save() + # TODO: Warning! this shall be removed once 3rd party credentials + # for existing courses are migrated according to new flow. + self._THREEPLAY_UPLOAD() + self._CIELO24_UPLOAD() + self.status = self._DETERMINE_STATUS() self._UPDATE_DATA() self._CLEANUP() @@ -629,6 +635,72 @@ def start_3play_transcription_process(self, encoded_file): self.video_query.studio_id, ) + def _CIELO24_UPLOAD(self): + """ + Note: This is part of the old flow which was deprecated when transcript phase 1 was reased. + """ + # TODO: this must be removed once existing 3rd party credentials are migrated according + # to the new workflow. + if self.video_query.inst_class.c24_proc is False: + return None + + if self.encode_profile != 'desktop_mp4': + return None + + C24 = Cielo24TranscriptOld( + veda_id=self.video_query.edx_id + ) + output = C24.perform_transcription() + print '[ %s ] : %s' % ( + 'Cielo24 JOB', self.video_query.edx_id + ) + + def _THREEPLAY_UPLOAD(self): + """ + Note: This is part of the old flow which was deprecated when transcript phase 1 was reased. + """ + # TODO: this must be removed once existing 3rd party credentials are migrated according + # to the new workflow. + if self.video_query.inst_class.tp_proc is False: + return None + + if self.encode_profile != 'desktop_mp4': + return None + + ftp1 = ftplib.FTP( + self.auth_dict['threeplay_ftphost'] + ) + user = self.video_query.inst_class.tp_username.strip() + passwd = self.video_query.inst_class.tp_password.strip() + try: + ftp1.login(user, passwd) + except: + ErrorObject.print_error( + message='3Play Authentication Failure' + ) + try: + ftp1.cwd( + self.video_query.inst_class.tp_speed + ) + except: + ftp1.mkd( + self.video_query.inst_class.tp_speed + ) + ftp1.cwd( + self.video_query.inst_class.tp_speed + ) + os.chdir(self.node_work_directory) + + ftp1.storbinary( + 'STOR ' + self.encoded_file, + open(os.path.join( + self.node_work_directory, + self.encoded_file + ), 'rb') + ) + + os.chdir(homedir) + def YOUTUBE_SFTP(self, review=False): if self.video_query.inst_class.yt_proc is False: if self.video_query.inst_class.review_proc is False: diff --git a/static_config.yaml b/static_config.yaml index 0e6dbc3..a81e474 100644 --- a/static_config.yaml +++ b/static_config.yaml @@ -1,6 +1,9 @@ --- # This configuration should only have static settings. +# 3PlayMedia FTP host for old workflow +threeplay_ftphost: ftp.3playmedia.com + # s3 bucket static prefixes edx_s3_processed_prefix: prod-edx/processed/ edx_s3_rejected_prefix: prod-edx/rejected/