Merge pull request #31 from q-verse/umar/qverse/add_support_for_utf-16_format

EDE-559 add support for utf 16 format

Author: mumarkhan999

openedx/features/qverse_features/registration/helpers.py

@@ -0,0 +1,55 @@
+"""
+Contains helper functions for Qverse registration application.
+"""
+from csv import reader, Sniffer
+import io
+import logging
+
+LOGGER = logging.getLogger(__name__)
+
+
+def get_file_header_row(file_content, encoding):
+    """
+    Returns fields of header row of the file.
+
+    Arguments:
+        file_content (str): File content that has been read from the file
+        encoding (str): File encoding format e.g: utf-8, utf-16
+
+    Returns:
+        header_row (list): List of fields of the header row of CSV file
+    """
+    decoded_file = file_content.decode(encoding, 'ignore')
+    io_string = io.StringIO(decoded_file)
+    dialect = Sniffer().sniff(io_string.readline())
+    io_string.seek(0)
+    header_row = reader(io_string, delimiter=dialect.delimiter).next()
+    return [heading.lower().strip() for heading in header_row]
+
+
+def get_file_encoding(file_path):
+    """
+    Returns the file encoding format.
+
+    Arguments:
+        file_path (str): Path of the file whose encoding format will be returned
+
+    Returns:
+        encoding (str): encoding format e.g: utf-8, utf-16, returns None if doesn't find
+                        any encoding format
+    """
+    try:
+        file = io.open(file_path, 'r', encoding='utf-8')
+        encoding = None
+        try:
+            encoding = 'utf-8'
+            _ = file.read()
+        except UnicodeDecodeError:
+            encoding = 'utf-16'
+
+        file.close()
+        return encoding
+
+    except IOError as error:
+        LOGGER.exception('({}) --- {}'.format(error.filename, error.strerror))
+        return None

openedx/features/qverse_features/registration/signals.py

@@ -1,6 +1,7 @@
 """
 Signals for qverse registration application.
 """
+import io
 import logging
 import re
 from csv import DictReader, DictWriter, Error, Sniffer
@@ -19,6 +20,7 @@
                                                                   SURNAME_MAX_LENGTH, FIRST_NAME_MAX_LENGTH,
                                                                   MOBILE_NUMBER_MAX_LENGTH, OTHER_NAME_MAX_LENGTH,
                                                                   MAX_LEVEL_CHOICES, MAX_PROGRAMME_CHOICES)
+from openedx.features.qverse_features.registration.helpers import get_file_encoding
 from openedx.features.qverse_features.registration.tasks import send_bulk_mail_to_newly_created_students
 from student.models import UserProfile
 
@@ -51,8 +53,15 @@ def create_users_from_csv_file(sender, instance, created, **kwargs):
     csv_file = None
     dialect = None
     try:
-        csv_file = open(instance.admission_file.path, 'r')
-        dialect = Sniffer().sniff(csv_file.readline())
+        encoding = get_file_encoding(instance.admission_file.path)
+        csv_file = io.open(instance.admission_file.path, 'r', encoding=encoding)
+        try:
+            dialect = Sniffer().sniff(csv_file.readline())
+        except Error:
+            LOGGER.exception('Could not determine delimiter in the file.')
+            csv_file.close()
+            return
+
         csv_file.seek(0)
     except IOError as error:
         LOGGER.exception('({}) --- {}'.format(error.filename, error.strerror))
@@ -66,6 +75,7 @@ def create_users_from_csv_file(sender, instance, created, **kwargs):
     reader = (dict((k.strip().lower(), v.strip() if v else v) for k, v in row.items()) for row in dict_reader)
     output_file_rows = []
     users_with_updated_emails = set()
+
     try:
         CsvRowValidator.prepare_csv_row_validator()
         for row in reader:

openedx/features/qverse_features/registration/validators.py

@@ -1,11 +1,12 @@
 """
 Validation utils for qverse registration application.
 """
-from csv import reader, Sniffer
-import io
+from csv import Error
 
 from django.core.exceptions import ValidationError
 
+from openedx.features.qverse_features.registration.helpers import get_file_header_row
+
 
 def validate_admission_file(file):
     """
@@ -22,18 +23,19 @@ def validate_admission_file(file):
                 'regno', 'firstname', 'surname', 'othername', 'levelid',
                 'programmeid', 'departmentid', 'mobile', 'email'
                 ]
-    decoded_file = file.read().decode('utf-8')
-    io_string = io.StringIO(decoded_file)
     header_row = []
     try:
-        dialect = Sniffer().sniff(io_string.readline())
-        io_string.seek(0)
-        header_row = reader(io_string, delimiter=dialect.delimiter).next()
-        header_row = [heading.lower().strip() for heading in header_row]
-    except StopIteration:
+        file_content = file.read()
+
+        try:
+            header_row = get_file_header_row(file_content, 'utf-8')
+        except Error:
+            header_row = get_file_header_row(file_content, 'utf-16')
+
+    except Exception:
         raise ValidationError('', code='invalid')
 
-    if not all(field_name in header_row for field_name in FIELD_NAMES):
+    if not all([field_name in header_row for field_name in FIELD_NAMES]):
         raise ValidationError('', code='invalid')
 
     if 'error' in header_row: