ENG-8475 | fix unable to render spreadsheet files (.xls and .xlsx)

Author: sh-andriy

mfr/extensions/tabular/libs/xlrd_tools.py

@@ -1,95 +1,39 @@
-import xlrd
-import zipfile
+from io import BytesIO
+from openpyxl import load_workbook
+import xlrd, zipfile
 from collections import OrderedDict
 from ..exceptions import TableTooBigError, MissingRequirementsError
-
-from ..utilities import header_population
+from ..utilities import (
+    header_population,
+    to_bytes,
+    parse_xls,
+    parse_xlsx
+)
 from mfr.extensions.tabular.compat import range, basestring
 
 
 def xlsx_xlrd(fp):
-    """Read and convert a xlsx file to JSON format using the xlrd library
-    :param fp: File pointer object
-    :return: tuple of table headers and data
     """
-    MAX_SIZE = 10000
-
-    try:
-        wb = xlrd.open_workbook(fp.name)
-        using_xlrd = True
-    except xlrd.biffh.XLRDError:
-        using_xlrd = False
-        try:
-            from openpyxl import load_workbook
-        except ImportError:
-            raise MissingRequirementsError(
-                'openpyxl is required to read .xlsx files',
-                function_preference='openpyxl'
-            )
-        try:
-            wb = load_workbook(fp.name, data_only=True)
-        except zipfile.BadZipFile:
-            raise xlrd.biffh.XLRDError("Excel xlsx file; not supported")
+    • .xls → xlrd
+    • .xlsx → openpyxl (xlrd ≥2.0 dropped xlsx support)
 
+    `fp` is the stream returned by WaterButler/MFR.  It may already have been
+    read, so we always rewind and copy to an in‑memory buffer that openpyxl (and
+    ZipFile) can seek inside safely.
+    """
     sheets = OrderedDict()
 
-    if using_xlrd:
-        for sheet in wb.sheets():
-            if sheet.ncols > MAX_SIZE or sheet.nrows > MAX_SIZE:
-                raise TableTooBigError('Table is too large to render.', '.xlsx',
-                                       nbr_cols=sheet.ncols, nbr_rows=sheet.nrows)
-
-            if sheet.ncols < 1 or sheet.nrows < 1:
-                sheets[sheet.name] = ([], [])
-                continue
-
-            fields = sheet.row_values(0) if sheet.nrows else []
-
-            fields = [
-                str(value)
-                if not isinstance(value, basestring) and value is not None
-                else value or f'Unnamed: {index + 1}'
-                for index, value in enumerate(fields)
-            ]
-
-            data = []
-            for i in range(1, sheet.nrows):
-                row = []
-                for cell in sheet.row(i):
-                    if cell.ctype == xlrd.XL_CELL_DATE:
-                        value = xlrd.xldate.xldate_as_datetime(cell.value, wb.datemode).isoformat()
-                    else:
-                        value = cell.value
-                    row.append(value)
-                data.append(dict(zip(fields, row)))
-
-            header = header_population(fields)
-            sheets[sheet.name] = (header, data)
-
-    else:
-        for name in wb.sheetnames:
-            ws = wb[name]
-            nrows = ws.max_row
-            ncols = ws.max_column
-            if ncols > MAX_SIZE or nrows > MAX_SIZE:
-                raise TableTooBigError('Table is too large to render.', '.xlsx',
-                                       nbr_cols=ncols, nbr_rows=nrows)
-
-            if nrows < 1 or ncols < 1:
-                sheets[name] = ([], [])
-                continue
-
-            header_row = next(ws.iter_rows(min_row=1, max_row=1, values_only=True))
-            fields = [
-                str(val) if val is not None else f'Unnamed: {i + 1}'
-                for i, val in enumerate(header_row)
-            ]
-
-            data = []
-            for row in ws.iter_rows(min_row=2, max_row=nrows, max_col=ncols, values_only=True):
-                data.append(dict(zip(fields, row)))
+    try:
+        wb = xlrd.open_workbook(file_contents=to_bytes(fp))
+        return parse_xls(wb, sheets)
+    except xlrd.biffh.XLRDError:
+        pass
 
-            header = header_population(fields)
-            sheets[name] = (header, data)
+    try:
+        wb = load_workbook(BytesIO(to_bytes(fp)), data_only=True, read_only=True)
+    except zipfile.BadZipFile as exc:
+        raise xlrd.biffh.XLRDError(
+            "Invalid xlsx file or corrupted ZIP structure"
+        ) from exc
 
-    return sheets
+    return parse_xlsx(wb, sheets)

mfr/extensions/tabular/utilities.py

@@ -1,4 +1,6 @@
 import re
+import xlrd
+
 from http import HTTPStatus
 from subprocess import (check_call,
                         TimeoutExpired,
@@ -11,6 +13,8 @@
                                              PSPP_CONVERT_TIMEOUT)
 
 
+MAX_SIZE = 10_000
+
 def header_population(headers):
     """make column headers from a list
     :param headers: list of column headers
@@ -83,3 +87,92 @@ def sav_to_csv(fp):
             exporter_class='tabular'
         )
     return csv_file
+
+
+def to_bytes(fp):
+    """
+    Return *exactly* the original bytes of the Excel file and rewind *fp*.
+    Handles both binary and text wrappers that WaterButler may give us.
+    """
+    try:
+        fp.seek(0)
+    except Exception:
+        pass
+
+    raw = fp.read()
+    if isinstance(raw, bytes):
+        try:
+            fp.seek(0)
+        except Exception:
+            pass
+        return raw
+
+    if hasattr(fp, "buffer"):
+        buf = fp.buffer
+        try:
+            buf.seek(0)
+        except Exception:
+            pass
+        data = buf.read()
+        try:
+            buf.seek(0)
+        except Exception:
+            pass
+    else:
+        data = raw.encode("utf-8", "surrogateescape")
+
+    try:
+        fp.seek(0)
+    except Exception:
+        pass
+    return data
+
+
+def parse_xls(wb, sheets):
+    for sheet in wb.sheets():
+        verify_size(sheet.nrows, sheet.ncols, '.xls')
+        fields = fix_headers(sheet.row_values(0))
+        rows = [
+            dict(zip(fields, row_vals(sheet.row(r), wb.datemode)))
+            for r in range(1, sheet.nrows)
+        ]
+        sheets[sheet.name] = (header_population(fields), rows)
+    return sheets
+
+
+def parse_xlsx(wb, sheets):
+    for name in wb.sheetnames:
+        ws = wb[name]
+        verify_size(ws.max_row, ws.max_column, '.xlsx')
+        header_row = next(ws.iter_rows(max_row=1, values_only=True))
+        fields = fix_headers(header_row)
+        rows = [
+            dict(zip(fields, row))
+            for row in ws.iter_rows(min_row=2,
+                                    max_row=ws.max_row,
+                                    max_col=ws.max_column,
+                                    values_only=True)
+        ]
+        sheets[name] = (header_population(fields), rows)
+    return sheets
+
+
+def verify_size(rows, cols, ext):
+    if rows > MAX_SIZE or cols > MAX_SIZE:
+        raise TableTooBigError('Table is too large to render.', ext,
+                               nbr_cols=cols, nbr_rows=rows)
+
+
+def fix_headers(raw):
+    return [str(v) if v not in (None, '') else f'Unnamed: {i+1}'
+            for i, v in enumerate(raw)]
+
+
+def row_vals(row, datemode):
+    out = []
+    for c in row:
+        if c.ctype == xlrd.XL_CELL_DATE:
+            out.append(xlrd.xldate.xldate_as_datetime(c.value, datemode).isoformat())
+        else:
+            out.append(c.value)
+    return out