Add optional sanitization of illegal XLSX characters on export (fixes #370)

AUTHORS

@@ -6,6 +6,7 @@ Here is a list of past and present much-appreciated contributors:
     Alex Gaynor
     Andrew Graham-Yooll
     Andrii Soldatenko
+    Apoorv Darshan
     Benjamin Wohlwend
     Bruno Soares
     Claude Paroz

docs/formats.rst

@@ -266,6 +266,21 @@ This works with ``Databook`` as well::
 
 The adaptive width will be calculated for each sheet in the databook.
 
+The ``export_set()`` and ``export_book()`` methods support a
+``sanitize_illegal_chars`` parameter. Some control characters (matched by
+``openpyxl.cell.cell.ILLEGAL_CHARACTERS_RE``) are not allowed in an XLSX file,
+and by default exporting a value containing one raises openpyxl's
+``IllegalCharacterError``. Pass a string to replace such characters instead
+(use ``""`` to strip them out)::
+
+    data = tablib.Dataset()
+    data.append(('some\x1ftext',))
+    data.export('xlsx', sanitize_illegal_chars='')
+
+.. versionadded:: 3.10.0
+    The ``sanitize_illegal_chars`` parameter for ``export_set()`` and
+    ``export_book()`` was added.
+
 .. versionchanged:: 3.8.0
     The ``column_width`` parameter for ``export_set()`` was added.
 

src/tablib/formats/_xlsx.py

@@ -14,9 +14,11 @@
 import re
 from io import BytesIO
 
+from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE
 from openpyxl.reader.excel import ExcelReader, load_workbook
 from openpyxl.styles import Alignment, Font
 from openpyxl.utils import get_column_letter
+from openpyxl.utils.exceptions import IllegalCharacterError
 from openpyxl.workbook import Workbook
 
 import tablib
@@ -46,7 +48,8 @@ def detect(cls, stream):
 
     @classmethod
     def export_set(cls, dataset, freeze_panes=True, invalid_char_subst="-",
-                   escape=False, column_width="adaptive"):
+                   escape=False, column_width="adaptive",
+                   sanitize_illegal_chars=None):
         """Returns XLSX representation of Dataset.
 
         If ``freeze_panes`` is True, Export will freeze panes only after first line.
@@ -65,6 +68,13 @@ def export_set(cls, dataset, freeze_panes=True, invalid_char_subst="-",
         set to that integer value. If it is set to None, the column width will be set as the
         default openpyxl.Worksheet width value.
 
+        If ``sanitize_illegal_chars`` is a string, characters which are illegal in
+        an XLSX file (control characters matched by
+        ``openpyxl.cell.cell.ILLEGAL_CHARACTERS_RE``) will be replaced with that
+        string in string cell values (use ``""`` to strip them). If it is None
+        (the default) or False, such characters are left untouched and openpyxl
+        raises ``IllegalCharacterError``.
+
         """
         wb = Workbook()
         ws = wb.worksheets[0]
@@ -74,7 +84,10 @@ def export_set(cls, dataset, freeze_panes=True, invalid_char_subst="-",
             if dataset.title else 'Tablib Dataset'
         )
 
-        cls.dset_sheet(dataset, ws, freeze_panes=freeze_panes, escape=escape)
+        cls.dset_sheet(
+            dataset, ws, freeze_panes=freeze_panes, escape=escape,
+            sanitize_illegal_chars=sanitize_illegal_chars,
+        )
 
         cls._adapt_column_width(ws, column_width)
 
@@ -84,7 +97,7 @@ def export_set(cls, dataset, freeze_panes=True, invalid_char_subst="-",
 
     @classmethod
     def export_book(cls, databook, freeze_panes=True, invalid_char_subst="-",
-                    escape=False, column_width=None):
+                    escape=False, column_width=None, sanitize_illegal_chars=None):
         """Returns XLSX representation of DataBook.
         See export_set().
         """
@@ -99,7 +112,10 @@ def export_book(cls, databook, freeze_panes=True, invalid_char_subst="-",
                 if dset.title else f"Sheet{i}"
             )
 
-            cls.dset_sheet(dset, ws, freeze_panes=freeze_panes, escape=escape)
+            cls.dset_sheet(
+                dset, ws, freeze_panes=freeze_panes, escape=escape,
+                sanitize_illegal_chars=sanitize_illegal_chars,
+            )
 
             cls._adapt_column_width(ws, column_width)
 
@@ -148,7 +164,8 @@ def import_book(cls, dbook, in_stream, headers=True, read_only=True):
             dbook.add_sheet(dset)
 
     @classmethod
-    def dset_sheet(cls, dataset, ws, freeze_panes=True, escape=False):
+    def dset_sheet(cls, dataset, ws, freeze_panes=True, escape=False,
+                   sanitize_illegal_chars=None):
         """Completes given worksheet from given Dataset."""
         _package = dataset._package(dicts=False)
 
@@ -183,6 +200,16 @@ def dset_sheet(cls, dataset, ws, freeze_panes=True, escape=False):
 
                 try:
                     cell.value = col
+                except IllegalCharacterError:
+                    # openpyxl refuses control characters that are illegal in
+                    # XLSX. Replace them only if a replacement string was
+                    # requested, otherwise let the exception bubble up.
+                    if (sanitize_illegal_chars is None
+                            or sanitize_illegal_chars is False):
+                        raise
+                    cell.value = ILLEGAL_CHARACTERS_RE.sub(
+                        sanitize_illegal_chars, str(col)
+                    )
                 except ValueError:
                     cell.value = str(col)
 

tests/test_tablib.py

@@ -1482,6 +1482,38 @@ def test_xlsx_wrong_char(self):
             data.append(('string', b'\x0cf'))
             data.xlsx
 
+    def test_xlsx_export_illegal_char_raises_by_default(self):
+        """Control chars illegal in XLSX raise IllegalCharacterError by default (#370)."""
+        from openpyxl.utils.exceptions import IllegalCharacterError
+
+        data.append(('\x1f',))
+        with self.assertRaises(IllegalCharacterError):
+            data.export('xlsx')
+
+    def test_xlsx_export_set_sanitize_illegal_chars(self):
+        """Illegal control chars are replaced when sanitize_illegal_chars is set (#370)."""
+        data.append(('a\x1fb',))
+
+        # Strip with empty string.
+        _xlsx = data.export('xlsx', sanitize_illegal_chars='')
+        wb = load_workbook(filename=BytesIO(_xlsx))
+        self.assertEqual('ab', wb.active['A1'].value)
+
+        # Replace with a substitute character.
+        _xlsx = data.export('xlsx', sanitize_illegal_chars='?')
+        wb = load_workbook(filename=BytesIO(_xlsx))
+        self.assertEqual('a?b', wb.active['A1'].value)
+
+    def test_xlsx_export_book_sanitize_illegal_chars(self):
+        """sanitize_illegal_chars also applies to Databook export (#370)."""
+        data.append(('x\x0cy',))
+        _book = tablib.Databook()
+        _book.add_sheet(data)
+
+        _xlsx = _book.export('xlsx', sanitize_illegal_chars='-')
+        wb = load_workbook(filename=BytesIO(_xlsx))
+        self.assertEqual('x-y', wb.active['A1'].value)
+
     def test_xlsx_cell_values(self):
         """Test cell values are read and not formulas"""
         xls_source = Path(__file__).parent / 'files' / 'xlsx_cell_values.xlsx'