feat: BI-6560 reuse values in excel parser (#1232)

feat: BI-6560 reuse values in excel parser (#1232)

Author: catsona

lib/dl_file_secure_reader_lib/dl_file_secure_reader_lib/resources/reader.py

@@ -3,21 +3,60 @@
 import asyncio
 import io
 import logging
-from typing import BinaryIO
+from typing import (
+    Any,
+    BinaryIO,
+)
 
 from aiohttp import web
 from aiohttp.multipart import BodyPartReader
+import attr
+import frozendict
 import openpyxl
+import openpyxl.cell.cell
 
 from dl_file_secure_reader_lib.settings import FileSecureReaderSettings
 
 
 LOGGER = logging.getLogger(__name__)
 
 
+@attr.s(frozen=True)
+class CachedCellProcessor:
+    """
+    Cache wrappers for cell values to prevent excessive memory usage when excel
+    file is made of duplicate values.
+    """
+
+    cache_values: dict[tuple[str, Any], Any] = attr.ib(factory=dict, init=False)
+
+    def process_cell(self, cell: openpyxl.cell.cell.Cell) -> dict:
+        if cell.data_type == "d":
+            cell_value = str(cell.value)
+        else:
+            cell_value = cell.value
+
+        if isinstance(cell.value, int):
+            cell_type = "i"
+        else:
+            cell_type = cell.data_type
+
+        cache_key = (cell_type, cell_value)
+
+        if cache_key not in self.cache_values:
+            self.cache_values[cache_key] = frozendict.frozendict(
+                data_type=cell_type,
+                value=cell_value,
+            )
+
+        return self.cache_values[cache_key]
+
+
 def parse_excel_data(data: BinaryIO, feature_excel_read_only: bool) -> list:
     result = []
 
+    cell_processor = CachedCellProcessor()
+
     try:
         wb = openpyxl.load_workbook(
             data,
@@ -36,16 +75,7 @@ def parse_excel_data(data: BinaryIO, feature_excel_read_only: bool) -> list:
         result.append(
             {
                 "sheetname": sheetname,
-                "data": [
-                    [
-                        {
-                            "value": str(cell.value) if cell.data_type == "d" else cell.value,
-                            "data_type": "i" if isinstance(cell.value, int) else cell.data_type,
-                        }
-                        for cell in row
-                    ]
-                    for row in sheet.rows
-                ],
+                "data": [[cell_processor.process_cell(cell) for cell in row] for row in sheet.rows],
             }
         )
     return result

lib/dl_file_secure_reader_lib/pyproject.toml

@@ -9,7 +9,9 @@ version = "0.1.0"
 
 [tool.poetry.dependencies]
 aiohttp = "*"
+attrs = "*"
 dl-settings = {path = "../dl_settings"}
+frozendict = "*"
 openpyxl = "*"
 pydantic = "*"
 python = ">=3.10, <3.13"