Corrige problema de memory leak

requirements.txt

@@ -6,6 +6,7 @@ pdfminer.six==20200517
 pdfplumber==0.5.25
 Pillow==8.1.0
 progressbar==2.5
+psutil==5.8.0
 pycryptodome==3.9.9
 python-dateutil==2.8.1
 pytz==2020.5

src/extract_data.py

@@ -1,11 +1,13 @@
 # coding: utf-8
+import gc
 import os
 import sys
 
 import json
 from decimal import Decimal
 
 import pdfplumber
+import psutil
 from validate_docbr import CPF
 
 from progress_download import ProgressDownload
@@ -47,7 +49,53 @@ def __init__(self, input_path, output_path):
                         'role'
                     ]
 
-        self.__open_file()
+        self.__pdf = None
+
+        # FIXME: Use '__find_columns' function to find the columns
+        #  dynamically. Do not receive them statically, initialize
+        #  this variable with 'None' value.
+
+        self.__columns = [
+            Decimal('38.904'),
+            Decimal('210.530'),
+            Decimal('273.050'),
+            Decimal('315.890'),
+            Decimal('455.110'),
+            Decimal('520.900'),
+            Decimal('586.300'),
+            Decimal('662.020'),
+            Decimal('763.599')
+        ]
+
+    @property
+    def pdf(self):
+
+        # FIXME: Remove this condition after making sure that the memory
+        #  leak problem is completely resolved.
+
+        if self.__pdf and (psutil.virtual_memory().percent > 90):
+            print('Exhausted {} bytes, reopening file...'.format(psutil.virtual_memory().used))
+
+            del self.__pdf
+            self.__pdf = None
+
+            print('Forcing garbage collector to release memory...')
+
+            gc.collect()
+
+        if not self.__pdf:
+            print('Opening file...')
+
+            self.__pdf = pdfplumber.open(self.input_path)
+
+        return self.__pdf
+
+    @property
+    def columns(self):
+        if not self.__columns:
+            self.__columns = self.__find_columns()
+
+        return self.__columns
 
     def __find_header_index(self, table):
         for i in range(len(table)):
@@ -64,24 +112,11 @@ def __format_cpf(cpf):
             cpf = cpf.zfill(11)
         return '{}.{}.{}-{}'.format(cpf[:3], cpf[3:6], cpf[6:9], cpf[9:])
 
-    def __open_file(self):
+    def __find_columns(self):
 
-        print('Opening file...')
+        # FIXME: Memory leak issue still occur in this function (at 2021-02-14),
+        #  while it is not resolved, the function is not being used
 
-        self.pdf = pdfplumber.open(self.input_path)
-        self.columns = [
-            Decimal('38.904'),
-            Decimal('210.530'),
-            Decimal('273.050'),
-            Decimal('315.890'),
-            Decimal('455.110'),
-            Decimal('520.900'),
-            Decimal('586.300'),
-            Decimal('662.020'),
-            Decimal('763.599')
-        ]
-
-    def __find_columns(self):
         candidate_cols = {}
         num_cols_to_find = len(self.header)+1
         allowed_minimum_distance_between_cols = 25  # pixels
@@ -125,6 +160,8 @@ def __find_columns(self):
             else:
                 candidate_cols[cols] = 1
 
+            self.pdf.pages[page].flush_cache()
+
         # Sort candidate columns by number of appearances
         candidate_cols = dict(sorted(candidate_cols.items(), key=lambda value: value[1], reverse=True))
 
@@ -221,6 +258,8 @@ def process(self):
 
                 i += 1
 
+            self.pdf.pages[page].flush_cache()
+
             progress_download(page+1, 1, pages)
 
         print('Saving output file...')