diff --git a/requirements.txt b/requirements.txt index 7aed156..db04e94 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ pdfminer.six==20200517 pdfplumber==0.5.25 Pillow==8.1.0 progressbar==2.5 +psutil==5.8.0 pycryptodome==3.9.9 python-dateutil==2.8.1 pytz==2020.5 diff --git a/src/extract_data.py b/src/extract_data.py index b85c6f6..3dfb3e6 100644 --- a/src/extract_data.py +++ b/src/extract_data.py @@ -1,4 +1,5 @@ # coding: utf-8 +import gc import os import sys @@ -6,6 +7,7 @@ from decimal import Decimal import pdfplumber +import psutil from validate_docbr import CPF from progress_download import ProgressDownload @@ -47,7 +49,53 @@ def __init__(self, input_path, output_path): 'role' ] - self.__open_file() + self.__pdf = None + + # FIXME: Use '__find_columns' function to find the columns + # dynamically. Do not receive them statically, initialize + # this variable with 'None' value. + + self.__columns = [ + Decimal('38.904'), + Decimal('210.530'), + Decimal('273.050'), + Decimal('315.890'), + Decimal('455.110'), + Decimal('520.900'), + Decimal('586.300'), + Decimal('662.020'), + Decimal('763.599') + ] + + @property + def pdf(self): + + # FIXME: Remove this condition after making sure that the memory + # leak problem is completely resolved. + + if self.__pdf and (psutil.virtual_memory().percent > 90): + print('Exhausted {} bytes, reopening file...'.format(psutil.virtual_memory().used)) + + del self.__pdf + self.__pdf = None + + print('Forcing garbage collector to release memory...') + + gc.collect() + + if not self.__pdf: + print('Opening file...') + + self.__pdf = pdfplumber.open(self.input_path) + + return self.__pdf + + @property + def columns(self): + if not self.__columns: + self.__columns = self.__find_columns() + + return self.__columns def __find_header_index(self, table): for i in range(len(table)): @@ -64,24 +112,11 @@ def __format_cpf(cpf): cpf = cpf.zfill(11) return '{}.{}.{}-{}'.format(cpf[:3], cpf[3:6], cpf[6:9], cpf[9:]) - def __open_file(self): + def __find_columns(self): - print('Opening file...') + # FIXME: Memory leak issue still occur in this function (at 2021-02-14), + # while it is not resolved, the function is not being used - self.pdf = pdfplumber.open(self.input_path) - self.columns = [ - Decimal('38.904'), - Decimal('210.530'), - Decimal('273.050'), - Decimal('315.890'), - Decimal('455.110'), - Decimal('520.900'), - Decimal('586.300'), - Decimal('662.020'), - Decimal('763.599') - ] - - def __find_columns(self): candidate_cols = {} num_cols_to_find = len(self.header)+1 allowed_minimum_distance_between_cols = 25 # pixels @@ -125,6 +160,8 @@ def __find_columns(self): else: candidate_cols[cols] = 1 + self.pdf.pages[page].flush_cache() + # Sort candidate columns by number of appearances candidate_cols = dict(sorted(candidate_cols.items(), key=lambda value: value[1], reverse=True)) @@ -221,6 +258,8 @@ def process(self): i += 1 + self.pdf.pages[page].flush_cache() + progress_download(page+1, 1, pages) print('Saving output file...')