From 83f0e9f9f998e309c46023cc026dd7e5ec033dc9 Mon Sep 17 00:00:00 2001 From: Jonathan Guerne Date: Mon, 15 Mar 2021 12:17:48 +0100 Subject: [PATCH 1/4] CO-3589 change smb et sftp for interaction with NAS regarding letters --- .../data/import_config_templates.xml | 8 +- sbc_switzerland/data/scan_letter_params.xml | 4 +- sbc_switzerland/models/correspondence.py | 118 +++-- .../models/import_letters_history.py | 501 ++++++++---------- .../models/import_letters_history.py | 14 +- 5 files changed, 284 insertions(+), 361 deletions(-) diff --git a/sbc_switzerland/data/import_config_templates.xml b/sbc_switzerland/data/import_config_templates.xml index a32470a43..f2effbf05 100644 --- a/sbc_switzerland/data/import_config_templates.xml +++ b/sbc_switzerland/data/import_config_templates.xml @@ -3,23 +3,23 @@ Standard - \Scan Lettres S2B\Standard + Standard Standard with attachments sent_by_mail - \Scan Lettres S2B\Standard_With_Attachments + Standard_With_Attachments No Standard - \Scan Lettres S2B\No_Standard + No_Standard No Standard with attachment sent_by_mail - \Scan Lettres S2B\No_Standard_With_Attachments + No_Standard_With_Attachments Web Letter diff --git a/sbc_switzerland/data/scan_letter_params.xml b/sbc_switzerland/data/scan_letter_params.xml index 830829d3e..f1737590c 100644 --- a/sbc_switzerland/data/scan_letter_params.xml +++ b/sbc_switzerland/data/scan_letter_params.xml @@ -3,11 +3,11 @@ sbc_switzerland.share_on_nas - Public + /share/Scan sbc_switzerland.scan_letter_imported - \Scan Lettres S2B\Imports\ + Imports sbc_switzerland.scan_letter_done diff --git a/sbc_switzerland/models/correspondence.py b/sbc_switzerland/models/correspondence.py index 1377c5874..fdeff9fbb 100644 --- a/sbc_switzerland/models/correspondence.py +++ b/sbc_switzerland/models/correspondence.py @@ -21,12 +21,12 @@ from odoo.exceptions import UserError from odoo.addons.sbc_compassion.models.correspondence_page import BOX_SEPARATOR - logger = logging.getLogger(__name__) try: from PyPDF2 import PdfFileReader, PdfFileWriter from smb.SMBConnection import SMBConnection + import pysftp except ImportError: logger.warning("Please install pyPdf and smb.") @@ -84,7 +84,7 @@ def create(self, vals): correspondence = super( Correspondence, self.with_context(no_comm_kit=True) ).create(vals) - correspondence.send_local_translate() + # correspondence.send_local_translate() else: correspondence = super().create(vals) @@ -411,59 +411,69 @@ def _transfer_file_on_nas(self, file_name): """ self.ensure_one() # Retrieve configuration - smb_user = config.get("smb_user") - smb_pass = config.get("smb_pwd") - smb_ip = config.get("smb_ip") - smb_port = int(config.get("smb_port", 0)) - if not (smb_user and smb_pass and smb_ip and smb_port): - raise Exception("No config SMB in file .conf") + sftp_user = config.get("sftp_user") + sftp_pass = config.get("sftp_pwd") + sftp_ip = config.get("sftp_ip") + sftp_port = int(config.get("sftp_port", 0)) + if not (sftp_user and sftp_pass and sftp_ip and sftp_port): + raise Exception("No config SFTP in file .conf") - # Copy file in the imported letter folder - smb_conn = SMBConnection(smb_user, smb_pass, "openerp", "nas") - if smb_conn.connect(smb_ip, smb_port): - file_ = BytesIO(self.get_image()) - nas_share_name = self.env.ref("sbc_switzerland.nas_share_name").value + cnopts = pysftp.CnOpts() + # TODO import hostkey for compassion nas + cnopts.hostkeys = None - nas_letters_store_path = ( - self.env.ref( - "sbc_switzerland.nas_letters_store_path").value + file_name - ) - smb_conn.storeFile(nas_share_name, nas_letters_store_path, file_) - else: - raise UserError(_("Connection to NAS failed")) + if cnopts.hostkeys is None: + logger.warning("No hostkeys defined in StfpConnection. Connection will be unsecured.") - # CRON Methods - ############## - @api.model - def check_local_translation_done(self): - reload(sys) - tc = translate_connector.TranslateConnect() - letters_to_update = tc.get_translated_letters() - - for letter in letters_to_update: - try: - with api.Environment.manage(): - with registry(self.env.cr.dbname).cursor() as new_cr: - # Create a new environment with new cursor database - new_env = api.Environment( - new_cr, self.env.uid, self.env.context - ) - correspondence = self.with_env(new_env).browse( - letter["letter_odoo_id"] - ) - logger.info( - f".....CHECK TRANSLATION FOR LETTER {correspondence.id}" - ) - correspondence.update_translation( - letter["target_lang"], - letter["text"], - letter["translator"], - letter["src_lang"], - ) - tc.update_translation_to_treated(letter["id"]) - except: - logger.error( - "Error fetching a translation on translation platform", - exc_info=True, + # Copy file in the imported letter folder + try: + sftp_conn = pysftp.Connection(host=sftp_ip, password=sftp_pass, username=sftp_user, + port=sftp_port, cnopts=cnopts) + except Exception: + raise UserError(_("Connection to NA failed.")) + + with sftp_conn as sftp: + file_ = BytesIO(self.get_image()) + with sftp.cd(self.env.ref("sbc_switzerland.nas_share_name").value): + nas_letters_store_path = ( + self.env.ref( + "sbc_switzerland.nas_letters_store_path").value + file_name ) - return True + sftp.putfo(file_, nas_letters_store_path) + + +# CRON Methods +############## +@api.model +def check_local_translation_done(self): + reload(sys) + tc = translate_connector.TranslateConnect() + letters_to_update = tc.get_translated_letters() + + for letter in letters_to_update: + try: + with api.Environment.manage(): + with registry(self.env.cr.dbname).cursor() as new_cr: + # Create a new environment with new cursor database + new_env = api.Environment( + new_cr, self.env.uid, self.env.context + ) + correspondence = self.with_env(new_env).browse( + letter["letter_odoo_id"] + ) + logger.info( + f".....CHECK TRANSLATION FOR LETTER {correspondence.id}" + ) + correspondence.update_translation( + letter["target_lang"], + letter["text"], + letter["translator"], + letter["src_lang"], + ) + tc.update_translation_to_treated(letter["id"]) + except: + logger.error( + "Error fetching a translation on translation platform", + exc_info=True, + ) + return True diff --git a/sbc_switzerland/models/import_letters_history.py b/sbc_switzerland/models/import_letters_history.py index 20beb5422..9b21f4d05 100644 --- a/sbc_switzerland/models/import_letters_history.py +++ b/sbc_switzerland/models/import_letters_history.py @@ -28,14 +28,72 @@ logger = logging.getLogger(__name__) try: - from smb.SMBConnection import SMBConnection from smb.smb_structs import OperationFailure from PyPDF2 import PdfFileWriter, PdfFileReader from PyPDF2.pdf import PageObject + import pysftp except ImportError: logger.warning("Please install python dependencies.") +class SftpConnection: + """ + Class helper used to handle connection between server and SFTP server. + """ + + def __init__(self): + self.sftp_config = { + "username": config.get("sftp_user"), + "password": config.get("sftp_pwd"), + "host": config.get("sftp_ip"), + "port": int(config.get("sftp_port", 22)) + } + + self.credential_ok = all(self.sftp_config.values()) + + if not self.credential_ok: + logger.error( + """ + Missing credentials for sftp connection to NAS. + Please update configuration file with : + - sftp_user + - sftp_pwd + - sftp_ip + - sftp_port (default 22) + """ + ) + + cnopts = pysftp.CnOpts() + # TODO import hostkey for compassion nas + cnopts.hostkeys = None + + if cnopts.hostkeys is None: + logger.warning( + """ + No hostkeys defined in StfpConnection. Connection will be unsecured. + """ + ) + + self.sftp_config.update({"cnopts": cnopts}) + + def get_connection(self, starting_dir=None): + """ + Use instance config to start a connection with sftp server + raise AssertionError if credential are not set or IOError if + starting_dir is not None and points to a not existing directory + :param starting_dir: (optional) starting dir for the connection + :return: Return a pysftp connection object + """ + + assert self.credential_ok, "Missing credentials for sftp connection to NAS." + + conn = pysftp.Connection(**self.sftp_config) + if starting_dir: + conn.chdir(starting_dir) + + return conn + + class ImportLettersHistory(models.Model): """ Keep history of imported letters. @@ -58,76 +116,85 @@ def _compute_nber_letters(self): Counts the number of scans. If a zip file is given, the number of scans inside is counted. """ - for letter in self: - if letter.manual_import or (letter.state and letter.state != "draft"): - super(ImportLettersHistory, letter)._compute_nber_letters() - else: - # files are not selected by user so we find them on NAS - # folder 'Imports' counter - tmp = 0 - - smb_conn = letter._get_smb_connection() - share_nas = letter.env.ref("sbc_switzerland.share_on_nas").value - imported_letter_path = letter.import_folder_path - if ( - smb_conn - and smb_conn.connect(SmbConfig.smb_ip, SmbConfig.smb_port) - and imported_letter_path - ): - imported_letter_path = letter.check_path(imported_letter_path) - - try: - list_paths = smb_conn.listPath(share_nas, imported_letter_path) - except OperationFailure: - logger.error("--------------- PATH NOT CORRECT ---------------") - list_paths = [] - - for shared_file in list_paths: - if func.check_file(shared_file.filename) == 1: - tmp += 1 - elif func.is_zip(shared_file.filename): - logger.debug( - "File to retrieve:" - f" {imported_letter_path + shared_file.filename}" - ) + # files are not selected by user so we find them on NAS + count_from_nas_letters = self.filtered(lambda x: + (x.state and x.state == "draft") and not x.manual_import) - file_obj = BytesIO() - smb_conn.retrieveFile( - share_nas, - imported_letter_path + shared_file.filename, - file_obj, - ) - try: - zip_ = zipfile.ZipFile(file_obj, "r") - list_file = zip_.namelist() - # loop over all files in zip - for tmp_file in list_file: - tmp += func.check_file(tmp_file) == 1 - except zipfile.BadZipfile: - raise exceptions.UserError( - _( - "Zip file corrupted (" - + shared_file.filename - + ")" - ) + for letter in self.filtered(lambda x: x not in count_from_nas_letters): + super(ImportLettersHistory, letter)._compute_nber_letters() + + # avoid setting up smtp connection if unneeded + if not len(count_from_nas_letters): + return + + sftp_con_handler = SftpConnection() + sftp = None + try: + sftp = sftp_con_handler.get_connection() + except (AssertionError, IOError) as e: + logger.error("Could not establish connection with sftp server") + count_from_nas_letters.write({"nber_letters": 0}) + return + + for letter in count_from_nas_letters: + # folder 'Imports' counter + imported_letter_path = letter.check_path(letter.import_folder_path) + + if not imported_letter_path: + logger.error("Failed to list files in imported folder due to unset import folder path") + letter.nber_letters = 0 + continue + + tmp = 0 + with sftp.cd(self.env.ref("sbc_switzerland.share_on_nas").value): + + try: + list_paths = sftp.listdir(imported_letter_path) + except FileNotFoundError: + logger.error("--------------- PATH NOT CORRECT ---------------") + list_paths = [] + + for shared_file in list_paths: + if func.check_file(shared_file) == 1: + tmp += 1 + elif func.is_zip(shared_file): + logger.debug( + "File to retrieve:" + f" {imported_letter_path + shared_file}" + ) + + file_obj = BytesIO() + sftp.getfo( + imported_letter_path + shared_file, + flo=file_obj, + ) + try: + zip_ = zipfile.ZipFile(file_obj, "r") + list_file = zip_.namelist() + # loop over all files in zip + for tmp_file in list_file: + tmp += func.check_file(tmp_file) == 1 + except zipfile.BadZipfile: + raise exceptions.UserError( + _( + "Zip file corrupted (" + + shared_file + + ")" ) - except zipfile.LargeZipFile: - raise exceptions.UserError( - _( - "Zip64 is not supported(" - + shared_file.filename - + ")" - ) + ) + except zipfile.LargeZipFile: + raise exceptions.UserError( + _( + "Zip64 is not supported(" + + shared_file + + ")" ) - smb_conn.close() - else: - logger.error( - f"""Failed to list files in imported folder \ - Imports oh the NAS in emplacement: {imported_letter_path}""" - ) + ) + + letter.nber_letters = tmp - letter.nber_letters = tmp + sftp.close() ########################################################################## # VIEW CALLBACKS # @@ -141,7 +208,8 @@ def button_import(self): # when letters are in a folder on NAS redefine method for letters_import in self: letters_import.state = "pending" - if self.env.context.get("async_mode", True): + # TODO change for get(...,True) + if self.env.context.get("async_mode", False): letters_import.with_delay()._run_analyze() else: letters_import._run_analyze() @@ -152,7 +220,7 @@ def button_import(self): for letters_import in self: if letters_import.data and self.env.context.get("async_mode", True): for attachment in letters_import.data: - self._save_imported_letter(attachment) + self._save_manual_imported_letter(attachment) return super().button_import() @@ -171,72 +239,21 @@ def button_save(self): import_letters.config_id.name == self.env.ref("sbc_switzerland.web_letter").name ): - for line in import_letters.import_line_ids: - # build part of filename, corresponding to this web - # import - # letters remove part after '-' caracter (including) - # corresponding to the page number - part_filename = (line.file_name[:-4]).split("-")[0] - share_nas = self.env.ref("sbc_switzerland.share_on_nas").value - smb_conn = self._get_smb_connection() - if smb_conn and smb_conn.connect( - SmbConfig.smb_ip, SmbConfig.smb_port - ): - imported_letter_path = self.env.ref( - "sbc_switzerland.scan_letter_imported" - ).value - list_paths = smb_conn.listPath( - share_nas, imported_letter_path - ) - # loop in import folder and find pdf corresponding - # to this web letter and eventual attached image - image_ext = None - file_to_save = False - for shared_file in list_paths: - ext = shared_file.filename[-4:] - if part_filename == shared_file.filename[:-4]: - if ext == ".pdf": - file_to_save = True - else: - image_ext = ext - - # move web letter on 'Done' folder on the NAS - if file_to_save: - filename = part_filename + ".pdf" - file_obj = BytesIO() - smb_conn.retrieveFile( - share_nas, imported_letter_path + filename, file_obj - ) - file_obj.seek(0) - self._copy_imported_to_done_letter( - filename, file_obj, False - ) - # delete files corresponding to web letter in - # 'Import' - try: - smb_conn.deleteFiles( - share_nas, imported_letter_path + filename - ) - except Exception as inst: - logger.warning( - f"Failed to delete pdf web letter {inst}" - ) - - # image is attached to this letter so we - # remove it - if image_ext: - try: - smb_conn.deleteFiles( - share_nas, - imported_letter_path - + part_filename - + image_ext, - ) - except Exception as inst: - logger.warning( - f"Failed to delete attached image {inst}" - ) + with SftpConnection().get_connection(self.env.ref("sbc_switzerland.share_on_nas").value) as sftp: + + imported_letter_path = self.env.ref("sbc_switzerland.scan_letter_imported").value + + # look for pdfs and images to remove + list_files = [l for l in sftp.listdir(imported_letter_path) if + l[:-4] in # could be pdf or image + map(lambda x: x[:-4].split("-")[0], # lines are store in multiple part + import_letters.import_line_ids.mapped('file_name'))] + try: + for file_to_remove in list_files: + sftp.remove(imported_letter_path + file_to_remove) + except Exception as inst: + logger.warning(f"Failed to delete pdf web letter {inst}") if import_letters.manual_import: self._manage_all_imported_files() @@ -258,6 +275,7 @@ def button_save(self): ######################################################################### # PRIVATE METHODS # ######################################################################### + @job(default_channel="root.sbc_compassion") @related_action(action="related_action_s2b_imports") def _run_analyze(self): @@ -268,49 +286,47 @@ def _run_analyze(self): - call _analyze_attachment for every resulting file """ self.ensure_one() + + if self.manual_import: + return super()._run_analyze() + # keep track of file names to detect duplicates # file_name_history = [] logger.info("Imported letters analysis started...") progress = 1 - share_nas = self.env.ref("sbc_switzerland.share_on_nas").value - - smb_conn = self._get_smb_connection() - if not self.manual_import: - imported_letter_path = self.check_path(self.import_folder_path) - if smb_conn and smb_conn.connect(SmbConfig.smb_ip, SmbConfig.smb_port): - list_paths = smb_conn.listPath(share_nas, imported_letter_path) + imported_letter_path = self.check_path(self.import_folder_path) + try: + with SftpConnection().get_connection(self.env.ref("sbc_switzerland.share_on_nas").value) as sftp: + list_paths = sftp.listdir(imported_letter_path) for shared_file in list_paths: - if func.check_file(shared_file.filename) == 1: - logger.info( - f"Analyzing letter {progress}" f"/{len(list_paths)}" - ) + + logger.info(f"Analyzing letter {progress}" f"/{len(list_paths)}") + + if func.check_file(shared_file) == 1: with NamedTemporaryFile() as file_obj: - smb_conn.retrieveFile( - share_nas, - imported_letter_path + shared_file.filename, + sftp.getfo( + imported_letter_path + shared_file, file_obj, ) self._analyze_attachment( - self._convert_pdf(file_obj), shared_file.filename + self._convert_pdf(file_obj), shared_file ) progress += 1 - elif func.is_zip(shared_file.filename): + elif func.is_zip(shared_file): zip_file = BytesIO() - # retrieve zip file from imports letters stored on - # NAS - smb_conn.retrieveFile( - share_nas, - imported_letter_path + shared_file.filename, + # retrieve zip file from imports letters stored on NAS + sftp.getfo( + imported_letter_path + shared_file, zip_file, ) zip_file.seek(0) zip_ = zipfile.ZipFile(zip_file, "r") # loop over files inside zip - files = len(zip_.namelist()) + files = zip_.namelist() for f in files: logger.info( f"Analyzing letter {progress}/{len(files)}" @@ -318,23 +334,18 @@ def _run_analyze(self): self._analyze_attachment(self._convert_pdf(zip_.open(f)), f) progress += 1 - smb_conn.close() - else: - logger.error( - f"Failed to list files in Imports on the NAS in \ - emplacement: {imported_letter_path}" - ) - else: - super()._run_analyze() + + except (AssertionError, IOError) as e: + logger.error("Could not establish connection with sftp server") + return # remove all the files (now they are inside import_line_ids) self.data.unlink() - if not self.manual_import: - self._manage_all_imported_files() + self._manage_all_imported_files() self.import_completed = True logger.info("Imported letters analysis completed.") - def _save_imported_letter(self, attachment): + def _save_manual_imported_letter(self, attachment): """ Save attachment letter to a shared folder on the NAS ('Imports') - attachment : the attachment to save @@ -342,25 +353,18 @@ def _save_imported_letter(self, attachment): """ # Store letter on a shared folder on the NAS: # Copy file in the imported letter folder - smb_conn = self._get_smb_connection() - if smb_conn and smb_conn.connect(SmbConfig.smb_ip, SmbConfig.smb_port): + with SftpConnection().get_connection(self.env.ref("sbc_switzerland.share_on_nas").value) as sftp: file_ = BytesIO( base64.b64decode(attachment.with_context(bin_size=False).datas) ) - share_nas = self.env.ref("sbc_switzerland.share_on_nas").value - if self.manual_import: - imported_letter_path = ( - self.env.ref("sbc_switzerland.scan_letter_imported").value + imported_letter_path = ( + self.check_path(self.env.ref("sbc_switzerland.scan_letter_imported").value) + attachment.name - ) - else: - imported_letter_path = ( - self.check_path(self.import_folder_path) + attachment.name - ) - - smb_conn.storeFile(share_nas, imported_letter_path, file_) - smb_conn.close() + ) + + sftp.putfo(file_, imported_letter_path) + return True def _manage_all_imported_files(self): @@ -370,43 +374,35 @@ def _manage_all_imported_files(self): - Delete files from their import location on the NAS Done by Michael Sandoz 02.2016 """ - share_nas = self.env.ref("sbc_switzerland.share_on_nas").value - # to delete after treatment - list_zip_to_delete = [] + imported_letter_path = "" if self.manual_import: imported_letter_path = self.env.ref( "sbc_switzerland.scan_letter_imported" ).value else: - imported_letter_path = self.check_path(self.import_folder_path) + imported_letter_path = self.import_folder_path + + imported_letter_path = self.check_path(imported_letter_path) + + with SftpConnection().get_connection(self.env.ref("sbc_switzerland.share_on_nas").value) as sftp: - smb_conn = self._get_smb_connection() - if smb_conn and smb_conn.connect(SmbConfig.smb_ip, SmbConfig.smb_port): + for shared_file in sftp.listdir(imported_letter_path): - list_paths = smb_conn.listPath(share_nas, imported_letter_path) - for shared_file in list_paths: - if func.check_file(shared_file.filename) == 1: + if func.check_file(shared_file) == 1: # when this is manual import we don't have to copy all # files, web letters are stored in the same folder... - if not self.manual_import or self.is_in_list_letter( - shared_file.filename - ): - file_obj = BytesIO() - smb_conn.retrieveFile( - share_nas, - imported_letter_path + shared_file.filename, - file_obj, - ) - file_obj.seek(0) - self._copy_imported_to_done_letter( - shared_file.filename, file_obj, True - ) - elif func.is_zip(shared_file.filename): + if not self.manual_import or self.is_in_list_letter(shared_file): + try: + sftp.remove(imported_letter_path + shared_file) + except Exception as inst: + logger.warning("Failed to delete a file on NAS : {}".format(inst)) + + elif func.is_zip(shared_file): zip_file = BytesIO() - smb_conn.retrieveFile( - share_nas, imported_letter_path + shared_file.filename, zip_file + sftp.getfo( + imported_letter_path + shared_file, zip_file ) zip_file.seek(0) zip_ = zipfile.ZipFile(zip_file, "r") @@ -416,21 +412,13 @@ def _manage_all_imported_files(self): # when this is manual import we are not sure that this # zip contains current letters treated if not self.manual_import or self.is_in_list_letter(f): - self._copy_imported_to_done_letter( - f, BytesIO(zip_.read(f)), False - ) zip_to_remove = True if zip_to_remove: - list_zip_to_delete.append(shared_file.filename) - - # delete zip file from origin import folder on the NAS - for filename in list_zip_to_delete: - try: - smb_conn.delete_files(share_nas, imported_letter_path + filename) - except Exception as inst: - logger.warning(f"Failed to delete zip file on NAS: {inst}") - smb_conn.close() + try: + sftp.remove(imported_letter_path + shared_file) + except Exception as inst: + logger.warning("Failed to delete a file on NAS : {}".format(inst)) def is_in_list_letter(self, filename): """ @@ -441,78 +429,12 @@ def is_in_list_letter(self, filename): return True return False - def _copy_imported_to_done_letter(self, filename, file_to_copy, delete_file): - """ - Copy letter from 'imported' folder to 'done' folder on a shared folder - on NAS - - filename: filename to give to saved file - - file_to_copy: the file to copy - - delete_file: set to true if file_to_copy must be deleted after - the copy - Done by Michael Sandoz 02.2016 - """ - smb_conn = self._get_smb_connection() - if smb_conn and smb_conn.connect(SmbConfig.smb_ip, SmbConfig.smb_port): - # Copy file in attachment in the done letter folder - share_nas = self.env.ref("sbc_switzerland.share_on_nas").value - - # Add end path corresponding to the end of the import folder path - end_path = "" - if self.import_folder_path: - end_path = ( - self.check_path(self.import_folder_path) - .replace("\\", "/") - .replace("//", "/") - ) - end_path = end_path.split("/")[-2] + "/" - done_letter_path = ( - self.env.ref("sbc_switzerland.scan_letter_done").value - + end_path - + filename - ) - - smb_conn.storeFile(share_nas, done_letter_path, file_to_copy) - - # Delete file in the imported letter folder - if delete_file: - if self.manual_import: - imported_letter_path = ( - self.env.ref("sbc_switzerland.scan_letter_imported").value - + filename - ) - else: - imported_letter_path = ( - self.check_path(self.import_folder_path) + filename - ) - - try: - smb_conn.deleteFiles(share_nas, imported_letter_path) - except Exception as inst: - logger.warning("Failed to delete a file on NAS : {}".format(inst)) - - smb_conn.close() - return True - def check_path(self, path): """" Add backslash at end of path if not contains ever one """ - if path and not path[-1] == "\\": - path = path + "\\" + if path and not path[-1] == "/": + path = path + "/" return path - def _get_smb_connection(self): - """" Retrieve configuration SMB """ - if not ( - SmbConfig.smb_user - and SmbConfig.smb_pass - and SmbConfig.smb_ip - and SmbConfig.smb_port - ): - return False - else: - return SMBConnection( - SmbConfig.smb_user, SmbConfig.smb_pass, "openerp", "nas" - ) - def analyze_webletter(self, pdf_letter): """ Look if the web letter has a minimum of 2 page. @@ -564,12 +486,3 @@ def _convert_pdf(self, temp_pdf_file): return temp_pdf_file.read() output.seek(0) return output.read() - - -class SmbConfig: - """" Little class who contains SMB configuration """ - - smb_user = config.get("smb_user") - smb_pass = config.get("smb_pwd") - smb_ip = config.get("smb_ip") - smb_port = int(config.get("smb_port", 0)) diff --git a/wordpress_connector/models/import_letters_history.py b/wordpress_connector/models/import_letters_history.py index ad79f4258..f658f928b 100644 --- a/wordpress_connector/models/import_letters_history.py +++ b/wordpress_connector/models/import_letters_history.py @@ -18,7 +18,7 @@ from io import BytesIO from odoo.addons.sbc_compassion.tools import import_letter_functions as func -from odoo.addons.sbc_switzerland.models.import_letters_history import SmbConfig +from odoo.addons.sbc_switzerland.models.import_letters_history import SftpConnection from werkzeug.utils import escape from odoo import models, api, fields @@ -173,16 +173,17 @@ def import_web_letter( import_config.import_completed = True # Copy file in attachment in the done letter folder - share_nas = self.env.ref("sbc_switzerland.share_on_nas").value import_letter_path = ( self.env.ref( "sbc_switzerland.scan_letter_imported").value + filename ) file_pdf = BytesIO(pdf_letter) - smb_conn = self._get_smb_connection() - if smb_conn and smb_conn.connect(SmbConfig.smb_ip, SmbConfig.smb_port): - smb_conn.storeFile(share_nas, import_letter_path, file_pdf) + + sftp_conn = SftpConnection().get_connection(self.env.ref("sbc_switzerland.share_on_nas").value) + + with sftp_conn as sftp: + sftp.putfo(file_pdf, import_letter_path) # save eventual attachment if attachment_url: @@ -192,8 +193,7 @@ def import_web_letter( ) file_attachment = BytesIO(attachment_data) - smb_conn.storeFile(share_nas, import_letter_path, file_attachment) - smb_conn.close() + sftp.putfo(file_attachment, import_letter_path) # Accept privacy statement sponsor_id.set_privacy_statement(origin="new_letter") From d21ab08e04cade809c19ea9c8d7e49c66ff1c5d5 Mon Sep 17 00:00:00 2001 From: Jonathan Guerne Date: Thu, 29 Apr 2021 12:59:39 +0200 Subject: [PATCH 2/4] add connection with ssh key and remove no longer wanted call --- sbc_switzerland/data/nas_parameters.xml | 6 ++ .../models/import_letters_history.py | 62 ++++++------------- .../models/import_letters_history.py | 3 +- 3 files changed, 26 insertions(+), 45 deletions(-) diff --git a/sbc_switzerland/data/nas_parameters.xml b/sbc_switzerland/data/nas_parameters.xml index 3f777114e..e8bf76707 100644 --- a/sbc_switzerland/data/nas_parameters.xml +++ b/sbc_switzerland/data/nas_parameters.xml @@ -1,5 +1,11 @@ + + + sbc_switzerland.nas_ssh_key + TO_CHANGE + + sbc_switzerland.nas_share_name GP diff --git a/sbc_switzerland/models/import_letters_history.py b/sbc_switzerland/models/import_letters_history.py index 9b21f4d05..9265f8f4e 100644 --- a/sbc_switzerland/models/import_letters_history.py +++ b/sbc_switzerland/models/import_letters_history.py @@ -32,6 +32,7 @@ from PyPDF2 import PdfFileWriter, PdfFileReader from PyPDF2.pdf import PageObject import pysftp + from pysftp import RSAKey except ImportError: logger.warning("Please install python dependencies.") @@ -41,7 +42,7 @@ class SftpConnection: Class helper used to handle connection between server and SFTP server. """ - def __init__(self): + def __init__(self, key_data=None): self.sftp_config = { "username": config.get("sftp_user"), "password": config.get("sftp_pwd"), @@ -64,15 +65,15 @@ def __init__(self): ) cnopts = pysftp.CnOpts() - # TODO import hostkey for compassion nas - cnopts.hostkeys = None - if cnopts.hostkeys is None: + try: + key = RSAKey(data=base64.decodebytes(key_data.encode('utf-8'))) + cnopts.hostkeys.add(config.get("sftp_ip"), "ssh-rsa", key) + except: + cnopts.hostkeys = None logger.warning( - """ - No hostkeys defined in StfpConnection. Connection will be unsecured. - """ - ) + "No hostkeys defined in StfpConnection. Connection will be unsecured. " + "Please configure parameter sbc_switzerland.nas_ssh_key with ssh_key data.") self.sftp_config.update({"cnopts": cnopts}) @@ -128,7 +129,7 @@ def _compute_nber_letters(self): if not len(count_from_nas_letters): return - sftp_con_handler = SftpConnection() + sftp_con_handler = SftpConnection(self.env.ref("sbc_switzerland.nas_ssh_key").value) sftp = None try: sftp = sftp_con_handler.get_connection() @@ -208,21 +209,13 @@ def button_import(self): # when letters are in a folder on NAS redefine method for letters_import in self: letters_import.state = "pending" - # TODO change for get(...,True) - if self.env.context.get("async_mode", False): + if self.env.context.get("async_mode", True): letters_import.with_delay()._run_analyze() else: letters_import._run_analyze() return True - else: - # when letters selected by user, save them on NAS and call - # super method - for letters_import in self: - if letters_import.data and self.env.context.get("async_mode", True): - for attachment in letters_import.data: - self._save_manual_imported_letter(attachment) - return super().button_import() + return super().button_import() @api.multi def button_save(self): @@ -240,7 +233,8 @@ def button_save(self): == self.env.ref("sbc_switzerland.web_letter").name ): - with SftpConnection().get_connection(self.env.ref("sbc_switzerland.share_on_nas").value) as sftp: + with SftpConnection(self.env.ref("sbc_switzerland.nas_ssh_key").value).get_connection( + self.env.ref("sbc_switzerland.share_on_nas").value) as sftp: imported_letter_path = self.env.ref("sbc_switzerland.scan_letter_imported").value @@ -297,7 +291,8 @@ def _run_analyze(self): imported_letter_path = self.check_path(self.import_folder_path) try: - with SftpConnection().get_connection(self.env.ref("sbc_switzerland.share_on_nas").value) as sftp: + with SftpConnection(self.env.ref("sbc_switzerland.nas_ssh_key").value).get_connection( + self.env.ref("sbc_switzerland.share_on_nas").value) as sftp: list_paths = sftp.listdir(imported_letter_path) for shared_file in list_paths: @@ -345,28 +340,6 @@ def _run_analyze(self): self.import_completed = True logger.info("Imported letters analysis completed.") - def _save_manual_imported_letter(self, attachment): - """ - Save attachment letter to a shared folder on the NAS ('Imports') - - attachment : the attachment to save - Done by Michael Sandoz 02.2016 - """ - # Store letter on a shared folder on the NAS: - # Copy file in the imported letter folder - with SftpConnection().get_connection(self.env.ref("sbc_switzerland.share_on_nas").value) as sftp: - file_ = BytesIO( - base64.b64decode(attachment.with_context(bin_size=False).datas) - ) - - imported_letter_path = ( - self.check_path(self.env.ref("sbc_switzerland.scan_letter_imported").value) - + attachment.name - ) - - sftp.putfo(file_, imported_letter_path) - - return True - def _manage_all_imported_files(self): """ File management at the end of correct import: @@ -385,7 +358,8 @@ def _manage_all_imported_files(self): imported_letter_path = self.check_path(imported_letter_path) - with SftpConnection().get_connection(self.env.ref("sbc_switzerland.share_on_nas").value) as sftp: + with SftpConnection(self.env.ref("sbc_switzerland.nas_ssh_key").value).get_connection( + self.env.ref("sbc_switzerland.share_on_nas").value) as sftp: for shared_file in sftp.listdir(imported_letter_path): diff --git a/wordpress_connector/models/import_letters_history.py b/wordpress_connector/models/import_letters_history.py index f658f928b..8271a9189 100644 --- a/wordpress_connector/models/import_letters_history.py +++ b/wordpress_connector/models/import_letters_history.py @@ -180,7 +180,8 @@ def import_web_letter( file_pdf = BytesIO(pdf_letter) - sftp_conn = SftpConnection().get_connection(self.env.ref("sbc_switzerland.share_on_nas").value) + sftp_conn = SftpConnection(self.env.ref("sbc_switzerland.nas_ssh_key").value).\ + get_connection(self.env.ref("sbc_switzerland.share_on_nas").value) with sftp_conn as sftp: sftp.putfo(file_pdf, import_letter_path) From 00a3d2a98697eb7a6ee932845902507147a9c59d Mon Sep 17 00:00:00 2001 From: Jonathan Guerne Date: Tue, 18 May 2021 12:20:44 +0200 Subject: [PATCH 3/4] CO-3627 remplace samba by sftp on forget me action --- partner_compassion/__manifest__.py | 2 +- .../models/partner_compassion.py | 61 +++++++++++-------- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/partner_compassion/__manifest__.py b/partner_compassion/__manifest__.py index d01d69537..387ee45c9 100644 --- a/partner_compassion/__manifest__.py +++ b/partner_compassion/__manifest__.py @@ -54,7 +54,7 @@ "l10n_ch_zip", # oca_addon/l10n_switzerland "web_view_google_map" # oca_addon/web_view_google_map ], - "external_dependencies": {"python": ["pandas", "pyminizip", "magic"]}, + "external_dependencies": {"python": ["pandas", "pyminizip", "magic", "pysftp"]}, "data": [ "security/ir.model.access.csv", "security/criminal_record_groups.xml", diff --git a/partner_compassion/models/partner_compassion.py b/partner_compassion/models/partner_compassion.py index c26b8b9e6..813d5f8b3 100644 --- a/partner_compassion/models/partner_compassion.py +++ b/partner_compassion/models/partner_compassion.py @@ -44,8 +44,8 @@ MAGIC_INSTALLED = True import pyminizip import csv - from smb.SMBConnection import SMBConnection - from smb.smb_structs import OperationFailure + import pysftp + from pysftp import RSAKey except ImportError: logger.warning("Please install python dependencies.", exc_info=True) @@ -620,18 +620,16 @@ def _secure_save_data(self): inside a password-protected ZIP file. :return: None """ - smb_conn = self._get_smb_connection() - if smb_conn and smb_conn.connect(SmbConfig.smb_ip, SmbConfig.smb_port): + sftp = self._get_sftp_connection() + if sftp: config_obj = self.env["ir.config_parameter"].sudo() - share_nas = config_obj.get_param("partner_compassion.share_on_nas") store_path = config_obj.get_param("partner_compassion.store_path") src_zip_file = tempfile.NamedTemporaryFile() - attrs = smb_conn.retrieveFile(share_nas, store_path, src_zip_file) - file_size = attrs[1] + file_size = sftp.getfo(store_path, src_zip_file) if file_size: src_zip_file.flush() zip_dir = tempfile.mkdtemp() - pyminizip.uncompress(src_zip_file.name, SmbConfig.file_pw, zip_dir, 0) + pyminizip.uncompress(src_zip_file.name, SftpConfig.file_pw, zip_dir, 0) csv_path = zip_dir + "/partner_data.csv" with open(csv_path, "a", newline="", encoding="utf-8") as csv_file: csv_writer = csv.writer(csv_file) @@ -644,29 +642,44 @@ def _secure_save_data(self): ] ) dst_zip_file = tempfile.NamedTemporaryFile() - pyminizip.compress( - csv_path, "", dst_zip_file.name, SmbConfig.file_pw, 5 - ) + pyminizip.compress(csv_path, "", dst_zip_file.name, SftpConfig.file_pw, 5) try: - smb_conn.storeFile(share_nas, store_path, dst_zip_file) - except OperationFailure: + sftp.putfo(dst_zip_file, store_path) + except Exception: logger.error( "Couldn't store secure partner data on NAS. " "Please do it manually by replicating the following " "file: " + dst_zip_file.name ) + finally: + src_zip_file.close() + dst_zip_file.close() - def _get_smb_connection(self): + def _get_sftp_connection(self): """" Retrieve configuration SMB """ if not ( - SmbConfig.smb_user - and SmbConfig.smb_pass - and SmbConfig.smb_ip - and SmbConfig.smb_port + SftpConfig.username + and SftpConfig.password + and SftpConfig.host + and SftpConfig.port ): return False else: - return SMBConnection(SmbConfig.smb_user, SmbConfig.smb_pass, "odoo", "nas") + + cnopts = pysftp.CnOpts() + + try: + key_data = self.env.ref("sbc_switzerland.nas_ssh_key").value + key = RSAKey(data=base64.decodebytes(key_data.encode('utf-8'))) + cnopts.hostkeys.add(SftpConfig.host, "ssh-rsa", key) + except: + cnopts.hostkeys = None + logger.warning( + "No hostkeys defined in StfpConnection. Connection will be unsecured. " + "Please configure parameter sbc_switzerland.nas_ssh_key with ssh_key data.") + + return pysftp.Connection(username=SftpConfig.username, password=SftpConfig.password, port=SftpConfig.port, + host=SftpConfig.host, cnopts=cnopts) def _get_active_sponsorships_domain(self): """ @@ -706,11 +719,11 @@ def _notify_prepare_email_values(self, message): return mail_values -class SmbConfig: +class SftpConfig: """" Little class who contains SMB configuration """ - smb_user = config.get("smb_user") - smb_pass = config.get("smb_pwd") - smb_ip = config.get("smb_ip") - smb_port = int(config.get("smb_port", 0)) + username = config.get("sftp_user") + password = config.get("sftp_pwd") + host = config.get("sftp_ip") + port = int(config.get("sftp_port", 22)) file_pw = config.get("partner_data_password") From 7a9d288c2e87fce1f93ca4a392099687e4bb1006 Mon Sep 17 00:00:00 2001 From: Emanuel Cino Date: Tue, 1 Jun 2021 16:43:15 +0200 Subject: [PATCH 4/4] FIX wrong indentation --- sbc_switzerland/models/correspondence.py | 71 ++++++++++++------------ 1 file changed, 35 insertions(+), 36 deletions(-) diff --git a/sbc_switzerland/models/correspondence.py b/sbc_switzerland/models/correspondence.py index 13ad9d3b9..6db2ed5ec 100644 --- a/sbc_switzerland/models/correspondence.py +++ b/sbc_switzerland/models/correspondence.py @@ -442,39 +442,38 @@ def _transfer_file_on_nas(self, file_name): ) sftp.putfo(file_, nas_letters_store_path) - -# CRON Methods -############## -@api.model -def check_local_translation_done(self): - reload(sys) - tc = translate_connector.TranslateConnect() - letters_to_update = tc.get_translated_letters() - - for letter in letters_to_update: - try: - with api.Environment.manage(): - with registry(self.env.cr.dbname).cursor() as new_cr: - # Create a new environment with new cursor database - new_env = api.Environment( - new_cr, self.env.uid, self.env.context - ) - correspondence = self.with_env(new_env).browse( - letter["letter_odoo_id"] - ) - logger.info( - f".....CHECK TRANSLATION FOR LETTER {correspondence.id}" - ) - correspondence.update_translation( - letter["target_lang"], - letter["text"], - letter["translator"], - letter["src_lang"], - ) - tc.update_translation_to_treated(letter["id"]) - except: - logger.error( - "Error fetching a translation on translation platform", - exc_info=True, - ) - return True + # CRON Methods + ############## + @api.model + def check_local_translation_done(self): + reload(sys) + tc = translate_connector.TranslateConnect() + letters_to_update = tc.get_translated_letters() + + for letter in letters_to_update: + try: + with api.Environment.manage(): + with registry(self.env.cr.dbname).cursor() as new_cr: + # Create a new environment with new cursor database + new_env = api.Environment( + new_cr, self.env.uid, self.env.context + ) + correspondence = self.with_env(new_env).browse( + letter["letter_odoo_id"] + ) + logger.info( + f".....CHECK TRANSLATION FOR LETTER {correspondence.id}" + ) + correspondence.update_translation( + letter["target_lang"], + letter["text"], + letter["translator"], + letter["src_lang"], + ) + tc.update_translation_to_treated(letter["id"]) + except: + logger.error( + "Error fetching a translation on translation platform", + exc_info=True, + ) + return True