EVA-3586 - Multiple Fixes from running in production

eva-accession-release-automation/run_release_in_embassy/analyze_vcf_validation_results.py

@@ -19,7 +19,7 @@
 import sys
 
 from ebi_eva_common_pyutils.command_utils import run_command_with_output
-from ebi_eva_common_pyutils.logger import logging_config as log_cfg
+from ebi_eva_common_pyutils.logger import logging_config as log_cfg, logging_config
 from run_release_in_embassy.release_metadata import vcf_validation_output_file_pattern, asm_report_output_file_pattern
 
 logger = log_cfg.get_logger(__name__)
@@ -84,6 +84,7 @@ def analyze_vcf_validation_results(species_release_folder, assembly_accession):
 @click.option("--assembly-accession", required=True)
 @click.command()
 def main(species_release_folder, assembly_accession):
+    logging_config.add_stdout_handler()
     analyze_vcf_validation_results(species_release_folder, assembly_accession)
 
 

eva-accession-release-automation/run_release_in_embassy/copy_accessioning_collections_to_embassy.py

@@ -19,6 +19,7 @@
 import traceback
 
 from ebi_eva_common_pyutils.command_utils import run_command_with_output
+from ebi_eva_common_pyutils.logger import logging_config
 from ebi_eva_internal_pyutils.config_utils import get_mongo_uri_for_eva_profile
 from ebi_eva_internal_pyutils.metadata_utils import get_metadata_connection_handle
 from ebi_eva_internal_pyutils.mongo_utils import copy_db
@@ -153,6 +154,7 @@ def copy_accessioning_collections_to_embassy(private_config_xml_file, profile, t
 @click.command()
 def main(private_config_xml_file, profile, taxonomy_id, assembly_accession, collections_to_copy, release_species_inventory_table,
          release_version, dump_dir):
+    logging_config.add_stdout_handler()
     copy_accessioning_collections_to_embassy(private_config_xml_file, profile, taxonomy_id, assembly_accession,
                                              collections_to_copy, release_species_inventory_table, release_version,
                                              dump_dir)

eva-accession-release-automation/run_release_in_embassy/count_rs_ids_in_release_files.py

@@ -16,6 +16,8 @@
 import os
 
 from ebi_eva_common_pyutils.command_utils import run_command_with_output
+from ebi_eva_common_pyutils.logger import logging_config
+
 from run_release_in_embassy.release_metadata import release_vcf_file_categories, release_text_file_categories
 from run_release_in_embassy.release_common_utils import get_release_vcf_file_name_genbank, get_release_text_file_name
 
@@ -47,6 +49,7 @@ def count_rs_ids_in_release_files(count_ids_script_path, taxonomy_id, assembly_a
 @click.option("--species-release-folder", required=True)
 @click.command()
 def main(count_ids_script_path, taxonomy_id, assembly_accession, species_release_folder):
+    logging_config.add_stdout_handler()
     count_rs_ids_in_release_files(count_ids_script_path, taxonomy_id, assembly_accession, species_release_folder)
 
 

eva-accession-release-automation/run_release_in_embassy/create_release_properties_file.py

@@ -14,6 +14,7 @@
 import os
 
 import click
+from ebi_eva_common_pyutils.logger import logging_config
 from ebi_eva_internal_pyutils.metadata_utils import get_metadata_connection_handle
 from ebi_eva_internal_pyutils.spring_properties import SpringPropertiesGenerator
 
@@ -67,6 +68,7 @@ def create_release_properties_file_for_assembly(private_config_xml_file, profile
 @click.command()
 def main(private_config_xml_file, profile, taxonomy_id, assembly_accession, release_species_inventory_table,
          release_version, species_release_folder):
+    logging_config.add_stdout_handler()
     create_release_properties_file_for_assembly(private_config_xml_file, profile, taxonomy_id, assembly_accession,
                                                 release_species_inventory_table, release_version,
                                                 species_release_folder)

eva-accession-release-automation/run_release_in_embassy/initiate_release_status_for_assembly.py

@@ -15,6 +15,7 @@
 import click
 import logging
 
+from ebi_eva_common_pyutils.logger import logging_config
 
 from run_release_in_embassy.release_metadata import update_release_progress_status
 from ebi_eva_internal_pyutils.metadata_utils import get_metadata_connection_handle
@@ -42,6 +43,7 @@ def initiate_release_status_for_assembly(private_config_xml_file, profile, relea
 @click.command()
 def main(private_config_xml_file, profile, release_species_inventory_table, taxonomy_id, assembly_accession,
          release_version):
+    logging_config.add_stdout_handler()
     initiate_release_status_for_assembly(private_config_xml_file, profile, release_species_inventory_table,
                                          taxonomy_id, assembly_accession, release_version)
 

eva-accession-release-automation/run_release_in_embassy/merge_dbsnp_eva_release_files.py

@@ -18,6 +18,7 @@
 import os
 
 from ebi_eva_common_pyutils.command_utils import run_command_with_output
+from ebi_eva_common_pyutils.logger import logging_config
 from ebi_eva_internal_pyutils.metadata_utils import get_metadata_connection_handle
 from run_release_in_embassy.release_metadata import release_vcf_file_categories, release_text_file_categories, \
     get_release_inventory_info_for_assembly
@@ -205,6 +206,7 @@ def merge_dbsnp_eva_release_files(private_config_xml_file, profile, bgzip_path,
 @click.command()
 def main(private_config_xml_file, profile, bgzip_path, bcftools_path, vcf_sort_script_path, taxonomy_id,
          assembly_accession, release_species_inventory_table, release_version, species_release_folder):
+    logging_config.add_stdout_handler()
     merge_dbsnp_eva_release_files(private_config_xml_file, profile, bgzip_path, bcftools_path, vcf_sort_script_path,
                                   taxonomy_id, assembly_accession, release_species_inventory_table, release_version,
                                   species_release_folder)

eva-accession-release-automation/run_release_in_embassy/release_common_utils.py

@@ -46,9 +46,8 @@ def open_mongo_port_to_tempmongo(private_config_xml_file, profile, taxonomy_id,
 
 def close_mongo_port_to_tempmongo(port_forwarding_process_id):
     os.kill(port_forwarding_process_id, signal.SIGTERM)
-    os.system('echo -e "Killed port forwarding from remote port with signal 1 - SIGTERM. '
-              '\\033[31;1;4mIGNORE OS MESSAGE '  # escape sequences for bold red and underlined text
-              '\'Killed by Signal 1\' in the preceding/following text\\033[0m".')
+    logger.info('Killed port forwarding from remote port with signal 1 - SIGTERM. '
+                'IGNORE OS MESSAGE \'Killed by Signal 1\' in the preceding/following text.')
 
 
 def get_bgzip_bcftools_index_commands_for_file(bgzip_path, bcftools_path, file):

eva-accession-release-automation/run_release_in_embassy/run_release_for_assembly.py

@@ -66,6 +66,7 @@ def run_release_for_assembly(private_config_xml_file, profile, taxonomy_id, asse
 @click.command()
 def main(private_config_xml_file, profile, taxonomy_id, assembly_accession, release_species_inventory_table,
          release_version, species_release_folder, release_jar_path, memory):
+    logging_config.add_stdout_handler()
     run_release_for_assembly(private_config_xml_file, profile, taxonomy_id, assembly_accession,
                              release_species_inventory_table, release_version, species_release_folder, release_jar_path,
                              memory)

eva-accession-release-automation/run_release_in_embassy/sort_bgzip_index_release_files.py

@@ -16,6 +16,8 @@
 import click
 
 from ebi_eva_common_pyutils.command_utils import run_command_with_output
+from ebi_eva_common_pyutils.logger import logging_config
+
 from run_release_in_embassy.release_metadata import release_vcf_file_categories, release_text_file_categories
 from run_release_in_embassy.release_common_utils import get_release_vcf_file_name_genbank, \
     get_unsorted_release_vcf_file_name, get_release_text_file_name, get_unsorted_release_text_file_name, \
@@ -66,6 +68,7 @@ def sort_bgzip_index_release_files(bgzip_path, bcftools_path, vcf_sort_script_pa
 @click.option("--species-release-folder", required=True)
 @click.command()
 def main(bgzip_path, bcftools_path, vcf_sort_script_path, taxonomy_id, assembly_accession, species_release_folder):
+    logging_config.add_stdout_handler()
     sort_bgzip_index_release_files(bgzip_path, bcftools_path, vcf_sort_script_path, taxonomy_id, assembly_accession,
                                    species_release_folder)
 

eva-accession-release-automation/run_release_in_embassy/update_release_status_for_assembly.py

@@ -15,6 +15,7 @@
 import click
 import logging
 
+from ebi_eva_common_pyutils.logger import logging_config
 
 from run_release_in_embassy.release_metadata import update_release_progress_status
 from ebi_eva_internal_pyutils.metadata_utils import get_metadata_connection_handle
@@ -41,6 +42,7 @@ def update_release_status_for_assembly(private_config_xml_file, profile, release
 @click.option("--release-version", help="ex: 2", type=int, required=True)
 @click.command()
 def main(private_config_xml_file, profile, release_species_inventory_table, taxonomy_id, assembly_accession, release_version):
+    logging_config.add_stdout_handler()
     update_release_status_for_assembly(private_config_xml_file, profile, release_species_inventory_table, taxonomy_id,
                                        assembly_accession, release_version)
 

eva-accession-release-automation/run_release_in_embassy/update_sequence_names_to_ena.py

@@ -14,6 +14,7 @@
 
 
 import click
+from ebi_eva_common_pyutils.logger import logging_config
 
 from run_release_in_embassy.release_common_utils import get_release_vcf_file_name, get_release_vcf_file_name_genbank
 from run_release_in_embassy.release_metadata import release_vcf_file_categories
@@ -44,6 +45,7 @@ def update_sequence_name(taxonomy_id, assembly_accession, species_release_folder
 @click.option("--bcftools-path", help="ex: /path/to/bcftools/binary", required=True)
 @click.command()
 def main(taxonomy_id, assembly_accession, species_release_folder, sequence_name_converter_path, bcftools_path):
+    logging_config.add_stdout_handler()
     update_sequence_name(taxonomy_id, assembly_accession, species_release_folder, sequence_name_converter_path, bcftools_path)
 
 

eva-accession-release-automation/run_release_in_embassy/validate_release_vcf_files.py

@@ -15,6 +15,8 @@
 import click
 import os
 
+from ebi_eva_common_pyutils.logger import logging_config
+
 from run_release_in_embassy.release_common_utils import get_release_vcf_file_name_genbank
 from run_release_in_embassy.release_metadata import get_release_inventory_info_for_assembly, \
     release_vcf_file_categories, vcf_validation_output_file_pattern, asm_report_output_file_pattern
@@ -79,6 +81,7 @@ def validate_release_vcf_files(private_config_xml_file, profile, taxonomy_id, as
 @click.command()
 def main(private_config_xml_file, profile, taxonomy_id, assembly_accession, release_species_inventory_table, release_version,
          species_release_folder, vcf_validator_path, assembly_checker_path):
+    logging_config.add_stdout_handler()
     validate_release_vcf_files(private_config_xml_file, profile, taxonomy_id, assembly_accession,
                                release_species_inventory_table, release_version, species_release_folder,
                                vcf_validator_path, assembly_checker_path)

eva-accession-release-automation/run_release_in_embassy/validate_rs_release_files.py

@@ -26,6 +26,8 @@
 
 from ebi_eva_common_pyutils.command_utils import run_command_with_output
 from ebi_eva_common_pyutils.file_utils import file_diff, FileDiffOption
+from ebi_eva_common_pyutils.logger import logging_config
+
 from run_release_in_embassy.release_common_utils import open_mongo_port_to_tempmongo, close_mongo_port_to_tempmongo, \
     get_release_db_name_in_tempmongo_instance
 from run_release_in_embassy.copy_accessioning_collections_to_embassy import collections_assembly_attribute_map, \
@@ -498,6 +500,7 @@ def validate_rs_release_files(private_config_xml_file, profile, taxonomy_id, ass
 @click.command()
 def main(private_config_xml_file, profile, taxonomy_id, assembly_accession, release_species_inventory_table,
          release_version, species_release_folder):
+    logging_config.add_stdout_handler()
     validate_rs_release_files(private_config_xml_file, profile, taxonomy_id, assembly_accession,
                               release_species_inventory_table, release_version, species_release_folder)