From 8c6b882154a1b8fcb64c7068b16e5ddd0d743ca6 Mon Sep 17 00:00:00 2001 From: Valeriu Predoi Date: Wed, 2 Oct 2024 16:16:58 +0100 Subject: [PATCH] Remove obsolete utility `esmvt_rose_wrapper` and its documentation and very obsolete `mip_convert` cmorizer (#3759) --- doc/sphinx/source/utils.rst | 61 -- .../mip_convert/config-mipconv-user.yml | 22 - .../mip_convert/esmvt_mipconv_setup.py | 527 ------------------ .../mip_convert/recipe_mip_convert.yml | 51 -- .../mip_convert/rose-suite-template.conf | 20 - .../utils/rose-cylc/esmvt_rose_wrapper.py | 258 --------- setup.py | 2 - 7 files changed, 941 deletions(-) delete mode 100644 esmvaltool/cmorizers/mip_convert/config-mipconv-user.yml delete mode 100644 esmvaltool/cmorizers/mip_convert/esmvt_mipconv_setup.py delete mode 100644 esmvaltool/cmorizers/mip_convert/recipe_mip_convert.yml delete mode 100644 esmvaltool/cmorizers/mip_convert/rose-suite-template.conf delete mode 100644 esmvaltool/utils/rose-cylc/esmvt_rose_wrapper.py diff --git a/doc/sphinx/source/utils.rst b/doc/sphinx/source/utils.rst index 03e2793dca..71de0e01f6 100644 --- a/doc/sphinx/source/utils.rst +++ b/doc/sphinx/source/utils.rst @@ -152,67 +152,6 @@ Next, get started with `cylc `. -Using Rose and cylc -------------------- -It is possible to run more than one recipe in one go: currently this relies on the user -having access to a HPC that has ``rose`` and ``cylc`` installed since the procedure involves -installing and submitting a Rose suite. The utility that allows you to do this is -``esmvaltool/utils/rose-cylc/esmvt_rose_wrapper.py``. - -Base suite -.......... -The base suite to run esmvaltool via rose-cylc is `u-bd684`; you can find -this suite in the Met Office Rose repository at: - -https://code.metoffice.gov.uk/svn/roses-u/b/d/6/8/4/trunk/ - -When ``rose`` will be working with python3.x, this location will become -default and the pipeline will aceess it independently of user, unless, of -course the user will specify ``-s $SUITE_LOCATION``; until then the user needs -to grab a copy of it in ``$HOME`` or specify the default location via ``-s`` option. - -Environment -........... -We will move to a unified and centrally-installed esmvaltool environment; -until then, the user will have to alter the env_setup script: - -``u-bd684/app/esmvaltool/env_setup`` - -with the correct pointers to esmvaltool installation, if desired. - -To be able to submit to cylc, you need to have the `/metomi/` suite in path -AND use a `python2.7` environment. Use the Jasmin-example below for guidance. - -Jasmin-example -.............. -This shows how to interact with rose-cylc and run esmvaltool under cylc -using this script: - -.. code:: bash - - export PATH=/apps/contrib/metomi/bin:$PATH - export PATH=/home/users/valeriu/miniconda2/bin:$PATH - mkdir esmvaltool_rose - cd esmvaltool_rose - cp ESMValTool/esmvaltool/utils/rose-cylc/esmvt_rose_wrapper.py . - svn checkout https://code.metoffice.gov.uk/svn/roses-u/b/d/6/8/4/trunk/ ~/u-bd684 - [enter Met Office password] - [configure ~/u-bd684/rose_suite.conf] - [configure ~/u-bd684/app/esmvaltool/env_setup] - python esmvt_rose_wrapper.py -c config-user.yml \ - -r recipe_autoassess_stratosphere.yml recipe_OceanPhysics.yml \ - -d $HOME/esmvaltool_rose - rose suite-run u-bd684 - -Note that you need to pass FULL PATHS to cylc, no `.` or `..` because all -operations are done remotely on different nodes. - -A practical actual example of running the tool can be found on JASMIN: -``/home/users/valeriu/esmvaltool_rose``. -There you will find the run shell: ``run_example``, as well as an example -how to set the configuration file. If you don't have Met Office credentials, -a copy of `u-bd684` is always located in ``/home/users/valeriu/roses/u-bd684`` on Jasmin. - .. _utils_batch_jobs: Using the scripts in `utils/batch-jobs` diff --git a/esmvaltool/cmorizers/mip_convert/config-mipconv-user.yml b/esmvaltool/cmorizers/mip_convert/config-mipconv-user.yml deleted file mode 100644 index 93362f92d7..0000000000 --- a/esmvaltool/cmorizers/mip_convert/config-mipconv-user.yml +++ /dev/null @@ -1,22 +0,0 @@ -############################################################################### -# User's configuration file for the ESMValTool with mip_convert -# For further details see the README document; current sections are -# mandatory and should be populated with valid entries. -# Author: V. Predoi / UREAD / November 2018 -############################################################################### ---- -# root to directory where mip_convert rose suites will be run -# make this different than your usual /roses/ dir -ROSES_ROOT: "/home/users/$USER/roses_mipconv" -# root to directory where mip_convert rose suites will write output -ROSES_OUTPUT: "/home/users/$USER/roses_mipconv_output" -# map dataset name to relevant UM suite -DATASET_TO_SUITE: {"UKESM1-0-LL": "u-ar766a"} -# map variable standard name to stream definition -STREAM_MAP: {"ps": "ap4", "ta": "ap4", "va": "ap4", "ua": "ap5", "mrsos": "ap5", "toz":"apm"} -# root directory where PP data lives -# this directory is in Jasmin/Archer structure; this one here -# is an actual directory with data -INPUT_DIR: "/group_workspaces/jasmin4/ncas_cms/valeriu/MASS_DATA" -# map streams to realm components -STREAM_COMPONENTS: {"ap4": ["atmos-physics", "land"], "apm": ["atmos-physics"], "ap5": ["land"]} diff --git a/esmvaltool/cmorizers/mip_convert/esmvt_mipconv_setup.py b/esmvaltool/cmorizers/mip_convert/esmvt_mipconv_setup.py deleted file mode 100644 index 8868827d5d..0000000000 --- a/esmvaltool/cmorizers/mip_convert/esmvt_mipconv_setup.py +++ /dev/null @@ -1,527 +0,0 @@ -""" -Run the first communication between esmvaltool's recipe and mip_convert. - -Description: ------------- - -This script sets up the correct rose suite directories to run mip_convert -on different UM suite data. You can run this tool in three different ways: - - (with -m --mode option) setup-only: will set up the mip convert rose - directories only; it will use the -c configuration file for user options; - - (with -m --mode option) setup-run-suites: will set up the mip convert rose - suites and will go ahead and submit them to cylc via rose suite-run; - - (with -m --mode option) postproc: will symlink newly created netCDF data - into a directory per esmvaltool recipe; note that for now, there is no - DRS-like path set up in that directory; - -Usage: ------- --c --config-file: [REQUIRED] user specific configuration file; --r --recipe-file: [REQUIRED] single or multiple (space-sep) recipe files; --m --mode: [OPTIONAL] running mode (setup-only, setup-run-suites, - postproc), default=setup-only --l --log-level: [OPTIONAL] log level, default=info - -Environment ------------ -current JASMIN rose/cyclc need python2.7; esmvaltool needs python3.x -So it is impossible at the moment to run this script as executable from an -esmvaltool environment. Instead, you can run it as a stand-alone tool in a -python 2.7 environment, intwo stages: - -[set up mip_convert suites and run them] -python esmvt_mipconv_setup.py -c config.yml -r recipe.yml -m setup-run-suites -[check succesful completion of mip_convert suites] -[run the symlinking] -python esmvt_mipconv_setup.py -c config.yml -r recipe.yml -m postproc - -A practical example of running the tool can be found on JASMIN: -/home/users/valeriu/esmvaltool_mip_convert -There you will find the two component shells: run_conversion -and run_symlink, as well as an example how to set the configuration file. - -The suite used is now on MOSRS (as of 3 December 2018): u-bd681 -You can use the default location on Jasmin: -DEFAULT_SUITE_LOCATION = "/home/users/valeriu/roses/u-bd681" -alternatively this can be turned off, should you want to check out the suite -off MOSRS and use it locally. - -Contact: --------- -author: Valeriu Predoi (UREAD, valeriu.predoi@ncas.ac.uk) -""" -import argparse -import configparser -import datetime -import logging -import os -import shutil -import subprocess -import socket - -import yaml - -#################### -# global variables # -#################### - -# the tool uses a specially tailored mip_convert Rose suite -# locations of the suite depends on the host -host_name = socket.gethostname().split('.') -if len(host_name) > 1: - if host_name[1] == 'ceda': - # default location for mip_convert suite on JASMIN: - # previous suite: u-ak283_esmvt; new one u-bd681 - # DEFAULT_SUITE_LOCATION = "/home/users/valeriu/roses/u-ak283_esmvt" - DEFAULT_SUITE_LOCATION = "/home/users/valeriu/roses/u-bd681" - # note that you can fcm checkout it straight from the MOSRS - -# stream mapping; taken from hadsdk.streams -# these are used to set defaults if not overrides -STREAM_MAP = { - 'CMIP5': { - '3hr': 'apk', - '6hrPlev': 'apc', - '6hrlev': 'apg', - 'Amon': 'apm', - 'Lmon': 'apm', - 'LImon': 'apm', - 'Oday': 'opa', - 'Omon': 'opm', - 'Oyr': 'opy', - 'CF3hr': 'apk', - 'CFday': 'apa', - 'CFmon': 'apm', - 'CFsubhr': 'ape', - 'day': 'apa' - }, - 'CMIP6': { - '3hr': 'ap8', - '6hrLev': 'ap7', - '6hrPlev': 'ap7', - '6hrPlevPt': 'ap7', - 'AERday': 'ap6', - 'AERhr': 'ap9', - 'AERmon': 'ap4', - 'AERmonZ': 'ap4', - 'Amon': 'ap5', - 'CF3hr': 'ap8', - 'CFday': 'ap6', - 'CFmon': 'ap5', - 'E1hr': 'ap9', - 'E1hrClimMon': 'ap9', - 'E3hr': 'ap8', - 'E3hrPt': 'ap8', - 'E6hrZ': 'ap7', - 'Eday': 'ap6', - 'EdayZ': 'ap6', - 'Efx': 'ancil', - 'Emon': 'ap5', - 'EmonZ': 'ap5', - 'Esubhr': 'ap8', - 'Eyr': 'ap5', - 'LImon': 'ap5', - 'Lmon': 'ap5', - 'Oday': 'ond', - 'Ofx': 'ancil', - 'Omon': 'onm', - 'SIday': 'ind', - 'SImon': 'inm', - 'day': 'ap6', - 'fx': 'ancil', - 'prim1hrpt': 'ap9', - 'prim3hr': 'ap8', - 'prim3hrpt': 'ap8', - 'prim6hr': 'ap7', - 'prim6hrpt': 'ap7', - 'primDay': 'ap6', - 'primMon': 'ap5', - 'primSIday': 'ap6' - } -} - -# set up logging -logger = logging.getLogger(__name__) - -# print the header -HEADER = r""" -______________________________________________________________________ - - ESMValTool + mip_convert: linking mip_convert to ESMValTool -______________________________________________________________________ - -""" + __doc__ - - -def get_args(): - """Define the `esmvaltool` command line.""" - # parse command line args - parser = argparse.ArgumentParser( - description=HEADER, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument( - '-c', - '--config-file', - default=os.path.join(os.path.dirname(__file__), 'config-user.yml'), - help='Configuration file') - parser.add_argument( - '-r', - '--recipe-files', - type=str, - nargs='+', - help='Recipe files (list or single file)') - parser.add_argument( - '-m', - '--mode', - default='setup-only', - choices=['setup-only', 'setup-run-suites', 'postproc'], - help='How to run: setup: sets up mipconvert suites only;\n' + - 'or setup-run-suites: sets up suites and runs them as well;\n' + - 'or postproc: grab the output from mip_convert and use it.') - parser.add_argument( - '-l', - '--log-level', - default='info', - choices=['debug', 'info', 'warning', 'error']) - args = parser.parse_args() - return args - - -def _set_logger(logging, out_dir, log_file, log_level): - # set logging for screen and file output - root_logger = logging.getLogger() - out_fmt = "%(asctime)s %(levelname)-8s %(name)s,%(lineno)s\t%(message)s" - logging.basicConfig( - filename=os.path.join(out_dir, log_file), - filemode='a', - format=out_fmt, - datefmt='%H:%M:%S', - level=logging.DEBUG) - root_logger.setLevel(log_level.upper()) - logfmt = logging.Formatter(out_fmt) - console_handler = logging.StreamHandler() - console_handler.setFormatter(logfmt) - root_logger.addHandler(console_handler) - - -def read_yaml_file(yaml_file): - """Read recipe into a dictionary.""" - with open(yaml_file, 'r') as yfile: - loaded_file = yaml.safe_load(yfile) - return loaded_file - - -def map_var_to_stream(diagnostics, stream_map): - """Map variable standard name to stream string.""" - stream_list = [] - for _, diag in diagnostics.items(): - for var in diag['variables']: - stream = stream_map[var] - stream_list.append(stream) - stream_list = list(set(stream_list)) - return stream_list - - -def write_rose_conf(rose_config_template, recipe_file, config_file, log_level): - """Write the new rose conf file per suite.""" - # Build the ConfigParser object - config = configparser.ConfigParser() - config.optionxform = str - config.read(rose_config_template) - recipe_object = read_yaml_file(recipe_file) - conf_file = read_yaml_file(config_file) - datasets = recipe_object['datasets'] - - # check if dataset needs analysis - datasets_to_analyze = [] - for dataset in datasets: - if dataset['dataset'] not in conf_file['DATASET_TO_SUITE']: - logger.warning("Dataset %s has no mapping to suite", - dataset['dataset']) - logger.warning("Assuming data retrival from elsewhere.") - else: - datasets_to_analyze.append(dataset) - diagnostics = recipe_object['diagnostics'] - active_streams = map_var_to_stream(diagnostics, conf_file['STREAM_MAP']) - - # set stream overrides to None and set components - # also set CYCLING_FREQUENCIES to P1Y overall - stream_overrides = {} - stream_components = {} - cycling_frequencies = {} - for stream in active_streams: - stream_overrides[stream] = 'None' - stream_components[stream] = conf_file['STREAM_COMPONENTS'][stream] - cycling_frequencies[stream] = 'P1Y' - - # set the logger to start outputting - if not os.path.exists(conf_file['ROSES_OUTPUT']): - os.makedirs(conf_file['ROSES_OUTPUT']) - _set_logger(logging, conf_file['ROSES_OUTPUT'], 'rose_suites_setup.log', - log_level) - logger.info(HEADER) - - # store the rose suite locations - rose_suite_locations = [] - - # loop through datasets (different suites for different datasets) - for dataset in datasets_to_analyze: - - # set correct paths - rose_suite = os.path.join( - conf_file['ROSES_ROOT'], - conf_file['DATASET_TO_SUITE'][dataset['dataset']]) - rose_suite_locations.append(rose_suite) - rose_output = os.path.join( - conf_file['ROSES_OUTPUT'], - conf_file['DATASET_TO_SUITE'][dataset['dataset']]) - if os.path.exists(rose_suite): - shutil.rmtree(rose_suite) - if os.path.exists(DEFAULT_SUITE_LOCATION): - shutil.copytree(DEFAULT_SUITE_LOCATION, rose_suite) - else: - logger.error("Default Suite Location not found: %s", - DEFAULT_SUITE_LOCATION) - break - if not os.path.exists(rose_output): - os.makedirs(rose_output) - new_mipconv_config = os.path.join(rose_suite, 'mip_convert_config') - - # start logging - logger.info("Working on dataset: %s", dataset) - logger.info("Mapping dataset to suite: %s", rose_suite) - logger.info("Output and logs written to: %s", rose_output) - logger.info("Creating rose suite directories...") - logger.info("Use rose-suite.conf template %s", rose_config_template) - logger.info("Use user config file %s", config_file) - - # write the file - config.set('jinja2:suite.rc', 'INPUT_DIR', - '"' + conf_file['INPUT_DIR'] + '"') - config.set('jinja2:suite.rc', 'OUTPUT_DIR', '"' + rose_output + '"') - config.set('jinja2:suite.rc', 'CDDS_DIR', - '"' + DEFAULT_SUITE_LOCATION + '"') - config.set('jinja2:suite.rc', 'MIP_CONVERT_CONFIG_DIR', - '"' + new_mipconv_config + '"') - config.set('jinja2:suite.rc', 'ACTIVE_STREAMS', str(active_streams)) - config.set('jinja2:suite.rc', 'STREAM_TIME_OVERRIDES', - str(stream_overrides)) - config.set('jinja2:suite.rc', 'FIRST_YEAR', str(dataset['start_year'])) - config.set('jinja2:suite.rc', 'REF_YEAR', str(dataset['start_year'])) - config.set('jinja2:suite.rc', 'FINAL_YEAR', str(dataset['end_year'])) - config.set('jinja2:suite.rc', 'STREAM_COMPONENTS', - str(stream_components)) - config.set('jinja2:suite.rc', 'CYCLING_FREQUENCIES', - str(cycling_frequencies)) - config.set( - 'jinja2:suite.rc', 'TARGET_SUITE_NAME', - '"' + conf_file['DATASET_TO_SUITE'][dataset['dataset']] + '"') - with open(os.path.join(rose_suite, 'rose-suite.conf'), 'w') as r_c: - logger.info("Writing rose-suite.conf file %s", - os.path.join(rose_suite, 'rose-suite.conf')) - config.write(r_c) - - # now that we have to conf file set up we need to - # edit the mip_convert configuration file with the correct data - for key, values in conf_file['STREAM_COMPONENTS'].items(): - for comp in values: - mipconv_config = os.path.join(new_mipconv_config, - 'mip_convert.cfg.' + comp) - _edit_mip_convert_config(mipconv_config, conf_file, dataset, - key) - - return rose_suite_locations - - -def _edit_mip_convert_config(mipconv_config, conf_file, dataset, stream): - """Edit the mip_convert file for correct runs.""" - # set the correct variables - base_date = str(dataset['start_year']) + '-01-01-00-00-00' - suite_id = conf_file['DATASET_TO_SUITE'][dataset['dataset']] - cdds_dir = os.path.join(DEFAULT_SUITE_LOCATION, 'mip_convert_aux') - - # Build the ConfigParser object - config = configparser.ConfigParser() - config.optionxform = str - config.read(mipconv_config) - - # set the correct fields - config.set('COMMON', 'cdds_dir', cdds_dir) - config.set('request', 'base_date', base_date) - config.set('request', 'suite_id', suite_id) - stream_section = '_'.join(['stream', stream]) - # add the section if not there already - if not config.has_section(stream_section): - config.add_section(stream_section) - if 'mip' not in dataset: - # can work without any mip in dataset - # will not take it from diagnostic (will assemble - # all possible mappings instead) - logger.warning("No mip in the recipe dataset section.") - logger.warning("Assigning mapping from default dictionary.") - stream_map_default = STREAM_MAP[dataset['project']] - variables = [] - cmip_types = [] - for key, val in conf_file['STREAM_MAP'].items(): - for key_def, val_def in stream_map_default.items(): - if val == val_def: - cmip_types.append('_'.join([dataset['project'], key_def])) - variables.append(key) - str_variables = ' '.join(list(set([v for v in variables]))) - if variables: - for cmip_type in cmip_types: - config.set(stream_section, cmip_type, str_variables) - else: - cmip_type = '_'.join([dataset['project'], dataset['mip']]) - all_vars = conf_file['STREAM_MAP'].keys() - str_variables = ' '.join( - [v for v in all_vars if conf_file['STREAM_MAP'][v] == stream]) - config.set(stream_section, cmip_type, str_variables) - - # write to file - with open(mipconv_config, 'w') as r_c: - logger.info("Writing mip_convert config file %s", mipconv_config) - config.write(r_c) - - -def _put_in_env(env_script): - """Put new system vars in environment.""" - logger.info("Setting environment for suite submission...") - - # First make it executable. - chmod_command = ["chmod", "+x", env_script] - proc = subprocess.Popen(chmod_command, stdout=subprocess.PIPE) - proc.communicate() - logger.info("Script %s is now executable.", env_script) - - # set the environment - for line in open(env_script, 'r'): - if line.split("=")[0] == 'export PATH': - logger.info("Appending %s to path...", - line.split("=")[1].strip("\n")) - add_path = line.split("=")[1].strip("\n").strip(":$PATH") - os.environ["PATH"] += os.pathsep + add_path - elif line.split("=")[0] == 'export PYTHONPATH': - logger.info("Exporting %s as PYTHONPATH...", - line.split("=")[1].strip("\n")) - os.environ["PYTHONPATH"] = line.split("=")[1].strip("\n") - - # print and check - logger.info("New path: %s", str(os.environ["PATH"])) - logger.info("mip_convert PYTHONPATH: %s", str(os.environ["PYTHONPATH"])) - proc = subprocess.Popen(["which", "rose"], stdout=subprocess.PIPE) - out, err = proc.communicate() - logger.info("rose: %s %s", out, err) - proc = subprocess.Popen(["which", "mip_convert"], stdout=subprocess.PIPE) - out, err = proc.communicate() - logger.info("mip_convert: %s %s", out, err) - - -def _source_envs(suite): - """Source relevant environments.""" - # source the Met Office rose/cylc environment - # and the suite specific environment - suite_env = os.path.join(suite, 'env_setup_command_line.sh') # suite env - env_file_mo = os.path.join(suite, 'sourcepaths.sh') # metomi env - _put_in_env(suite_env) - _put_in_env(env_file_mo) - - -def _run_suite(suite): - """Run the mip_convert suite.""" - os.chdir(suite) - logger.info("Submitting suite from %s", suite) - proc = subprocess.Popen(["rose", "suite-run"], stdout=subprocess.PIPE) - out, err = proc.communicate() - logger.info("Rose communications: %s %s", str(out), str(err)) - - -def symlink_data(recipe_file, config_file, log_level): - """Grab the mip_converted output and manage it for ESMValTool.""" - # get configuration and recipe - recipe_object = read_yaml_file(recipe_file) - conf_file = read_yaml_file(config_file) - datasets = recipe_object['datasets'] - - # create directory that stores all the output netCDF files - now = datetime.datetime.utcnow().strftime("%Y%m%d_%H%M%S") - new_subdir = '_'.join((recipe_file.strip('.yml'), now)) - sym_output_dir = os.path.join(conf_file['ROSES_OUTPUT'], - 'mip_convert_symlinks', new_subdir) - if not os.path.exists(sym_output_dir): - os.makedirs(sym_output_dir) - - # set the logger to start outputting - _set_logger(logging, conf_file['ROSES_OUTPUT'], 'file_simlink.log', - log_level) - logger.info(HEADER) - - # loop through all datasets to symlink output - for dataset in datasets: - rose_output = os.path.join( - conf_file['ROSES_OUTPUT'], - conf_file['DATASET_TO_SUITE'][dataset['dataset']]) - logger.info("Working on dataset: %s", dataset) - logger.info("Output and logs written to: %s", rose_output) - - # create the dataset dir - dataset_output = os.path.join(sym_output_dir, dataset['dataset']) - if os.path.exists(dataset_output): - shutil.rmtree(dataset_output) - os.makedirs(dataset_output) - - # loop through files - for root, _, files in os.walk(rose_output): - for xfile in files: - real_file = os.path.join(root, xfile) - imag_file = os.path.join(dataset_output, xfile) - - # symlink it if nc file - if real_file.endswith('.nc') and \ - xfile.split('_')[2] == dataset['dataset']: - if not os.path.islink(imag_file): - logger.info("File to symlink: %s", real_file) - logger.info("Symlinked file: %s", imag_file) - os.symlink(real_file, imag_file) - else: - logger.info("Symlinked file exists...") - logger.info("Original file: %s", real_file) - logger.info("Symlinked file: %s", imag_file) - - -def main(): - """Run the the meat of the code.""" - logger.info("Running main function...") - args = get_args() - rose_config_template = os.path.join( - os.path.dirname(__file__), "rose-suite-template.conf") - - # make sure the file is retrieved nonetheless - if not os.path.isfile(rose_config_template): - logger.info("Fetching rose template config from suite %s", - DEFAULT_SUITE_LOCATION) - rose_config_template = os.path.join(DEFAULT_SUITE_LOCATION, - "rose-suite-template.conf") - - recipe_files = args.recipe_files - config_file = args.config_file - log_level = args.log_level - for recipe_file in recipe_files: - if args.mode == 'setup-only': - # set up the rose suites - write_rose_conf(rose_config_template, recipe_file, config_file, - log_level) - elif args.mode == 'setup-run-suites': - # setup roses - roses = write_rose_conf(rose_config_template, recipe_file, - config_file, log_level) - # set up the environment and submit - for rose in roses: - _source_envs(rose) - _run_suite(rose) - elif args.mode == 'postproc': - symlink_data(recipe_file, config_file, log_level) - - -if __name__ == '__main__': - main() diff --git a/esmvaltool/cmorizers/mip_convert/recipe_mip_convert.yml b/esmvaltool/cmorizers/mip_convert/recipe_mip_convert.yml deleted file mode 100644 index 8d5168a975..0000000000 --- a/esmvaltool/cmorizers/mip_convert/recipe_mip_convert.yml +++ /dev/null @@ -1,51 +0,0 @@ -#### summary -# Example of ESMValTool recipe that can be used with the mip_convert capability -# Data for this recipe exists in pp format on JASMIN, ready for mip_convert-ion -# The recipe is no different than any typical ESMValTool recipes, but can be used -# for a test run of mip_convert capability; see the README document and the included -# config-mipconv-user.yml configuration file. -# Author: V. Predoi (Uni Reading, valeriu.predoi@ncas.ac.uk) -# Date: first draft/November 2018 -########################################################################################################### ---- - -datasets: - - {dataset: UKESM1-0-LL, project: CMIP6, mip: Amon, exp: piControl-spinup, ensemble: r1i1p1f1_gn, start_year: 1850, end_year: 1860} - -preprocessors: - pp_rad: - regrid: - target_grid: 1x1 - scheme: linear - -diagnostics: - validation_mip_convert: - description: "Test with mip convert" - variables: - # mapping of standard_name to stream for CMIP6 - # see the associated config file for input - # "ps": "ap4", "ta": "ap4", "va": "ap4", "ua": "ap5", "mrsos": "ap5", "toz":"apm" - ps: - preprocessor: pp_rad - field: T2Ms - ta: - preprocessor: pp_rad - field: T2Ms - va: - preprocessor: pp_rad - field: T2Ms - ua: - preprocessor: pp_rad - field: T2Ms - toz: - preprocessor: pp_rad - field: T2Ms - scripts: - meridional_mean: - script: validation.py - title: "" - control_model: UKESM1-0-LL - exper_model: UKESM1-0-LL - analysis_type: meridional_mean - seasonal_analysis: True - diff --git a/esmvaltool/cmorizers/mip_convert/rose-suite-template.conf b/esmvaltool/cmorizers/mip_convert/rose-suite-template.conf deleted file mode 100644 index 5562333fed..0000000000 --- a/esmvaltool/cmorizers/mip_convert/rose-suite-template.conf +++ /dev/null @@ -1,20 +0,0 @@ -[jinja2:suite.rc] -ACTIVE_STREAMS = -CONCATENATE = "FALSE" -CYCLING_FREQUENCIES = -DUMMY_RUN = "FALSE" -FINAL_YEAR = -FIRST_YEAR = -REF_YEAR = -INPUT_DIR = -LOCATION = "LOTUS" -MEMORY = "70000" -MIP_CONVERT_CONFIG_DIR = -OUTPUT_DIR = -PARALLEL_TASKS = "20" -NTHREADS_CONCATENATE = "6" -CDDS_DIR = -STREAM_COMPONENTS = -STREAM_TIME_OVERRIDES = -TARGET_SUITE_NAME = -WALL_TIME = "6:00:00" diff --git a/esmvaltool/utils/rose-cylc/esmvt_rose_wrapper.py b/esmvaltool/utils/rose-cylc/esmvt_rose_wrapper.py deleted file mode 100644 index 5965877717..0000000000 --- a/esmvaltool/utils/rose-cylc/esmvt_rose_wrapper.py +++ /dev/null @@ -1,258 +0,0 @@ -r""" -Install and run u-bd684 - the esmvaltool rose-cylc suite. - -Usage: ------- --c --config-file: [REQUIRED] user specific configuration file; --r --recipe-file: [REQUIRED] single or multiple (space-sep) recipe files; --d --main-dir: [OPTIONAL] main run dir name (full path); - defaults to $HOME/ESMVALTOOL_ROSE; --s --suite-dir [OPTIONAL] u-bd684 dir full path; can be set by user; - defaults to $HOME/u-bd684; --n --no-submit [OPTIONAL] if specified, will not submit suite to cylc; --l --log-level: [OPTIONAL] log level, default=info - -Example: --------- -python esmvt_rose_wrapper.py -c /home/users/valeriu/input/config-user.yml \ - -r /home/users/valeriu/recipes/recipe1.yml \ - /home/users/valeriu/recipes/recipe2.yml \ - -d /home/users/valeriu/esmvat_WRAPPER \ - -s /home/users/valeriu/u-bd684/ \ - -n - -Base suite: ------------ -The base suite to run esmvaltool via rose-cylc is u-bd684; you can find -this suite in the Met Office Rose repository at: - -https://code.metoffice.gov.uk/svn/roses-u/b/d/6/8/4/trunk/ - -When rose (exec.) will be working with python3.x, this location will become -default and the pipeline will aceess it independently of user, unless, of -course the user will specify -s $SUITE_LOCATION; until then the user needs -to grab a copy of it in $HOME or specify the default location via -s option. - -Environment: ------------- -We will move to a unified and centrally-installed esmvaltool environment; -until then, the user will have to alter the env_setup script: - -u-bd684/app/esmvaltool/env_setup - -with the correct pointers to esmvaltool installation, if desired; -NOTE that the defaults are working pointers for an install on CEDA-Jasmin. - -To be able to submit to cylc, you need to have the /metomi/ suite in path -AND use a python2.7 environment. Use the Jasmin-example below for guidance. - -Jasmin-example: ---------------- -This shows how to interact with rose-cylc and run esmvaltool under cylc -using this script: - -export PATH=/apps/contrib/metomi/bin:$PATH -export PATH=/home/users/valeriu/miniconda2/bin:$PATH -mkdir esmvaltool_rose -cd esmvaltool_rose -cp $esmvaltool/utils/rose-cylc/esmvt_rose_wrapper.py . -[get u-bd684 in $HOME, get your recipes and the config] -python esmvt_rose_wrapper.py -c config-user.yml \ --r recipe_autoassess_stratosphere.yml recipe_OceanPhysics.yml \ --d $HOME/esmvaltool_rose - -Note that you need to pass FULL PATHS to cylc, no . or .. because all -operations are done remotely on different nodes. - -A practical actual example of running the tool can be found on JASMIN: -/home/users/valeriu/esmvaltool_rose -There you will find the run shell: run_example, as well as an example -how to set the configuration file. A copy of u-bd684 is always located -in /home/users/valeriu/roses/u-bd684. - -Contact: --------- -author: Valeriu Predoi (UREAD, valeriu.predoi@ncas.ac.uk) -""" -import argparse -import configparser -import logging -import os -import subprocess -import shutil - -import yaml - - -# set up logging -logger = logging.getLogger(__name__) - -# print the header -HEADER = r""" -______________________________________________________________________ - - ESMValTool Rose-Cylc Wrapper -______________________________________________________________________ - -""" + __doc__ - - -def get_args(): - """Define the `esmvaltool` command line.""" - # parse command line args - parser = argparse.ArgumentParser( - description=HEADER, - formatter_class=argparse.RawDescriptionHelpFormatter) - parser.add_argument( - '-c', - '--config-file', - default=os.path.join(os.path.dirname(__file__), 'config-user.yml'), - help='Configuration file') - parser.add_argument( - '-r', - '--recipe-files', - type=str, - nargs='+', - help='Recipe files (list or single file)') - parser.add_argument( - '-d', - '--main-dir', - default=os.path.join(os.environ['HOME'], 'ESMVALTOOL_ROSE'), - help='Main analysis directory; default to $HOME/ESMVALTOOL_ROSE') - parser.add_argument( - '-s', - '--suite-dir', - default=os.path.join(os.environ['HOME'], 'u-bd684'), - help='u-bd684 suite directory; default to $HOME/u-bd684') - parser.add_argument( - '-n', - '--no-submit', - action='store_true', - help="Flag to NOT submit the Rose suite.") - parser.add_argument( - '-l', - '--log-level', - default='info', - choices=['debug', 'info', 'warning', 'error']) - args = parser.parse_args() - return args - - -def _set_logger(logging, out_dir, log_file, log_level): - # set logging for screen and file output - root_logger = logging.getLogger() - out_fmt = "%(asctime)s %(levelname)-8s %(name)s,%(lineno)s\t%(message)s" - logging.basicConfig( - filename=os.path.join(out_dir, log_file), - filemode='a', - format=out_fmt, - datefmt='%H:%M:%S', - level=logging.DEBUG) - root_logger.setLevel(log_level.upper()) - logfmt = logging.Formatter(out_fmt) - console_handler = logging.StreamHandler() - console_handler.setFormatter(logfmt) - root_logger.addHandler(console_handler) - - -def read_yaml_file(yaml_file): - """Read recipe into a dictionary.""" - with open(yaml_file, 'r') as yfile: - loaded_file = yaml.safe_load(yfile) - return loaded_file - - -def _setup_work(rose_config_template, recipe_files, - config_file, main_dir, default_suite, log_level): - """Write the new rose conf file per suite.""" - # Build the ConfigParser object - config = configparser.ConfigParser() - config.optionxform = str - config.read(rose_config_template) - - # set the main work dir - if not os.path.exists(main_dir): - os.makedirs(main_dir) - - # assemble work tree - if not os.path.isfile(os.path.join(main_dir, config_file)): - shutil.copy2(config_file, main_dir) - if not os.path.exists(os.path.join(main_dir, 'recipes')): - os.makedirs(os.path.join(main_dir, 'recipes')) - if not os.path.exists(os.path.join(main_dir, - os.path.basename(config_file))): - shutil.copy2(config_file, main_dir) - recipes_field = [] - for recipe in recipe_files: - if not os.path.exists(os.path.join(main_dir, 'recipes', - os.path.basename(recipe))): - shutil.copy2(recipe, os.path.join(main_dir, 'recipes')) - recipes_field.append(os.path.basename(recipe).strip('.yml')) - rose_suite = os.path.join(main_dir, 'u-bd684') - if os.path.exists(rose_suite): - shutil.rmtree(rose_suite) - shutil.copytree(default_suite, rose_suite) - out_dir = os.path.join(main_dir, 'output') - if not os.path.exists(out_dir): - os.makedirs(out_dir) - - # set logging - _set_logger(logging, out_dir, 'setup.log', log_level) - logger.info(HEADER) - - # start logging - logger.info("Main working directory: %s", main_dir) - logger.info("Using Rose-Cylc suite base: %s", default_suite) - logger.info("Output and logs written to: %s", out_dir) - logger.info("Creating rose suite directories...") - logger.info("Use rose-suite.conf template %s", rose_config_template) - logger.info("Use user config file %s", config_file) - - # write the file - config.set('jinja2:suite.rc', 'INPUT_DIR', - '"' + main_dir + '"') - config.set('jinja2:suite.rc', 'OUTPUT_DIR', '"' + out_dir + '"') - config.set('jinja2:suite.rc', 'RECIPES', str(recipes_field)) - with open(os.path.join(rose_suite, 'rose-suite.conf'), 'w') as r_c: - logger.info("Writing rose-suite.conf file %s", - os.path.join(rose_suite, 'rose-suite.conf')) - config.write(r_c) - - return rose_suite - - -def _run_suite(suite): - """Run the mip_convert suite.""" - os.chdir(suite) - logger.info("Submitting suite from %s", suite) - proc = subprocess.Popen(["rose", "suite-run"], stdout=subprocess.PIPE) - out, err = proc.communicate() - logger.info("Rose communications: %s %s", str(out), str(err)) - - -def main(): - """Run the the meat of the code.""" - logger.info("Running main function...") - args = get_args() - # rose suite default location - if args.suite_dir: - default_suite = args.suite_dir - rose_config_template = os.path.join(default_suite, "rose-suite.conf") - - # get command line arguments - recipe_files = args.recipe_files - config_file = args.config_file - main_dir = args.main_dir - log_level = args.log_level - - # setup rose suite - run_rose = _setup_work(rose_config_template, recipe_files, - config_file, main_dir, default_suite, log_level) - - # submit to cylc - if not args.no_submit: - _run_suite(run_rose) - - -if __name__ == '__main__': - main() diff --git a/setup.py b/setup.py index 33ec620fbf..d2bccff2c9 100755 --- a/setup.py +++ b/setup.py @@ -246,8 +246,6 @@ def read_description(filename): }, entry_points={ 'console_scripts': [ - 'mip_convert_setup = ' - 'esmvaltool.cmorizers.mip_convert.esmvt_mipconv_setup:main', 'nclcodestyle = esmvaltool.utils.nclcodestyle.nclcodestyle:_main', 'test_recipe = ' 'esmvaltool.utils.testing.recipe_settings.install_expand_run:main',