diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-WATERVAPOUR.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-WATERVAPOUR.yml index 400fb7e8da..eda705c000 100644 --- a/esmvaltool/cmorizers/data/cmor_config/ESACCI-WATERVAPOUR.yml +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-WATERVAPOUR.yml @@ -1,34 +1,35 @@ --- # Common global attributes for Cmorizer output - -# Input -# CDR-2 -filename: 'ESACCI-WATERVAPOUR-L3S-TCWV-*-05deg-{year}{month}-fv3.1.nc' -# CDR-1 -# filename: 'dataset3_1/CDR-1/monthlies/ESACCI-WATERVAPOUR-L3?-TCWV-*-05deg-{year}{month}-fv3.1.nc' attributes: dataset_id: ESACCI-WATERVAPOUR - # CDR-2 - version: 'CDR2-L3S-05deg_fv3.1' - # CDR-1 - # version: 'CDR1-L3-05deg_fv3.1' + version: CDR2-L3-COMBI-05deg-fv3.1 tier: 3 modeling_realm: sat - project_id: OBS - source: "ftp.brockmann-consult.de, access currently restricted" + project_id: OBS6 + source: https://wui.cmsaf.eu/safira/action/viewDoiDetails?acronym=COMBI_V001 reference: ["esacci-watervapour"] - comment: "Preliminary data." + comment: "This CMORizer is for the CDR2 version of dataset (global coverage). + Download is possible after registration and ordering of the data." # Variables to cmorize (here use only filename prefix) variables: - prw: +# monthly frequency + prw_mon: + short_name: prw mip: Amon raw: tcwv - # Output automatially added: Amon__200910-200910.nc - # CDR-2 - file: OBS_ESACCI-WATERVAPOUR-CDR2-L3S-TCWV-05deg - # CDR-1 - # file: OBS_ESACCI-WATERVAPOUR-CDR1-L3-TCWV-05deg - start_year: 2003 + frequency: mon + filename: HTWmm{year}*000000313WVCCI01GL.nc + start_year: 2002 + end_year: 2017 + +# daily frequency + prw_day: + short_name: prw + mip: Eday + raw: tcwv + frequency: day + filename: HTWdm{year}*000000313WVCCI01GL.nc + start_year: 2002 end_year: 2017 diff --git a/esmvaltool/cmorizers/data/datasets.yml b/esmvaltool/cmorizers/data/datasets.yml index 53bc40dc75..757fce7d67 100644 --- a/esmvaltool/cmorizers/data/datasets.yml +++ b/esmvaltool/cmorizers/data/datasets.yml @@ -534,14 +534,12 @@ datasets: ESACCI-WATERVAPOUR: tier: 3 - source: Marc Schröder, ftp.brockmann-consult.de - last_access: 2021-03-29 + source: https://wui.cmsaf.eu/safira/action/viewDoiDetails?acronym=COMBI_V001 + last_access: 2024-02-21 info: | - Currently still restricted because preliminary. - Download and processing instructions: - FTP server: ftp.brockmann-consult.de, access currently restricted - data/tcwv/dataset3_1/CDR-*/... - All files need to be in one directory, not in yearly subdirectories. + CDR2 requires registration at EUMETSAT CM SAF, the information on how to + download the order will be emailed once the order is ready. + All files need to be in one directory, not in yearly subdirectories. ESDC: tier: 2 diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_watervapour.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_watervapour.py index 0985e5d6e6..d4901007cc 100644 --- a/esmvaltool/cmorizers/data/formatters/datasets/esacci_watervapour.py +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_watervapour.py @@ -1,24 +1,27 @@ """ESMValTool CMORizer for ESACCI-WATERVAPOUR data. Tier - Tier 3: currently still restricted because preliminary. + Tier 3: CDR2 requires registration at EUMETSAT CM SAF. Source - Marc Schröder, ftp.brockmann-consult.de + https://wui.cmsaf.eu/safira/action/viewDoiDetails?acronym=COMBI_V001 Last access - 20210329 + 20240221 Download and processing instructions - FTP server: ftp.brockmann-consult.de, access currently restricted - data/tcwv/dataset3_1/CDR-*/... + CDR2 requires registration at EUMETSAT CM SAF, the information on how to + download the order will be emailed once the order is ready. All files need to be in one directory, not in yearly subdirectories. Modification history + 20240221-malinina_elizaveta: Adjust for daily cmorization and updated + filenames, remove CDR1 due to irrelevance. 20210607-weigel_katja: Fix for monthly time bounds. 20210408-weigel_katja: written. """ +import glob import logging import os @@ -60,30 +63,30 @@ def extract_variable(var_info, raw_info, attrs, year): def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): """Cmorize data.""" - # cmor_table = cfg['cmor_table'] glob_attrs = cfg['attributes'] # run the cmorization - for var, vals in cfg['variables'].items(): + for var_name, vals in cfg['variables'].items(): + var = vals['short_name'] var_info = cfg['cmor_table'].get_variable(vals['mip'], var) glob_attrs['mip'] = vals['mip'] - raw_info = {'name': vals['raw'], 'file': vals['file']} - inpfile = os.path.join(in_dir, cfg['filename']) - logger.info("CMORizing var %s from file type %s", var, inpfile) - # years = range(vals['start_year'], vals['end_year'] + 1) - months = ["0" + str(mo) for mo in range(1, 10)] + ["10", "11", "12"] + raw_info = {'name': vals['raw']} + inpfile_pattern = os.path.join(in_dir, vals['filename']) + logger.info("CMORizing var %s from file type %s", var, inpfile_pattern) for year in range(vals['start_year'], vals['end_year'] + 1): - monthly_cubes = [] - for month in months: - raw_info['file'] = inpfile.format(year=year, month=month) + data_cubes = [] + year_inpfile_pattern = inpfile_pattern.format(year=year) + inpfiles = sorted(glob.glob(year_inpfile_pattern)) + for inpfile in inpfiles: + raw_info['file'] = inpfile logger.info("CMORizing var %s from file type %s", var, raw_info['file']) - monthly_cubes.append( + data_cubes.append( extract_variable(var_info, raw_info, glob_attrs, year)) - yearly_cube = concatenate(monthly_cubes) + yearly_cube = concatenate(data_cubes) # Fix monthly time bounds time = yearly_cube.coord('time') - time.bounds = get_time_bounds(time, 'mon') + time.bounds = get_time_bounds(time, vals['frequency']) save_variable(yearly_cube, var, out_dir, diff --git a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig1_clim.yml b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig1_clim.yml index f63f527da4..0ecd79e458 100644 --- a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig1_clim.yml +++ b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig1_clim.yml @@ -238,8 +238,8 @@ diagnostics: end_year: 2005 reference_dataset: ESACCI-WATERVAPOUR additional_datasets: - - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat, - version: CDR2-L3S-05deg_fv3.1, tier: 3, start_year: 2003, + - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat, + version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3, start_year: 2003, end_year: 2017} - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2, start_year: 1984, end_year: 2016} @@ -393,8 +393,8 @@ diagnostics: end_year: 2014 reference_dataset: ESACCI-WATERVAPOUR additional_datasets: - - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat, - version: CDR2-L3S-05deg_fv3.1, tier: 3, + - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat, + version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3, start_year: 2003, end_year: 2017} - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2, start_year: 1984, end_year: 2016} diff --git a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig1_clim_amip.yml b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig1_clim_amip.yml index 65818e71e4..8ae73c9da5 100644 --- a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig1_clim_amip.yml +++ b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig1_clim_amip.yml @@ -253,8 +253,8 @@ diagnostics: end_year: 2005 reference_dataset: ESACCI-WATERVAPOUR additional_datasets: - - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat, - version: CDR2-L3S-05deg_fv3.1, tier: 3, + - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat, + version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3, start_year: 2003, end_year: 2017} - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2, start_year: 1984, end_year: 2016} @@ -408,8 +408,8 @@ diagnostics: end_year: 2014 reference_dataset: ESACCI-WATERVAPOUR additional_datasets: - - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat, - version: CDR2-L3S-05deg_fv3.1, tier: 3, + - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat, + version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3, start_year: 2003, end_year: 2017} - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2, start_year: 1984, end_year: 2016} diff --git a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig2_taylor.yml b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig2_taylor.yml index 8bce0895cb..c36e633164 100644 --- a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig2_taylor.yml +++ b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig2_taylor.yml @@ -136,8 +136,8 @@ diagnostics: exp: historical reference_dataset: ESACCI-WATERVAPOUR additional_datasets: - - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat, - version: CDR2-L3S-05deg_fv3.1, tier: 3, + - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat, + version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3, start_year: 2003, end_year: 2017} - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2, start_year: 1984, end_year: 2016} diff --git a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig2_taylor_amip.yml b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig2_taylor_amip.yml index 0c36bc8dc9..426746571b 100644 --- a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig2_taylor_amip.yml +++ b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig2_taylor_amip.yml @@ -135,8 +135,8 @@ diagnostics: exp: amip reference_dataset: ESACCI-WATERVAPOUR additional_datasets: - - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat, - version: CDR2-L3S-05deg_fv3.1, tier: 3, + - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat, + version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3, start_year: 2003, end_year: 2017} - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2, start_year: 1984, end_year: 2016} diff --git a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig6_interannual.yml b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig6_interannual.yml index f5c76aad79..cb8912eee9 100644 --- a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig6_interannual.yml +++ b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig6_interannual.yml @@ -226,8 +226,8 @@ diagnostics: end_year: 2005 reference_dataset: ESACCI-WATERVAPOUR additional_datasets: - - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat, - version: CDR2-L3S-05deg_fv3.1, tier: 3, + - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat, + version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3, start_year: 2003, end_year: 2017} - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2, start_year: 1984, end_year: 2016} @@ -372,8 +372,8 @@ diagnostics: end_year: 2014 reference_dataset: ESACCI-WATERVAPOUR additional_datasets: - - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat, - version: CDR2-L3S-05deg_fv3.1, tier: 3, + - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat, + version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3, start_year: 2003, end_year: 2017} - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2, start_year: 1984, end_year: 2016} diff --git a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig7_seas.yml b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig7_seas.yml index aedba080b7..998f38b4cb 100644 --- a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig7_seas.yml +++ b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig7_seas.yml @@ -231,8 +231,8 @@ diagnostics: end_year: 2005 reference_dataset: ESACCI-WATERVAPOUR additional_datasets: - - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat, - version: CDR2-L3S-05deg_fv3.1, tier: 3, + - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat, + version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3, start_year: 2003, end_year: 2017} - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2, start_year: 1984, end_year: 2016} @@ -384,8 +384,8 @@ diagnostics: end_year: 2014 reference_dataset: ESACCI-WATERVAPOUR additional_datasets: - - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat, - version: CDR2-L3S-05deg_fv3.1, tier: 3, + - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat, + version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3, start_year: 2003, end_year: 2017} - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2, start_year: 1984, end_year: 2016} diff --git a/esmvaltool/recipes/examples/recipe_check_obs.yml b/esmvaltool/recipes/examples/recipe_check_obs.yml index 325d9685b8..70368c87dc 100644 --- a/esmvaltool/recipes/examples/recipe_check_obs.yml +++ b/esmvaltool/recipes/examples/recipe_check_obs.yml @@ -1577,8 +1577,11 @@ diagnostics: variables: prw: additional_datasets: - - {dataset: ESACCI-WATERVAPOUR, project: OBS, mip: Amon, tier: 3, - type: sat, version: CDR2-L3S-05deg_fv3.1, + - {dataset: ESACCI-WATERVAPOUR, project: OBS6, mip: Amon, tier: 3, + type: sat, version: CDR2-L3-COMBI-05deg-fv3.1, + start_year: 2003, end_year: 2017} + - {dataset: ESACCI-WATERVAPOUR, project: OBS6, mip: Eday, tier: 3, + type: sat, version: CDR2-L3-COMBI-05deg-fv3.1, start_year: 2003, end_year: 2017} scripts: null @@ -1942,4 +1945,4 @@ diagnostics: additional_datasets: - {dataset: UWisc, project: OBS, mip: Amon, tier: 3, type: sat, version: v2, start_year: 1988, end_year: 2007} - scripts: null \ No newline at end of file + scripts: null diff --git a/esmvaltool/recipes/model_evaluation/recipe_model_evaluation_basics.yml b/esmvaltool/recipes/model_evaluation/recipe_model_evaluation_basics.yml index 06f8ad3b12..9b7c187140 100644 --- a/esmvaltool/recipes/model_evaluation/recipe_model_evaluation_basics.yml +++ b/esmvaltool/recipes/model_evaluation/recipe_model_evaluation_basics.yml @@ -212,7 +212,7 @@ diagnostics: preprocessor: timeseries_regular # timerange MUST NOT start before 2003 since the observations are not available before 2003 additional_datasets: - - {project: OBS, dataset: ESACCI-WATERVAPOUR, type: sat, version: CDR2-L3S-05deg_fv3.1, tier: 3, timerange: '2003/2007', reference_for_monitor_diags: true} + - {project: OBS6, dataset: ESACCI-WATERVAPOUR, type: sat, version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3, timerange: '2003/2007', reference_for_monitor_diags: true} scripts: plot: <<: *plot_multi_dataset_default diff --git a/esmvaltool/recipes/model_evaluation/recipe_model_evaluation_clouds_clim.yml b/esmvaltool/recipes/model_evaluation/recipe_model_evaluation_clouds_clim.yml index c45a4971ad..aaf964dc24 100644 --- a/esmvaltool/recipes/model_evaluation/recipe_model_evaluation_clouds_clim.yml +++ b/esmvaltool/recipes/model_evaluation/recipe_model_evaluation_clouds_clim.yml @@ -152,7 +152,7 @@ diagnostics: mip: Amon preprocessor: full_climatology additional_datasets: - - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat, version: CDR2-L3S-05deg_fv3.1, tier: 3, timerange: '2003/2017', reference_for_monitor_diags: true} + - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat, version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3, timerange: '2003/2017', reference_for_monitor_diags: true} scripts: plot: <<: *plot_multi_dataset_default diff --git a/esmvaltool/recipes/recipe_cmug_h2o.yml b/esmvaltool/recipes/recipe_cmug_h2o.yml index a44358f664..fbd70e7036 100644 --- a/esmvaltool/recipes/recipe_cmug_h2o.yml +++ b/esmvaltool/recipes/recipe_cmug_h2o.yml @@ -81,8 +81,8 @@ diagnostics: additional_datasets: # - {dataset: ESACCI-WV, project: OBS, type: sat, # version: HOAPS-v4.0, start_year: 2001, end_year: 2012, tier: 2} - - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat, - version: CDR2-L3S-05deg_fv3.1, start_year: 2003, end_year: 2014, tier: 3} + - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat, + version: CDR2-L3-COMBI-05deg-fv3.1, start_year: 2003, end_year: 2014, tier: 3} additional_datasets: - {dataset: GFDL-CM3, exp: piControl, start_year: 1, end_year: 150} - {dataset: GFDL-CM3, exp: abrupt4xCO2, start_year: 1, end_year: 150}