Updating ESACCI-WATERVAPOUR cmorizer (#3282)

Co-authored-by: Elizaveta Malinina <[email protected]> Co-authored-by: Lisa Bock <[email protected]>
ESMValGroup · Mar 8, 2024 · ec42e89 · ec42e89
1 parent c97c12f
commit ec42e89
Show file tree

Hide file tree

Showing 13 changed files with 77 additions and 72 deletions.
diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-WATERVAPOUR.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-WATERVAPOUR.yml
@@ -1,34 +1,35 @@
 ---
 # Common global attributes for Cmorizer output
 
-
-# Input
-# CDR-2
-filename: 'ESACCI-WATERVAPOUR-L3S-TCWV-*-05deg-{year}{month}-fv3.1.nc'
-# CDR-1
-# filename: 'dataset3_1/CDR-1/monthlies/ESACCI-WATERVAPOUR-L3?-TCWV-*-05deg-{year}{month}-fv3.1.nc'
 attributes:
   dataset_id: ESACCI-WATERVAPOUR
-  # CDR-2
-  version: 'CDR2-L3S-05deg_fv3.1'
-  # CDR-1
-  # version: 'CDR1-L3-05deg_fv3.1'
+  version: CDR2-L3-COMBI-05deg-fv3.1
   tier: 3
   modeling_realm: sat
-  project_id: OBS
-  source: "ftp.brockmann-consult.de, access currently restricted"
+  project_id: OBS6
+  source: https://wui.cmsaf.eu/safira/action/viewDoiDetails?acronym=COMBI_V001
   reference: ["esacci-watervapour"]
-  comment: "Preliminary data."
+  comment: "This CMORizer is for the CDR2 version of dataset (global coverage).
+            Download is possible after registration and ordering of the data."
 
 # Variables to cmorize (here use only filename prefix)
 variables:
-  prw:
+# monthly frequency
+  prw_mon:
+    short_name: prw
     mip: Amon
     raw: tcwv
-    # Output automatially added: Amon_<var>_200910-200910.nc
-    # CDR-2
-    file: OBS_ESACCI-WATERVAPOUR-CDR2-L3S-TCWV-05deg
-    # CDR-1
-    # file: OBS_ESACCI-WATERVAPOUR-CDR1-L3-TCWV-05deg
-    start_year: 2003
+    frequency: mon
+    filename: HTWmm{year}*000000313WVCCI01GL.nc
+    start_year: 2002
+    end_year: 2017
+
+# daily frequency 
+  prw_day:
+    short_name: prw
+    mip: Eday
+    raw: tcwv
+    frequency: day
+    filename: HTWdm{year}*000000313WVCCI01GL.nc
+    start_year: 2002
     end_year: 2017
diff --git a/esmvaltool/cmorizers/data/datasets.yml b/esmvaltool/cmorizers/data/datasets.yml
@@ -534,14 +534,12 @@ datasets:
 
   ESACCI-WATERVAPOUR:
     tier: 3
-    source: Marc Schröder, ftp.brockmann-consult.de
-    last_access: 2021-03-29
+    source: https://wui.cmsaf.eu/safira/action/viewDoiDetails?acronym=COMBI_V001
+    last_access: 2024-02-21
     info: |
-      Currently still restricted because preliminary.
-      Download and processing instructions:
-        FTP server: ftp.brockmann-consult.de, access currently restricted
-                    data/tcwv/dataset3_1/CDR-*/...
-        All files need to be in one directory, not in yearly subdirectories.
+      CDR2 requires registration at EUMETSAT CM SAF, the information on how to 
+          download the order will be emailed once the order is ready.
+      All files need to be in one directory, not in yearly subdirectories.
 
   ESDC:
     tier: 2

diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_watervapour.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_watervapour.py
@@ -1,24 +1,27 @@
 """ESMValTool CMORizer for ESACCI-WATERVAPOUR data.
 
 Tier
-   Tier 3: currently still restricted because preliminary.
+   Tier 3: CDR2 requires registration at EUMETSAT CM SAF.
 
 Source
-   Marc Schröder, ftp.brockmann-consult.de
+   https://wui.cmsaf.eu/safira/action/viewDoiDetails?acronym=COMBI_V001
 
 Last access
-   20210329
+   20240221
 
 Download and processing instructions
-   FTP server: ftp.brockmann-consult.de, access currently restricted
-               data/tcwv/dataset3_1/CDR-*/...
+   CDR2 requires registration at EUMETSAT CM SAF, the information on how to
+        download the order will be emailed once the order is ready.
    All files need to be in one directory, not in yearly subdirectories.
 
 Modification history
+   20240221-malinina_elizaveta: Adjust for daily cmorization and updated
+                                filenames, remove CDR1 due to irrelevance.
    20210607-weigel_katja: Fix for monthly time bounds.
    20210408-weigel_katja: written.
 """
 
+import glob
 import logging
 import os
 
@@ -60,30 +63,30 @@ def extract_variable(var_info, raw_info, attrs, year):
 
 def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date):
     """Cmorize data."""
-    # cmor_table = cfg['cmor_table']
     glob_attrs = cfg['attributes']
 
     # run the cmorization
-    for var, vals in cfg['variables'].items():
+    for var_name, vals in cfg['variables'].items():
+        var = vals['short_name']
         var_info = cfg['cmor_table'].get_variable(vals['mip'], var)
         glob_attrs['mip'] = vals['mip']
-        raw_info = {'name': vals['raw'], 'file': vals['file']}
-        inpfile = os.path.join(in_dir, cfg['filename'])
-        logger.info("CMORizing var %s from file type %s", var, inpfile)
-        # years = range(vals['start_year'], vals['end_year'] + 1)
-        months = ["0" + str(mo) for mo in range(1, 10)] + ["10", "11", "12"]
+        raw_info = {'name': vals['raw']}
+        inpfile_pattern = os.path.join(in_dir, vals['filename'])
+        logger.info("CMORizing var %s from file type %s", var, inpfile_pattern)
         for year in range(vals['start_year'], vals['end_year'] + 1):
-            monthly_cubes = []
-            for month in months:
-                raw_info['file'] = inpfile.format(year=year, month=month)
+            data_cubes = []
+            year_inpfile_pattern = inpfile_pattern.format(year=year)
+            inpfiles = sorted(glob.glob(year_inpfile_pattern))
+            for inpfile in inpfiles:
+                raw_info['file'] = inpfile
                 logger.info("CMORizing var %s from file type %s", var,
                             raw_info['file'])
-                monthly_cubes.append(
+                data_cubes.append(
                     extract_variable(var_info, raw_info, glob_attrs, year))
-            yearly_cube = concatenate(monthly_cubes)
+            yearly_cube = concatenate(data_cubes)
             # Fix monthly time bounds
             time = yearly_cube.coord('time')
-            time.bounds = get_time_bounds(time, 'mon')
+            time.bounds = get_time_bounds(time, vals['frequency'])
             save_variable(yearly_cube,
                           var,
                           out_dir,

diff --git a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig1_clim.yml b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig1_clim.yml
@@ -238,8 +238,8 @@ diagnostics:
         end_year: 2005
         reference_dataset: ESACCI-WATERVAPOUR
         additional_datasets:
-          - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat,
-             version: CDR2-L3S-05deg_fv3.1, tier: 3, start_year: 2003,
+          - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat,
+             version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3, start_year: 2003,
              end_year: 2017}
           - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2,
              start_year: 1984, end_year: 2016}
@@ -393,8 +393,8 @@ diagnostics:
         end_year: 2014
         reference_dataset: ESACCI-WATERVAPOUR
         additional_datasets:
-          - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat,
-             version: CDR2-L3S-05deg_fv3.1, tier: 3,
+          - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat,
+             version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3,
              start_year: 2003, end_year: 2017}
           - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2,
              start_year: 1984, end_year: 2016}

diff --git a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig1_clim_amip.yml b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig1_clim_amip.yml
@@ -253,8 +253,8 @@ diagnostics:
         end_year: 2005
         reference_dataset: ESACCI-WATERVAPOUR
         additional_datasets:
-          - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat,
-             version: CDR2-L3S-05deg_fv3.1, tier: 3,
+          - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat,
+             version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3,
              start_year: 2003, end_year: 2017}
           - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2,
              start_year: 1984, end_year: 2016}
@@ -408,8 +408,8 @@ diagnostics:
         end_year: 2014
         reference_dataset: ESACCI-WATERVAPOUR
         additional_datasets:
-          - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat,
-             version: CDR2-L3S-05deg_fv3.1, tier: 3,
+          - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat,
+             version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3,
              start_year: 2003, end_year: 2017}
           - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2,
              start_year: 1984, end_year: 2016}

diff --git a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig2_taylor.yml b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig2_taylor.yml
@@ -136,8 +136,8 @@ diagnostics:
         exp: historical
         reference_dataset: ESACCI-WATERVAPOUR
         additional_datasets:
-          - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat,
-             version: CDR2-L3S-05deg_fv3.1, tier: 3,
+          - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat,
+             version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3,
              start_year: 2003, end_year: 2017}
           - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2,
              start_year: 1984, end_year: 2016}

diff --git a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig2_taylor_amip.yml b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig2_taylor_amip.yml
@@ -135,8 +135,8 @@ diagnostics:
         exp: amip
         reference_dataset: ESACCI-WATERVAPOUR
         additional_datasets:
-          - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat,
-             version: CDR2-L3S-05deg_fv3.1, tier: 3,
+          - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat,
+             version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3,
              start_year: 2003, end_year: 2017}
           - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2,
              start_year: 1984, end_year: 2016}

diff --git a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig6_interannual.yml b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig6_interannual.yml
@@ -226,8 +226,8 @@ diagnostics:
         end_year: 2005
         reference_dataset: ESACCI-WATERVAPOUR
         additional_datasets:
-          - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat,
-             version: CDR2-L3S-05deg_fv3.1, tier: 3,
+          - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat,
+             version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3,
              start_year: 2003, end_year: 2017}
           - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2,
              start_year: 1984, end_year: 2016}
@@ -372,8 +372,8 @@ diagnostics:
         end_year: 2014
         reference_dataset: ESACCI-WATERVAPOUR
         additional_datasets:
-          - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat,
-             version: CDR2-L3S-05deg_fv3.1, tier: 3,
+          - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat,
+             version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3,
              start_year: 2003, end_year: 2017}
           - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2,
              start_year: 1984, end_year: 2016}

diff --git a/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig7_seas.yml b/esmvaltool/recipes/clouds/recipe_lauer22jclim_fig7_seas.yml
@@ -231,8 +231,8 @@ diagnostics:
         end_year: 2005
         reference_dataset: ESACCI-WATERVAPOUR
         additional_datasets:
-          - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat,
-             version: CDR2-L3S-05deg_fv3.1, tier: 3,
+          - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat,
+             version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3,
              start_year: 2003, end_year: 2017}
           - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2,
              start_year: 1984, end_year: 2016}
@@ -384,8 +384,8 @@ diagnostics:
         end_year: 2014
         reference_dataset: ESACCI-WATERVAPOUR
         additional_datasets:
-          - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat,
-             version: CDR2-L3S-05deg_fv3.1, tier: 3,
+          - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat,
+             version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3,
              start_year: 2003, end_year: 2017}
           - {dataset: ISCCP-FH, project: OBS, type: sat, version: v0, tier: 2,
              start_year: 1984, end_year: 2016}

diff --git a/esmvaltool/recipes/examples/recipe_check_obs.yml b/esmvaltool/recipes/examples/recipe_check_obs.yml
@@ -1577,8 +1577,11 @@ diagnostics:
     variables:
       prw:
     additional_datasets:
-      - {dataset: ESACCI-WATERVAPOUR, project: OBS, mip: Amon, tier: 3,
-         type: sat, version: CDR2-L3S-05deg_fv3.1,
+      - {dataset: ESACCI-WATERVAPOUR, project: OBS6, mip: Amon, tier: 3,
+         type: sat, version: CDR2-L3-COMBI-05deg-fv3.1,
+         start_year: 2003, end_year: 2017}
+      - {dataset: ESACCI-WATERVAPOUR, project: OBS6, mip: Eday, tier: 3,
+         type: sat, version: CDR2-L3-COMBI-05deg-fv3.1,
          start_year: 2003, end_year: 2017}
     scripts: null
 
@@ -1942,4 +1945,4 @@ diagnostics:
     additional_datasets:
       - {dataset: UWisc, project: OBS, mip: Amon, tier: 3,
          type: sat, version: v2, start_year: 1988, end_year: 2007}
-    scripts: null
+    scripts: null
diff --git a/esmvaltool/recipes/model_evaluation/recipe_model_evaluation_basics.yml b/esmvaltool/recipes/model_evaluation/recipe_model_evaluation_basics.yml
@@ -212,7 +212,7 @@ diagnostics:
         preprocessor: timeseries_regular
         # timerange MUST NOT start before 2003 since the observations are not available before 2003
         additional_datasets:
-          - {project: OBS, dataset: ESACCI-WATERVAPOUR, type: sat, version: CDR2-L3S-05deg_fv3.1, tier: 3, timerange: '2003/2007', reference_for_monitor_diags: true}
+          - {project: OBS6, dataset: ESACCI-WATERVAPOUR, type: sat, version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3, timerange: '2003/2007', reference_for_monitor_diags: true}
     scripts:
       plot:
         <<: *plot_multi_dataset_default

diff --git a/esmvaltool/recipes/model_evaluation/recipe_model_evaluation_clouds_clim.yml b/esmvaltool/recipes/model_evaluation/recipe_model_evaluation_clouds_clim.yml
@@ -152,7 +152,7 @@ diagnostics:
         mip: Amon
         preprocessor: full_climatology
         additional_datasets:
-          - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat, version: CDR2-L3S-05deg_fv3.1, tier: 3, timerange: '2003/2017', reference_for_monitor_diags: true}
+          - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat, version: CDR2-L3-COMBI-05deg-fv3.1, tier: 3, timerange: '2003/2017', reference_for_monitor_diags: true}
     scripts:
       plot:
         <<: *plot_multi_dataset_default

diff --git a/esmvaltool/recipes/recipe_cmug_h2o.yml b/esmvaltool/recipes/recipe_cmug_h2o.yml
@@ -81,8 +81,8 @@ diagnostics:
         additional_datasets:
           # - {dataset: ESACCI-WV, project: OBS, type: sat,
           #    version: HOAPS-v4.0, start_year: 2001, end_year: 2012, tier: 2}
-          - {dataset: ESACCI-WATERVAPOUR, project: OBS, type: sat,
-             version: CDR2-L3S-05deg_fv3.1, start_year: 2003, end_year: 2014, tier: 3}
+          - {dataset: ESACCI-WATERVAPOUR, project: OBS6, type: sat,
+             version: CDR2-L3-COMBI-05deg-fv3.1, start_year: 2003, end_year: 2014, tier: 3}
     additional_datasets:
       - {dataset: GFDL-CM3, exp: piControl, start_year: 1, end_year: 150}
       - {dataset: GFDL-CM3, exp: abrupt4xCO2, start_year: 1, end_year: 150}