diff --git a/dev/scripts/exglobal_archive_tars.py b/dev/scripts/exglobal_archive_tars.py index e05433c14ac..24de7033e28 100755 --- a/dev/scripts/exglobal_archive_tars.py +++ b/dev/scripts/exglobal_archive_tars.py @@ -3,6 +3,7 @@ import os from pygfs.task.archive import Archive +from pygfs.utils.archive_tar_vars import ArchiveTarVars from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, logit, chdir # initialize root logger @@ -24,35 +25,9 @@ def main(): except KeyError: logger.info(f"key ({key}) not found in archive.task_config!") - # Pull out all the configuration keys needed to run the rest of archive steps - keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY', - 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_PREP_OBS_AERO', - 'PARMgfs', 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'PSLOT', - 'DO_JEDISNOWDA', 'DO_ARCHCOM', 'ARCHCOM_TO', 'ROTDIR', 'ARCH_WARMICFREQ', - 'ARCH_FCSTICFREQ', 'ARCH_CYC', 'assim_freq', 'ARCDIR', 'SDATE', - 'FHMIN_GFS', 'FHMAX_GFS', 'FHOUT_GFS', 'ARCH_GAUSSIAN', 'MODE', - 'FHOUT_OCN', 'FHOUT_ICE', 'FHOUT_OCN_GFS', 'FHOUT_ICE_GFS', 'DO_BUFRSND', 'DOHYBVAR', - 'ARCH_GAUSSIAN_FHMAX', 'ARCH_GAUSSIAN_FHINC', 'ARCH_GAUSSIAN_FHINC', - 'DOIAU', 'OCNRES', 'ICERES', 'NUM_SND_COLLECTIVES', 'FHOUT_WAV', 'FHOUT_WAV_GFS', - 'FHOUT_HF_WAV', 'FHMAX_WAV', 'FHMAX_HF_WAV', 'FHMAX_WAV_GFS', - 'restart_interval_gdas', 'restart_interval_gfs', 'DO_ARCHCOM', - 'DO_AERO_ANL', 'DO_AERO_FCST', 'DO_CA', 'DOBNDPNT_WAVE', 'DO_JEDIOCNVAR', 'DOHYBVAR_OCN', - 'DOLETKF_OCN', 'NMEM_ENS', 'DO_JEDIATMVAR', 'FHMAX_FITS', 'waveGRD', - 'IAUFHRS', 'DO_FIT2OBS', 'NET', 'FHOUT_HF_GFS', 'FHMAX_HF_GFS', - 'OFFSET_START_HOUR', 'ARCH_EXPDIR', 'EXPDIR', 'ARCH_EXPDIR_FREQ', 'ARCH_HASHES', - 'ARCH_DIFFS', 'SDATE', 'EDATE', 'HOMEgfs', 'DO_GEMPAK', 'DATASETS_YAML', - 'WAVE_OUT_GRIDS', 'DO_GSISOILDA', 'DO_LAND_IAU', 'TARBALL_TYPE', 'ATMINC_GRID'] - - archive_dict = AttrDict() - for key in keys: - archive_dict[key] = archive.task_config.get(key) - if archive_dict[key] is None: - logger.warning(f"WARNING: key ({key}) not found in task_config!") - - # Also import all COMIN* and COMOUT* directory and template variables - for key in archive.task_config.keys(): - if key.startswith(("COM_", "COMIN_", "COMOUT_")): - archive_dict[key] = archive.task_config.get(key) + # Collect all variables for YAML templates (config keys, COM paths, cycle vars, member paths) + # Note: COM paths are relative to ROTDIR for portability in tar archives + archive_dict = ArchiveTarVars.get_all_yaml_vars(archive.task_config) with chdir(config.ROTDIR): logger.debug(f"Changed working directory to {config.ROTDIR}") diff --git a/dev/scripts/exglobal_archive_vrfy.py b/dev/scripts/exglobal_archive_vrfy.py index 634c1c1db34..0a614319e70 100755 --- a/dev/scripts/exglobal_archive_vrfy.py +++ b/dev/scripts/exglobal_archive_vrfy.py @@ -3,7 +3,7 @@ import os from pygfs.task.archive import Archive -from pygfs.utils.archive_vars import ArchiveVrfyVars +from pygfs.utils.archive_vrfy_vars import ArchiveVrfyVars from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, logit, chdir # initialize root logger diff --git a/dev/scripts/exglobal_enkf_earc_tars.py b/dev/scripts/exglobal_enkf_earc_tars.py index 354d374a0c2..6af2c9ac8e9 100755 --- a/dev/scripts/exglobal_enkf_earc_tars.py +++ b/dev/scripts/exglobal_enkf_earc_tars.py @@ -3,6 +3,7 @@ import os from pygfs.task.archive import Archive +from pygfs.utils.archive_tar_vars import ArchiveTarVars from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, chdir, logit # initialize root logger @@ -17,30 +18,13 @@ def main(): # Instantiate the Archive object archive = Archive(config) - # Pull out all the configuration keys needed to run the rest of archive steps - keys = ['ATARDIR', 'current_cycle', 'IAUFHRS', 'RUN', 'PDY', - 'PSLOT', 'DO_ARCHCOM', 'ARCHCOM_TO', 'ROTDIR', 'PARMgfs', - 'ARCDIR', 'SDATE', 'MODE', 'ENSGRP', 'NMEM_EARCGRP', - 'NMEM_ENS', 'DO_CALC_INCREMENT_ENKF_GFS', 'DO_JEDIATMENS', - 'lobsdiag_forenkf', 'FHMIN_ENKF', 'FHMAX_ENKF_GFS', - 'FHOUT_ENKF_GFS', 'FHMAX_ENKF', 'FHOUT_ENKF', 'ENKF_SPREAD', - 'restart_interval_enkfgdas', 'restart_interval_enkfgfs', - 'DOHYBVAR', 'DOIAU_ENKF', 'IAU_OFFSET', 'DOIAU', 'DO_CA', - 'DO_CALC_INCREMENT', 'assim_freq', 'ARCH_CYC', 'DO_JEDISNOWDA', - 'ARCH_WARMICFREQ', 'ARCH_FCSTICFREQ', 'DOHYBVAR_OCN', - 'DOLETKF_OCN', 'IAUFHRS_ENKF', 'NET', 'NMEM_ENS_GFS', 'DO_GSISOILDA', 'DO_LAND_IAU'] - - archive_dict = AttrDict() - for key in keys: - archive_dict[key] = archive.task_config.get(key) - if archive_dict[key] is None: - print(f"Warning: key ({key}) not found in task_config!") - - # Also import all COMIN* directory and template variables - for key in archive.task_config.keys(): - if key.startswith(("COM_", "COMIN_")): - archive_dict[key] = archive.task_config.get(key) + # Collect all variables for YAML templates (config keys, COM paths, cycle vars, member paths) + # Note: COM paths are relative to ROTDIR for portability in tar archives + archive_dict = ArchiveTarVars.get_all_yaml_vars(archive.task_config) + # Change to ROTDIR for the entire archiving workflow so that relative paths + # in YAML templates (e.g., logs/..., enkfgdas.20211221/...) are resolved + # correctly during both file existence checks (glob) and tar creation with chdir(config.ROTDIR): # Determine which archives to create diff --git a/dev/scripts/exglobal_enkf_earc_vrfy.py b/dev/scripts/exglobal_enkf_earc_vrfy.py index 4315cf0ca69..5144f135336 100755 --- a/dev/scripts/exglobal_enkf_earc_vrfy.py +++ b/dev/scripts/exglobal_enkf_earc_vrfy.py @@ -3,7 +3,7 @@ import os from pygfs.task.archive import Archive -from pygfs.utils.archive_vars import ArchiveVrfyVars +from pygfs.utils.archive_vrfy_vars import ArchiveVrfyVars from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, chdir, logit # initialize root logger diff --git a/parm/archive/chem.yaml.j2 b/parm/archive/chem.yaml.j2 index 33c6dcef575..953f9d4f40e 100644 --- a/parm/archive/chem.yaml.j2 +++ b/parm/archive/chem.yaml.j2 @@ -4,4 +4,4 @@ chem: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/chem.tar" required: # TODO explicitize this set - - "{{ COMIN_CHEM_HISTORY | relpath(ROTDIR) }}/{{ head }}*" + - "{{ COMIN_CHEM_HISTORY }}/{{ head }}*" diff --git a/parm/archive/enkf.yaml.j2 b/parm/archive/enkf.yaml.j2 index b5ced5e5898..d31c1a32c4f 100644 --- a/parm/archive/enkf.yaml.j2 +++ b/parm/archive/enkf.yaml.j2 @@ -7,9 +7,7 @@ enkf: {% for mem in range(1, nmem_ens + 1) %} - "logs/{{ cycle_YMDH }}/{{ RUN }}_fcst_mem{{ '%03d' % mem }}.log" {% endfor %} - {% set fhrs = range(fhmin, fhmax + fhout, fhout) %} - {% set ngrps = fhrs | length %} - {% for grp in range(0, ngrps) %} + {% for grp in range(0, enkf_epos_ngrps) %} - "logs/{{ cycle_YMDH }}/{{ RUN }}_epos{{ '%03d' % grp }}.log" {% endfor %} {% if not DO_JEDIATMENS %} @@ -18,7 +16,7 @@ enkf: {% endif %} - "logs/{{ cycle_YMDH }}/{{ RUN }}_esfc.log" {% if not DO_JEDIATMENS %} - {% for grp in range(IAUFHRS | length) %} + {% for grp in range(iaufhrs_str | length) %} - "logs/{{ cycle_YMDH }}/{{ RUN }}_ecen{{ '%03d' % grp }}.log" {% endfor %} {% endif %} @@ -49,7 +47,7 @@ enkf: {% if DOHYBVAR %} {% if DOIAU %} {% for fhr in range(3, fhmax + 1, 3) %} - - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head |replace('enkf','')}}ensres.atm.f{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_HISTORY }}/{{ head |replace('enkf','')}}ensres.atm.f{{ '%03d' % fhr }}.nc" {% endfor %} {% endif %} {% endif %} @@ -58,10 +56,10 @@ enkf: # Ensemble mean and spread {% if is_gdas %} {% for fhr in range(3, fhmax + 1, 3) %} - - "{{ COMIN_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}ensmean.atm.f{{ '%03d' % fhr }}.nc" - - "{{ COMIN_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}ensmean.sfc.f{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_HISTORY_ENSSTAT }}/{{ head }}ensmean.atm.f{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_HISTORY_ENSSTAT }}/{{ head }}ensmean.sfc.f{{ '%03d' % fhr }}.nc" {% if ENKF_SPREAD %} - - "{{ COMIN_ATMOS_HISTORY_ENSSTAT | relpath(ROTDIR) }}/{{ head }}ensspread.atm.f{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_HISTORY_ENSSTAT }}/{{ head }}ensspread.atm.f{{ '%03d' % fhr }}.nc" {% endif %} {% endfor %} {% endif %} @@ -91,40 +89,40 @@ enkf: {% endif %} {% endif %} {% for file in da_stat_files %} - - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}{{ file }}" + - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}{{ file }}" {% endfor %} {% for file in da_conf_files %} - - "{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}{{ file }}" + - "{{ COMIN_CONF }}/{{ head }}{{ file }}" {% endfor %} {% if DO_JEDISNOWDA %} - - "{{ COMIN_SNOW_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}snow_analysis.ioda_hofx.ensmean.tar.gz" + - "{{ COMIN_SNOW_ANALYSIS_ENSSTAT }}/{{ head }}snow_analysis.ioda_hofx.ensmean.tar.gz" {% for itile in range(1,7) %} # Snow analysis is 3dvar - - "{{ COMIN_SNOW_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.snow_increment.sfc_data.tile{{ itile }}.nc" + - "{{ COMIN_SNOW_ANALYSIS_ENSSTAT }}/{{ cycle_YMD }}.{{ cycle_HH }}0000.snow_increment.sfc_data.tile{{ itile }}.nc" {% endfor %} {% endif %} # Ensemble mean analyses/increments # soil DA increments {% if DO_GSISOILDA %} - {% for fhr in IAUFHRS %} - - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}ensmean_increment.sfc.i{{ '%03d' % fhr }}.nc" + {% for fhr in iaufhrs_str %} + - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}ensmean_increment.sfc.i{{ fhr }}.nc" {% endfor %} {% endif %} # Atmospheric analyses/increment - {% for fhr in IAUFHRS %} + {% for fhr in iaufhrs_str %} {% if not DO_JEDIATMENS %} {% if do_calc_increment %} # Store analyses instead of increments - - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}ensmean_analysis.atm.a{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}ensmean_analysis.atm.a{{ fhr }}.nc" {% else %} # Store increments - - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}ensmean_increment.atm.i{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}ensmean_increment.atm.i{{ fhr }}.nc" {% endif %} {% else %} - - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}csg_ensmean_jedi_analysis.atm.a{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_ANALYSIS_ENSSTAT }}/{{ head }}csg_ensmean_jedi_analysis.atm.a{{ fhr }}.nc" {% endif %} {% endfor %} @@ -132,6 +130,6 @@ enkf: # End of analysis mean increments/analyses {% if DOHYBVAR_OCN %} - - "{{ COMIN_OCEAN_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}bg_ensvar.nc" - - "{{ COMIN_ICE_ANALYSIS_ENSSTAT | relpath(ROTDIR) }}/{{ head }}bg_ensvar.nc" + - "{{ COMIN_OCEAN_ANALYSIS_ENSSTAT }}/{{ head }}bg_ensvar.nc" + - "{{ COMIN_ICE_ANALYSIS_ENSSTAT }}/{{ head }}bg_ensvar.nc" {% endif %} diff --git a/parm/archive/enkf_arcdir.yaml.j2 b/parm/archive/enkf_arcdir.yaml.j2 index 5219f035610..4b5c06c607b 100644 --- a/parm/archive/enkf_arcdir.yaml.j2 +++ b/parm/archive/enkf_arcdir.yaml.j2 @@ -1,4 +1,4 @@ -# Variables provided by archive_vars.py: +# Variables provided by archive_vrfy_vars.py: # - cycle_HH, cycle_YMDH, cycle_YMD, head # - ARCDIR # - All COMIN_* paths diff --git a/parm/archive/enkf_grp.yaml.j2 b/parm/archive/enkf_grp.yaml.j2 index 6b6cbac9475..e5106477fac 100644 --- a/parm/archive/enkf_grp.yaml.j2 +++ b/parm/archive/enkf_grp.yaml.j2 @@ -2,49 +2,40 @@ enkf_grp: name: "ENKF_GRP" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_grp{{ ENSGRP }}.tar" required: - {% for mem in range(first_group_mem, last_group_mem + 1) %} - {% set imem = mem - first_group_mem %} - # Construct member COM directories - {% set COMIN_ATMOS_ANALYSIS_MEM = COMIN_ATMOS_ANALYSIS_MEM_list[imem] %} - {% set COMIN_ATMOS_HISTORY_MEM = COMIN_ATMOS_HISTORY_MEM_list[imem] %} - {% set COMIN_ATMOS_RESTART_MEM = COMIN_ATMOS_RESTART_MEM_list[imem] %} - # Forecast data - {% if is_gdas %} - {% for fhr in range(3, 10, 3) %} - - "{{ COMIN_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}atm.f{{ "%03d" % fhr }}.nc" - - "{{ COMIN_ATMOS_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}sfc.f{{ "%03d" % fhr }}.nc" - {% endfor %} - {% endif %} + {% if is_gdas %} + {% for fhr in range(3, 10, 3) %} + - "{{ COMIN_ATMOS_HISTORY_MEM }}/{{ head }}atm.f{{ "%03d" % fhr }}.nc" + - "{{ COMIN_ATMOS_HISTORY_MEM }}/{{ head }}sfc.f{{ "%03d" % fhr }}.nc" + {% endfor %} + {% endif %} # Store the individual member analysis data - {% if not lobsdiag_forenkf %} - - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}gsistat.txt" - {% endif %} - {% if DO_JEDIATMENS %} - {% for iaufhr in IAUFHRS %} - {% for itile in range(6) %} - - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}recentered_jedi_increment.atm.i{{ '%03d' % iaufhr }}.tile{{ itile+1 }}.nc" - {% endfor %} + {% if not lobsdiag_forenkf %} + - "{{ COMIN_ATMOS_RESTART_MEM }}/{{ head }}gsistat.txt" + {% endif %} + {% if DO_JEDIATMENS %} + {% for iaufhr in iaufhrs_str %} + {% for itile in range(6) %} + - "{{ COMIN_ATMOS_ANALYSIS_MEM }}/{{ head }}recentered_jedi_increment.atm.i{{ iaufhr }}.tile{{ itile+1 }}.nc" {% endfor %} + {% endfor %} + {% else %} + {% if do_calc_increment %} + - "{{ COMIN_ATMOS_ANALYSIS_MEM }}/{{ head }}analysis.atm.a006.nc" {% else %} - {% if do_calc_increment %} - - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}analysis.atm.a006.nc" - {% else %} - {% for iaufhr in IAUFHRS %} - {% set iaufhr = iaufhr %} - - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}recentered_increment.atm.i{{ '%03d' % iaufhr }}.nc" - {% endfor %} # iaufhr in IAUFHRS - {% endif %} + {% for iaufhr in iaufhrs_str %} + {% set iaufhr = iaufhr %} + - "{{ COMIN_ATMOS_ANALYSIS_MEM }}/{{ head }}recentered_increment.atm.i{{ iaufhr }}.nc" + {% endfor %} # iaufhr in iaufhrs_str {% endif %} + {% endif %} # soil DA increments - {% if is_gdas %} - {% if DO_GSISOILDA %} - {% for iaufhr in IAUFHRS %} - - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}increment.sfc.i{{ '%03d' % iaufhr }}.nc" - {% endfor %} - {% endif %} + {% if is_gdas %} + {% if DO_GSISOILDA %} + {% for iaufhr in iaufhrs_str %} + - "{{ COMIN_ATMOS_ANALYSIS_MEM }}/{{ head }}increment.sfc.i{{ iaufhr }}.nc" + {% endfor %} {% endif %} - - {% endfor %} # first_group_mem to last_group_mem + {% endif %} diff --git a/parm/archive/enkf_marine_grp.yaml.j2 b/parm/archive/enkf_marine_grp.yaml.j2 index 73912542e60..88bc7ab84a3 100644 --- a/parm/archive/enkf_marine_grp.yaml.j2 +++ b/parm/archive/enkf_marine_grp.yaml.j2 @@ -2,35 +2,18 @@ enkf_marine_grp: name: "ENKF_MARINE_GRP" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_marine_grp{{ ENSGRP }}.tar" required: - {% for mem in range(first_group_mem, last_group_mem + 1) %} - {% set imem = mem - first_group_mem %} - # Construct member COM directories - {% set COMIN_OCEAN_ANALYSIS_MEM = COMIN_OCEAN_ANALYSIS_MEM_list[imem] %} - {% set COMIN_OCEAN_LETKF_MEM = COMIN_OCEAN_LETKF_MEM_list[imem] %} - {% set COMIN_OCEAN_HISTORY_MEM = COMIN_OCEAN_HISTORY_MEM_list[imem] %} - {% set COMIN_ICE_ANALYSIS_MEM = COMIN_ICE_ANALYSIS_MEM_list[imem] %} - {% set COMIN_ICE_LETKF_MEM = COMIN_ICE_LETKF_MEM_list[imem] %} - {% set COMIN_ICE_HISTORY_MEM = COMIN_ICE_HISTORY_MEM_list[imem] %} - # Forecast data {% if is_gdas %} - - "{{ COMIN_OCEAN_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}inst.f009.nc" + - "{{ COMIN_OCEAN_HISTORY_MEM }}/{{ head }}inst.f009.nc" {% for body in ["ic", "inst.f009"] %} - - "{{ COMIN_ICE_HISTORY_MEM | relpath(ROTDIR) }}/{{ head }}{{ body }}.nc" + - "{{ COMIN_ICE_HISTORY_MEM }}/{{ head }}{{ body }}.nc" {% endfor %} {% endif %} # Store the individual member analysis data - {% if DOIAU_ENKF %} - {% set anl_delta = "-3H" | to_timedelta %} - {% else %} - {% set anl_delta = "0H" | to_timedelta %} - {% endif %} - {% set anl_time = current_cycle | add_to_datetime(anl_delta) %} {% if DOLETKF_OCN %} - - "{{ COMIN_OCEAN_LETKF_MEM | relpath(ROTDIR) }}/{{ head }}jedi_analysis.a006.nc" - - "{{ COMIN_ICE_LETKF_MEM | relpath(ROTDIR) }}/{{ head }}jedi_analysis.a006.nc" - - "{{ COMIN_CONF | relpath(ROTDIR) }}/soca_letkf.yaml" + - "{{ COMIN_OCEAN_LETKF_MEM }}/{{ head }}jedi_analysis.a006.nc" + - "{{ COMIN_ICE_LETKF_MEM }}/{{ head }}jedi_analysis.a006.nc" + - "{{ COMIN_CONF }}/soca_letkf.yaml" {% endif %} - {% endfor %} # first_group_mem to last_group_mem diff --git a/parm/archive/enkf_restarta_grp.yaml.j2 b/parm/archive/enkf_restarta_grp.yaml.j2 index 596d18f3aaf..685487daa77 100644 --- a/parm/archive/enkf_restarta_grp.yaml.j2 +++ b/parm/archive/enkf_restarta_grp.yaml.j2 @@ -2,81 +2,55 @@ enkf_restarta_grp: name: "ENKF_RESTARTA_GRP" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restarta_grp{{ ENSGRP }}.tar" required: - {% for mem in range(first_group_mem, last_group_mem + 1) %} - {% set imem = mem - first_group_mem %} - # Construct the pertinent member COM directories - {% set COMIN_ATMOS_ANALYSIS_MEM = COMIN_ATMOS_ANALYSIS_MEM_list[imem] %} - {% set COMIN_OCEAN_ANALYSIS_MEM = COMIN_OCEAN_ANALYSIS_MEM_list[imem] %} - {% set COMIN_ICE_ANALYSIS_MEM = COMIN_ICE_ANALYSIS_MEM_list[imem] %} - {% set COMIN_ATMOS_HISTORY_MEM = COMIN_ATMOS_HISTORY_MEM_list[imem] %} - {% set COMIN_ATMOS_RESTART_MEM = COMIN_ATMOS_RESTART_MEM_list[imem] %} - {% set COMIN_OCEAN_RESTART_MEM = COMIN_OCEAN_RESTART_MEM_list[imem] %} - {% set COMIN_ICE_RESTART_MEM = COMIN_ICE_RESTART_MEM_list[imem] %} - {% set COMIN_MED_RESTART_MEM = COMIN_MED_RESTART_MEM_list[imem] %} - # Store bias data - {% if not lobsdiag_forenkf %} - - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias.txt" - - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_air.txt" - - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_int.txt" - - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}abias_pc.txt" - {% endif %} + {% if not lobsdiag_forenkf %} + - "{{ COMIN_ATMOS_RESTART_MEM }}/{{ head }}abias.txt" + - "{{ COMIN_ATMOS_RESTART_MEM }}/{{ head }}abias_air.txt" + - "{{ COMIN_ATMOS_RESTART_MEM }}/{{ head }}abias_int.txt" + - "{{ COMIN_ATMOS_RESTART_MEM }}/{{ head }}abias_pc.txt" + {% endif %} # Member surface analysis data - {% if DOIAU_ENKF %} - {% set anl_delta = "-3H" | to_timedelta %} - {% else %} - {% set anl_delta = "0H" | to_timedelta %} - {% endif %} - {% set anl_time = current_cycle | add_to_datetime(anl_delta) %} - {% for itile in range(1, 7) %} - - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" - {% endfor %} + {% for itile in range(1, 7) %} + - "{{ COMIN_ATMOS_RESTART_MEM }}/{{ anl_YMD }}.{{ anl_HH }}0000.sfcanl_data.tile{{ itile }}.nc" + {% endfor %} # Member atmospheric increments or analysis data - {% if DO_JEDIATMENS %} - {% for iaufhr in IAUFHRS %} - {% for itile in range(6) %} - - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}recentered_jedi_increment.atm.i{{ '%03d' % iaufhr }}.tile{{ itile+1 }}.nc" - {% endfor %} + {% if DO_JEDIATMENS %} + {% for iaufhr in iaufhrs_str %} + {% for itile in range(6) %} + - "{{ COMIN_ATMOS_ANALYSIS_MEM }}/{{ head }}recentered_jedi_increment.atm.i{{ iaufhr }}.tile{{ itile+1 }}.nc" {% endfor %} - {% else %} - {% for iaufhr in IAUFHRS %} - {% if do_calc_increment %} - - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}analysis.atm.a{{ '%03d' % iaufhr }}.nc" - {% else %} - - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}recentered_increment.atm.i{{ '%03d' % iaufhr }}.nc" - {% endif %} - {% endfor %} # iaufhr in IAUFHRS - {% endif %} + {% endfor %} + {% else %} + {% for iaufhr in iaufhrs_str %} + {% if do_calc_increment %} + - "{{ COMIN_ATMOS_ANALYSIS_MEM }}/{{ head }}analysis.atm.a{{ iaufhr }}.nc" + {% else %} + - "{{ COMIN_ATMOS_ANALYSIS_MEM }}/{{ head }}recentered_increment.atm.i{{ iaufhr }}.nc" + {% endif %} + {% endfor %} # iaufhr in iaufhrs_str + {% endif %} # soil DA increments - {% if DO_GSISOILDA %} - {% for iaufhr in IAUFHRS %} - - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}increment.sfc.i{{ '%03d' % iaufhr }}.nc" - {% endfor %} # iaufhr in IAUFHRS - {% if DO_LAND_IAU %} #sfc inc tiles needed for warm start - {% for itile in range(1,7) %} - - "{{ COMIN_ATMOS_ANALYSIS_MEM | relpath(ROTDIR) }}/increment.sfc.tile{{ itile }}.nc" - {% endfor %} - {% endif %} + {% if DO_GSISOILDA %} + {% for iaufhr in iaufhrs_str %} + - "{{ COMIN_ATMOS_ANALYSIS_MEM }}/{{ head }}increment.sfc.i{{ iaufhr }}.nc" + {% endfor %} # iaufhr in iaufhrs_str + {% if DO_LAND_IAU %} #sfc inc tiles needed for warm start + {% for itile in range(1,7) %} + - "{{ COMIN_ATMOS_ANALYSIS_MEM }}/increment.sfc.tile{{ itile }}.nc" + {% endfor %} {% endif %} + {% endif %} # Conventional data - {% if not lobsdiag_forenkf and not DO_JEDIATMENS %} - - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ head }}cnvstat" - {% endif %} - - {% if DOHYBVAR_OCN %} - {% if DOIAU_ENKF %} - {% set rst_delta = "+3H" | to_timedelta %} - {% else %} - {% set rst_delta = "+6H" | to_timedelta %} - {% endif %} # DOIAU_ENKF - {% set rst_time = current_cycle | add_to_datetime(rst_delta) %} - - "{{ COMIN_MED_RESTART_MEM | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ rst_time | strftime("%H") }}0000.ufs.cpld.cpl.r.nc" - - "{{ COMIN_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ head }}mom6_increment.i006.nc" - - "{{ COMIN_ICE_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ anl_time | to_fv3time }}.analysis.cice_model.res.nc" - {% endif %} # DOHYBVAR_OCN + {% if not lobsdiag_forenkf and not DO_JEDIATMENS %} + - "{{ COMIN_ATMOS_RESTART_MEM }}/{{ head }}cnvstat" + {% endif %} - {% endfor %} # first_group_mem to last_group_mem + {% if DOHYBVAR_OCN %} + - "{{ COMIN_MED_RESTART_MEM }}/{{ cycle_YMD }}.{{ rst_HH_restarta }}0000.ufs.cpld.cpl.r.nc" + - "{{ COMIN_OCEAN_ANALYSIS_MEM }}/{{ head }}mom6_increment.i006.nc" + - "{{ COMIN_ICE_ANALYSIS_MEM }}/{{ anl_YMD }}.{{ anl_HH }}0000.analysis.cice_model.res.nc" + {% endif %} # DOHYBVAR_OCN diff --git a/parm/archive/enkf_restartb_grp.yaml.j2 b/parm/archive/enkf_restartb_grp.yaml.j2 index 337be34296b..c956af9b350 100644 --- a/parm/archive/enkf_restartb_grp.yaml.j2 +++ b/parm/archive/enkf_restartb_grp.yaml.j2 @@ -2,53 +2,30 @@ enkf_restartb_grp: name: "ENKF_RESTARTB_GRP" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/{{ RUN }}_restartb_grp{{ ENSGRP }}.tar" required: - {% for mem in range(first_group_mem, last_group_mem + 1) %} - {% set imem = mem - first_group_mem %} - {% set COMIN_ATMOS_RESTART_MEM = COMIN_ATMOS_RESTART_MEM_list[imem] %} - {% set COMIN_OCEAN_RESTART_MEM = COMIN_OCEAN_RESTART_MEM_list[imem] %} - {% set COMIN_ICE_RESTART_MEM = COMIN_ICE_RESTART_MEM_list[imem] %} - {% set COMIN_MED_RESTART_MEM = COMIN_MED_RESTART_MEM_list[imem] %} - # Grab surface analysis data. # If IAU is on, grab the beginning of the window, otherwise grab the center. - {% if DOIAU_ENKF %} - {% set offset_td = "-3H" | to_timedelta %} - {% else %} - {% set offset_td = "0H" | to_timedelta %} - {% endif %} - {% set offset_dt = current_cycle | add_to_datetime(offset_td) %} - {% set offset_YMD = offset_dt | to_YMD %} - {% set offset_HH = offset_dt | strftime("%H") %} - {% set prefix = offset_YMD + "." + offset_HH + "0000" %} {% for itile in range(1, 7) %} - - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ prefix }}.sfcanl_data.tile{{ itile }}.nc" + - "{{ COMIN_ATMOS_RESTART_MEM }}/{{ anl_YMD }}.{{ anl_HH }}0000.sfcanl_data.tile{{ itile }}.nc" {% endfor %} # Now get the restart files. {% if is_gdas %} - {% for r_time in range(restart_interval, fhmax + 1, restart_interval) %} - {% set r_timedelta = (r_time | string + "H") | to_timedelta %} - {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} - {% set r_prefix = r_dt | to_YMD + "." + r_dt | strftime("%H") + "0000" %} - {% for itile in range(1, 7) %} - {% for datatype in ["fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" - {% if DO_CA %} - - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.ca_data.tile{{ itile }}.nc" - {% endif %} - {% endfor %} - {% endfor %} - - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" - - "{{ COMIN_ATMOS_RESTART_MEM | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" - {% endfor %} + {% for r_prefix in restart_prefixes %} + {% for itile in range(1, 7) %} + {% for datatype in ["fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} + - "{{ COMIN_ATMOS_RESTART_MEM }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" + {% if DO_CA %} + - "{{ COMIN_ATMOS_RESTART_MEM }}/{{ r_prefix }}.ca_data.tile{{ itile }}.nc" + {% endif %} + {% endfor %} + {% endfor %} + - "{{ COMIN_ATMOS_RESTART_MEM }}/{{ r_prefix }}.coupler.res" + - "{{ COMIN_ATMOS_RESTART_MEM }}/{{ r_prefix }}.fv_core.res.nc" + {% endfor %} {% endif %} # Add ocean restarts {% if DOHYBVAR_OCN %} - {% set rst_delta = "+3H" | to_timedelta %} - {% set rst_time = current_cycle | add_to_datetime(rst_delta) %} - - "{{ COMIN_OCEAN_RESTART_MEM | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ rst_time | strftime("%H") }}0000.MOM.r*.nc" - - "{{ COMIN_MED_RESTART_MEM | relpath(ROTDIR) }}/{{ cycle_YMD }}.{{ rst_time | strftime("%H") }}0000.ufs.cpld.cpl.r.nc" + - "{{ COMIN_OCEAN_RESTART_MEM }}/{{ cycle_YMD }}.{{ rst_HH_restartb }}0000.MOM.r*.nc" + - "{{ COMIN_MED_RESTART_MEM }}/{{ cycle_YMD }}.{{ rst_HH_restartb }}0000.ufs.cpld.cpl.r.nc" {% endif %} - {% endfor %} - diff --git a/parm/archive/gcafs_arcdir.yaml.j2 b/parm/archive/gcafs_arcdir.yaml.j2 index 1216755bf65..d5756f09a1b 100644 --- a/parm/archive/gcafs_arcdir.yaml.j2 +++ b/parm/archive/gcafs_arcdir.yaml.j2 @@ -1,4 +1,4 @@ -# Variables provided by archive_vars.py: +# Variables provided by archive_vrfy_vars.py: # - cycle_HH, cycle_YMDH, cycle_YMD, head # - VFYARC # - All COMIN_* paths diff --git a/parm/archive/gdas.yaml.j2 b/parm/archive/gdas.yaml.j2 index 44299fdd945..b28c0ab5deb 100644 --- a/parm/archive/gdas.yaml.j2 +++ b/parm/archive/gdas.yaml.j2 @@ -41,40 +41,40 @@ gdas: {% endif %} # Analysis Master GRIB2 data - - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}master.analysis.grib2" - - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}master.analysis.grib2.idx" + - "{{ COMIN_ATMOS_MASTER }}/{{ head }}master.analysis.grib2" + - "{{ COMIN_ATMOS_MASTER }}/{{ head }}master.analysis.grib2.idx" # Analysis GRIB2 (sub-sampled) data - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_a.0p25.analysis.grib2" - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_a.0p25.analysis.grib2.idx" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_a.1p00.analysis.grib2" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_a.1p00.analysis.grib2.idx" - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_b.0p25.analysis.grib2" - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_b.0p25.analysis.grib2.idx" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_b.1p00.analysis.grib2" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_b.1p00.analysis.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_a.0p25.analysis.grib2" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_a.0p25.analysis.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_a.1p00.analysis.grib2" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_a.1p00.analysis.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_b.0p25.analysis.grib2" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_b.0p25.analysis.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_b.1p00.analysis.grib2" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_b.1p00.analysis.grib2.idx" # Analysis netCDF (raw) data {% if DO_JEDIATMVAR %} {% if ATMINC_GRID == 'gaussian' %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}analysis.atm.a006.nc" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}analysis.sfc.a006.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}analysis.atm.a006.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}analysis.sfc.a006.nc" {% else %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}jedi_analysis.atm.a006.nc" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}jedi_analysis.sfc.a006.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}jedi_analysis.atm.a006.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}jedi_analysis.sfc.a006.nc" {% endif %} {% else %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}analysis.atm.a006.nc" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}analysis.sfc.a006.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}analysis.atm.a006.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}analysis.sfc.a006.nc" {% endif %} {% if DOHYBVAR %} # Ensemble-resolution analysis {% if DOIAU %} # Ensemble-resolution IAU analyses - {% for fhr in IAUFHRS %} + {% for fhr in iaufhrs_str %} {% if not DO_JEDIATMVAR %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}ensres_analysis.atm.a{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}ensres_analysis.atm.a{{ fhr }}.nc" {% endif %} {% endfor %} {% endif %} @@ -83,70 +83,70 @@ gdas: # Analysis state {% if DO_JEDIATMVAR %} - - "{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}anlvar.atm.yaml" - - "{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}anlvar.fv3.atm.yaml" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmos_analysis.ioda_hofx.tar.gz" - - "{{ COMIN_ATMOS_ANLMON | relpath(ROTDIR) }}/{{ head }}atmos_analysis.ioda_hofx_stats.tar.gz" + - "{{ COMIN_CONF }}/{{ head }}anlvar.atm.yaml" + - "{{ COMIN_CONF }}/{{ head }}anlvar.fv3.atm.yaml" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}atmos_analysis.ioda_hofx.tar.gz" + - "{{ COMIN_ATMOS_ANLMON }}/{{ head }}atmos_analysis.ioda_hofx_stats.tar.gz" {% else %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat.txt" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat.tar" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias.txt" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_air.txt" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_pc.txt" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int.txt" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}gsistat.txt" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}cnvstat.tar" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}abias.txt" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}abias_air.txt" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}abias_pc.txt" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}abias_int.txt" {% endif %} {% if DO_AERO_ANL %} - - "{{ COMIN_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aero_analysis.ioda_hofx.tar.gz" - - "{{ COMIN_CHEM_ANLMON | relpath(ROTDIR) }}/{{ head }}aero_analysis.ioda_hofx_stats.tar.gz" - - "{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}aerovar.yaml" + - "{{ COMIN_CHEM_ANALYSIS }}/{{ head }}aero_analysis.ioda_hofx.tar.gz" + - "{{ COMIN_CHEM_ANLMON }}/{{ head }}aero_analysis.ioda_hofx_stats.tar.gz" + - "{{ COMIN_CONF }}/{{ head }}aerovar.yaml" {% for itile in range(1,7) %} - - "{{ COMIN_CHEM_ANALYSIS | relpath(ROTDIR) }}/aeroinc.{{ cycle_YMD }}.{{ cycle_HH }}0000.fv_tracer.res.tile{{ itile }}.nc" + - "{{ COMIN_CHEM_ANALYSIS }}/aeroinc.{{ cycle_YMD }}.{{ cycle_HH }}0000.fv_tracer.res.tile{{ itile }}.nc" {% endfor %} {% endif %} {% if DO_PREP_OBS_AERO %} - - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs" - - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs" + - "{{ COMIN_OBS }}/{{ head }}aeroobs" + - "{{ COMIN_OBS }}/{{ head }}aerorawobs" {% endif %} {% if DO_JEDIOCNVAR %} - - "{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}marine_analysis.ioda_hofx.tar.gz" + - "{{ COMIN_OCEAN_ANALYSIS }}/{{ head }}marine_analysis.ioda_hofx.tar.gz" {% endif %} # Snow analysis data {% if DO_JEDISNOWDA %} {% for itile in range(1,7) %} - - "{{ COMIN_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ current_cycle | to_fv3time }}.snow_increment.sfc_data.tile{{ itile }}.nc" + - "{{ COMIN_SNOW_ANALYSIS }}/{{ cycle_fv3time }}.snow_increment.sfc_data.tile{{ itile }}.nc" {% endfor %} - - "{{ COMIN_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ head }}snow_analysis.ioda_hofx.tar.gz" - - "{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}snowanlvar.yaml" - - "{{ COMIN_SNOW_ANLMON | relpath(ROTDIR) }}/{{ head }}snow_analysis.ioda_hofx_stats.tar.gz" + - "{{ COMIN_SNOW_ANALYSIS }}/{{ head }}snow_analysis.ioda_hofx.tar.gz" + - "{{ COMIN_CONF }}/{{ head }}snowanlvar.yaml" + - "{{ COMIN_SNOW_ANLMON }}/{{ head }}snow_analysis.ioda_hofx_stats.tar.gz" {% endif %} # Ozone verification {% if DO_VERFOZN %} - - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_cnt.{{ cycle_YMDH }}" - - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_diag.{{ cycle_YMDH }}" - - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/time/bad_pen.{{ cycle_YMDH }}" - - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/time/stdout.time.tar.gz" - - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/horiz/stdout.horiz.tar.gz" + - "{{ COMIN_ATMOS_OZNMON }}/time/bad_cnt.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_OZNMON }}/time/bad_diag.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_OZNMON }}/time/bad_pen.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_OZNMON }}/time/stdout.time.tar.gz" + - "{{ COMIN_ATMOS_OZNMON }}/horiz/stdout.horiz.tar.gz" - "logs/{{ cycle_YMDH }}/{{ RUN }}_verfozn.log" {% endif %} # Radiance verification {% if DO_VERFRAD %} - - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_angle.tar.gz" - - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcoef.tar.gz" - - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_bcor.tar.gz" - - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/radmon_time.tar.gz" + - "{{ COMIN_ATMOS_RADMON }}/radmon_angle.tar.gz" + - "{{ COMIN_ATMOS_RADMON }}/radmon_bcoef.tar.gz" + - "{{ COMIN_ATMOS_RADMON }}/radmon_bcor.tar.gz" + - "{{ COMIN_ATMOS_RADMON }}/radmon_time.tar.gz" - "logs/{{ cycle_YMDH }}/{{ RUN }}_verfrad.log" {% endif %} # Minimization monitor {% if DO_VMINMON %} - - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" - - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" - - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" - - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" - - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" + - "{{ COMIN_ATMOS_MINMON }}/{{ cycle_YMDH }}.costs.txt" + - "{{ COMIN_ATMOS_MINMON }}/{{ cycle_YMDH }}.cost_terms.txt" + - "{{ COMIN_ATMOS_MINMON }}/{{ cycle_YMDH }}.gnorms.ieee_d" + - "{{ COMIN_ATMOS_MINMON }}/{{ cycle_YMDH }}.reduction.ieee_d" + - "{{ COMIN_ATMOS_MINMON }}/gnorm_data.txt" - "logs/{{ cycle_YMDH }}/{{ RUN }}_vminmon.log" {% endif %} {% endif %} # End of cycled data @@ -161,39 +161,39 @@ gdas: - "logs/{{ cycle_YMDH }}/{{ RUN }}_atmos_upp_f{{ fhr3 }}.log" {% endif %} ## not WRITE_DOPOST # Forecast GRIB2 data - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_a.0p25.f{{ fhr3 }}.grib2" - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_a.0p25.f{{ fhr3 }}.grib2.idx" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_a.1p00.f{{ fhr3 }}.grib2" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_a.1p00.f{{ fhr3 }}.grib2.idx" - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_b.0p25.f{{ fhr3 }}.grib2" - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_b.0p25.f{{ fhr3 }}.grib2.idx" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_b.1p00.f{{ fhr3 }}.grib2" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_b.1p00.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_a.0p25.f{{ fhr3 }}.grib2" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_a.0p25.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_a.1p00.f{{ fhr3 }}.grib2" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_a.1p00.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_b.0p25.f{{ fhr3 }}.grib2" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_b.0p25.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_b.1p00.f{{ fhr3 }}.grib2" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_b.1p00.f{{ fhr3 }}.grib2.idx" # Forecast GRIB2 fluxes - - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sflux.f{{ fhr3 }}.grib2" - - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sflux.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_ATMOS_MASTER }}/{{ head }}sflux.f{{ fhr3 }}.grib2" + - "{{ COMIN_ATMOS_MASTER }}/{{ head }}sflux.f{{ fhr3 }}.grib2.idx" # FV3 log - - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}log.f{{ fhr3 }}.txt" + - "{{ COMIN_ATMOS_HISTORY }}/{{ head }}log.f{{ fhr3 }}.txt" # Raw netCDF forecasts - - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.f{{ fhr3 }}.nc" - - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfc.f{{ fhr3 }}.nc" + - "{{ COMIN_ATMOS_HISTORY }}/{{ head }}atm.f{{ fhr3 }}.nc" + - "{{ COMIN_ATMOS_HISTORY }}/{{ head }}sfc.f{{ fhr3 }}.nc" {% endfor %} # Forecast Master GRIB2 data for F006 - - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}master.f003.grib2" - - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}master.f006.grib2" + - "{{ COMIN_ATMOS_MASTER }}/{{ head }}master.f003.grib2" + - "{{ COMIN_ATMOS_MASTER }}/{{ head }}master.f006.grib2" optional: {% if MODE == "cycled" %} # Ozone and Radiance diagnostic files may not always be created - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}oznstat.tar" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat.tar" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}oznstat.tar" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}radstat.tar" # Radiance verification (only created if there are problems) {% if DO_VERFRAD %} - - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/bad_diag.{{ cycle_YMDH }}" - - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/bad_pen.{{ cycle_YMDH }}" - - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/low_count.{{ cycle_YMDH }}" - - "{{ COMIN_ATMOS_RADMON | relpath(ROTDIR) }}/warning.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_RADMON }}/bad_diag.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_RADMON }}/bad_pen.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_RADMON }}/low_count.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_RADMON }}/warning.{{ cycle_YMDH }}" {% endif %} {% if DO_VERFOZN %} @@ -209,10 +209,10 @@ gdas: {% set suffix = "" %} {% endif %} {% for type in oznmon_types %} - - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.analysis.ctl" - - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.analysis.{{ cycle_YMDH }}.ieee_d{{ suffix }}" - - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.guess.ctl" - - "{{ COMIN_ATMOS_OZNMON | relpath(ROTDIR) }}/{{ group }}/{{ type }}.guess.{{ cycle_YMDH }}.ieee_d{{ suffix }}" + - "{{ COMIN_ATMOS_OZNMON }}/{{ group }}/{{ type }}.analysis.ctl" + - "{{ COMIN_ATMOS_OZNMON }}/{{ group }}/{{ type }}.analysis.{{ cycle_YMDH }}.ieee_d{{ suffix }}" + - "{{ COMIN_ATMOS_OZNMON }}/{{ group }}/{{ type }}.guess.ctl" + - "{{ COMIN_ATMOS_OZNMON }}/{{ group }}/{{ type }}.guess.{{ cycle_YMDH }}.ieee_d{{ suffix }}" {% endfor %} {% endfor %} {% endif %} diff --git a/parm/archive/gdas_restarta.yaml.j2 b/parm/archive/gdas_restarta.yaml.j2 index 4589b423311..07dda9ce567 100644 --- a/parm/archive/gdas_restarta.yaml.j2 +++ b/parm/archive/gdas_restarta.yaml.j2 @@ -5,15 +5,15 @@ gdas_restarta: required: # Deterministic analysis increments {% if DO_JEDIATMVAR and not (ATMINC_GRID == 'gaussian') %} - {% for iaufhr in IAUFHRS %} + {% for iaufhr in iaufhrs_str %} {% for itile in range(6) %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}jedi_increment.atm.i{{ "%03d" % iaufhr }}.tile{{ itile+1 }}.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}jedi_increment.atm.i{{ iaufhr }}.tile{{ itile+1 }}.nc" {% endfor %} {% endfor %} {% else %} # IAU increments - {% for iaufhr in IAUFHRS %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}increment.atm.i{{ "%03d" % iaufhr }}.nc" + {% for iaufhr in iaufhrs_str %} + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}increment.atm.i{{ iaufhr }}.nc" {% endfor %} {% endif %} @@ -21,39 +21,37 @@ gdas_restarta: {% if DO_GSISOILDA %} {% if DO_LAND_IAU %} #regridded tiles for land iau {% for itile in range(1,7) %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/increment.sfc.tile{{ itile }}.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/increment.sfc.tile{{ itile }}.nc" {% endfor %} {% endif %} {% endif %} # Surface analysis tiles - {% if DOHYBVAR and DOIAU %} - {% set anl_offset = "-3H" %} - {% else %} - {% set anl_offset = "0H" %} - {% endif %} - {% set anl_timedelta = anl_offset | to_timedelta %} - {% set anl_time = current_cycle | add_to_datetime(anl_timedelta) %} + # anl_time_YMD and anl_time_HH are pre-calculated in Python: -3H if DOHYBVAR and DOIAU, 0H otherwise {% for itile in range(1,7) %} - - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ itile }}.nc" + - "{{ COMIN_ATMOS_RESTART }}/{{ anl_time_YMD }}.{{ anl_time_HH }}0000.sfcanl_data.tile{{ itile }}.nc" {% endfor %} # Initial biases {% if not DO_JEDIATMVAR %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias.txt" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_air.txt" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_pc.txt" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}cnvstat.tar" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}abias_int.txt" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}increment.dtf.i006.nc" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}analysis.done.txt" + {% for suffix in [ + 'abias.txt', + 'abias_air.txt', + 'abias_pc.txt', + 'cnvstat.tar', + 'abias_int.txt', + 'increment.dtf.i006.nc', + 'analysis.done.txt' + ] %} + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}{{ suffix }}" + {% endfor %} {% else %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}rad_varbc_params.tar" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}rad_varbc_params.tar" {% endif %} # Input BUFR files - - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" - - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" - - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" + {% for suffix in ['nsstbufr', 'prepbufr', 'prepbufr.acft_profiles'] %} + - "{{ COMIN_OBS }}/{{ head }}{{ suffix }}" + {% endfor %} optional: - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}radstat.tar" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}radstat.tar" diff --git a/parm/archive/gdas_restartb.yaml.j2 b/parm/archive/gdas_restartb.yaml.j2 index 620e0fb9884..eaeae4a2159 100644 --- a/parm/archive/gdas_restartb.yaml.j2 +++ b/parm/archive/gdas_restartb.yaml.j2 @@ -3,40 +3,30 @@ gdas_restartb: name: "GDAS_RESTARTB" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdas_restartb.tar" required: + {# All time prefixes are pre-calculated in Python: offset_prefix (-3H if DOIAU), center_prefix, r_prefix_list #} # Grab the surface analysis data. # If IAU is on, grab the beginning of the window. {% if DOIAU %} - {% set offset_td = "-3H" | to_timedelta %} - {% set offset_dt = current_cycle | add_to_datetime(offset_td) %} - {% set offset_YMD = offset_dt | to_YMD %} - {% set offset_HH = offset_dt | strftime("%H") %} - {% set prefix = offset_YMD + "." + offset_HH + "0000" %} {% for itile in range(1, 7) %} - - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ prefix }}.sfcanl_data.tile{{ itile }}.nc" + - "{{ COMIN_ATMOS_RESTART }}/{{ offset_prefix }}.sfcanl_data.tile{{ itile }}.nc" {% endfor %} {% endif %} # Regardless, always grab the center surface analysis data. - {% set prefix = cycle_YMD + "." + cycle_HH + "0000" %} {% for itile in range(1, 7) %} - - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ prefix }}.sfcanl_data.tile{{ itile }}.nc" + - "{{ COMIN_ATMOS_RESTART }}/{{ center_prefix }}.sfcanl_data.tile{{ itile }}.nc" {% endfor %} # Now get the restart files. - {% for r_time in range(restart_interval_gdas, FHMAX + 1, restart_interval_gdas) %} - {% set r_timedelta = (r_time | string + "H") | to_timedelta %} - {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} - {% set r_YMD = r_dt | to_YMD %} - {% set r_HH = r_dt | strftime("%H") %} - {% set r_prefix = r_YMD + "." + r_HH + "0000" %} + {% for r_prefix in r_prefix_list %} {% for itile in range(1, 7) %} {% for datatype in ["fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" + - "{{ COMIN_ATMOS_RESTART }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" {% if DO_CA %} - - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.ca_data.tile{{ itile }}.nc" + - "{{ COMIN_ATMOS_RESTART }}/{{ r_prefix }}.ca_data.tile{{ itile }}.nc" {% endif %} {% endfor %} {% endfor %} - - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" - - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" + - "{{ COMIN_ATMOS_RESTART }}/{{ r_prefix }}.coupler.res" + - "{{ COMIN_ATMOS_RESTART }}/{{ r_prefix }}.fv_core.res.nc" {% endfor %} diff --git a/parm/archive/gdasice.yaml.j2 b/parm/archive/gdasice.yaml.j2 index 9c341be04a2..53e181f2fd6 100644 --- a/parm/archive/gdasice.yaml.j2 +++ b/parm/archive/gdasice.yaml.j2 @@ -2,6 +2,6 @@ gdasice: name: "GDASICE" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice.tar" required: - - "{{ COMIN_ICE_HISTORY | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.ic.nc" - - "{{ COMIN_ICE_HISTORY | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.inst.f006.nc" - - '{{ COMIN_CONF | relpath(ROTDIR) }}/ufs.ice_in' + - "{{ COMIN_ICE_HISTORY }}/{{ RUN }}.t{{ cycle_HH }}z.ic.nc" + - "{{ COMIN_ICE_HISTORY }}/{{ RUN }}.t{{ cycle_HH }}z.inst.f006.nc" + - '{{ COMIN_CONF }}/ufs.ice_in' diff --git a/parm/archive/gdasice_restart.yaml.j2 b/parm/archive/gdasice_restart.yaml.j2 index c066e4e7d9f..b895f98cd8d 100644 --- a/parm/archive/gdasice_restart.yaml.j2 +++ b/parm/archive/gdasice_restart.yaml.j2 @@ -3,4 +3,4 @@ gdasice_restart: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasice_restart.tar" required: # TODO: explicitly name the restart files to archive - - '{{ COMIN_ICE_RESTART | relpath(ROTDIR) }}/*' + - '{{ COMIN_ICE_RESTART }}/*' diff --git a/parm/archive/gdasocean.yaml.j2 b/parm/archive/gdasocean.yaml.j2 index c05d746a7ba..f08b7335225 100644 --- a/parm/archive/gdasocean.yaml.j2 +++ b/parm/archive/gdasocean.yaml.j2 @@ -3,6 +3,6 @@ gdasocean: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean.tar" # the FHMIN-hour forecast is skipped when DOIAU="NO" required: - - "{{ COMIN_OCEAN_HISTORY | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.inst.f006.nc" - - '{{ COMIN_CONF | relpath(ROTDIR) }}/ufs.MOM_input' - - '{{ COMIN_CONF | relpath(ROTDIR) }}/MOM_parameter_doc.all' + - "{{ COMIN_OCEAN_HISTORY }}/{{ RUN }}.t{{ cycle_HH }}z.inst.f006.nc" + - '{{ COMIN_CONF }}/ufs.MOM_input' + - '{{ COMIN_CONF }}/MOM_parameter_doc.all' diff --git a/parm/archive/gdasocean_analysis.yaml.j2 b/parm/archive/gdasocean_analysis.yaml.j2 index 4ec133fa108..a60e25dbfa5 100644 --- a/parm/archive/gdasocean_analysis.yaml.j2 +++ b/parm/archive/gdasocean_analysis.yaml.j2 @@ -3,43 +3,38 @@ gdasocean_analysis: name: "GDASOCEAN_ANALYSIS" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_analysis.tar" required: - {% if DOIAU %} - {% set anl_delta = "-3H" | to_timedelta %} - {% else %} - {% set anl_delta = "0H" | to_timedelta %} - {% endif %} - {% set anl_time = current_cycle | add_to_datetime(anl_delta) %} + {# anl_time_YMD and anl_time_HH are pre-calculated in Python: -3H if DOIAU, 0H otherwise #} # analysis and analysis increments - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}jedi_analysis.a006.nc' - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}jedi_increment.i006.nc' - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}mom6_increment.i006.nc' - - '{{ COMIN_ICE_ANALYSIS | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.analysis.cice_model.res.nc' - - '{{ COMIN_ICE_ANALYSIS | relpath(ROTDIR) }}/{{ head }}jedi_analysis.a006.nc' - - '{{ COMIN_ICE_ANALYSIS | relpath(ROTDIR) }}/{{ head }}jedi_increment.i006.nc' + - '{{ COMIN_OCEAN_ANALYSIS }}/{{ head }}jedi_analysis.a006.nc' + - '{{ COMIN_OCEAN_ANALYSIS }}/{{ head }}jedi_increment.i006.nc' + - '{{ COMIN_OCEAN_ANALYSIS }}/{{ head }}mom6_increment.i006.nc' + - '{{ COMIN_ICE_ANALYSIS }}/{{ anl_time_YMD }}.{{ anl_time_HH }}0000.analysis.cice_model.res.nc' + - '{{ COMIN_ICE_ANALYSIS }}/{{ head }}jedi_analysis.a006.nc' + - '{{ COMIN_ICE_ANALYSIS }}/{{ head }}jedi_increment.i006.nc' # static background error - - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}bkgerr_parametric_stddev.nc' - - '{{ COMIN_ICE_BMATRIX | relpath(ROTDIR) }}/{{ head }}bkgerr_parametric_stddev.nc' + - '{{ COMIN_OCEAN_BMATRIX }}/{{ head }}bkgerr_parametric_stddev.nc' + - '{{ COMIN_ICE_BMATRIX }}/{{ head }}bkgerr_parametric_stddev.nc' # ensemble background error {% if NMEM_ENS > 2 %} - - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}recentering_error.nc' - - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_ensb.yaml' - - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_ensweights.yaml' + - '{{ COMIN_OCEAN_BMATRIX }}/{{ head }}recentering_error.nc' + - '{{ COMIN_CONF }}/{{ head }}soca_ensb.yaml' + - '{{ COMIN_CONF }}/{{ head }}soca_ensweights.yaml' {% endif %} # runtime configs - - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_setcorscales.yaml' - - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_chgres.yaml' - - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_diagb.yaml' - - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}gridgen.yaml' - - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_vtscales.yaml' - - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}bmat_fields_metadata.yaml' - - '{{ COMIN_CONF | relpath(ROTDIR) }}/var.yaml' + - '{{ COMIN_CONF }}/{{ head }}soca_setcorscales.yaml' + - '{{ COMIN_CONF }}/{{ head }}soca_chgres.yaml' + - '{{ COMIN_CONF }}/{{ head }}soca_diagb.yaml' + - '{{ COMIN_CONF }}/{{ head }}gridgen.yaml' + - '{{ COMIN_CONF }}/{{ head }}soca_vtscales.yaml' + - '{{ COMIN_CONF }}/{{ head }}bmat_fields_metadata.yaml' + - '{{ COMIN_CONF }}/var.yaml' # soca_parameters_diffusion_vt, and soca_parameters_diffusion_hz when present - - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_parameters_diffusion_??.yaml' - - '{{ COMIN_CONF | relpath(ROTDIR) }}/soca_incpostproc.yaml' + - '{{ COMIN_CONF }}/{{ head }}soca_parameters_diffusion_??.yaml' + - '{{ COMIN_CONF }}/soca_incpostproc.yaml' optional: # obs space diags - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags/{{ head }}ocn.*.stats.csv' - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags/*.nc' + - '{{ COMIN_OCEAN_ANALYSIS }}/diags/{{ head }}ocn.*.stats.csv' + - '{{ COMIN_OCEAN_ANALYSIS }}/diags/*.nc' diff --git a/parm/archive/gdasocean_restart.yaml.j2 b/parm/archive/gdasocean_restart.yaml.j2 index ff6a0776477..478f34868fc 100644 --- a/parm/archive/gdasocean_restart.yaml.j2 +++ b/parm/archive/gdasocean_restart.yaml.j2 @@ -3,5 +3,5 @@ gdasocean_restart: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdasocean_restart.tar" required: # TODO: explicitly name the restart files to archive - - '{{ COMIN_OCEAN_RESTART | relpath(ROTDIR) }}/*' - - '{{ COMIN_MED_RESTART | relpath(ROTDIR) }}/*' + - '{{ COMIN_OCEAN_RESTART }}/*' + - '{{ COMIN_MED_RESTART }}/*' diff --git a/parm/archive/gdaswave.yaml.j2 b/parm/archive/gdaswave.yaml.j2 index cd5809f2652..eeed96b586f 100644 --- a/parm/archive/gdaswave.yaml.j2 +++ b/parm/archive/gdaswave.yaml.j2 @@ -3,24 +3,14 @@ gdaswave: name: "GDASWAVE" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gdaswave.tar" required: - #Save HAFS restart file: - {% set offset_dt = current_cycle | add_to_datetime("+6H" | to_timedelta) %} - - "{{ COMIN_WAVE_RESTART | relpath(ROTDIR) }}/{{ offset_dt | to_fv3time }}.restart.ww3.nc" - + {# offset_dt_fv3 is pre-calculated in Python: +6H (FV3 format: YYYYMMDD.HHMMSS) #} + - "{{ COMIN_WAVE_RESTART }}/{{ offset_dt_fv3 }}.restart.ww3.nc" # TODO explicitly name the wave grid files to archive - {% set WAVE_OUT_GRIDS_list = WAVE_OUT_GRIDS.split(' ') %} - {% for grd in WAVE_OUT_GRIDS_list %} - {% set tmpl_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':RUN, - '${YMD}':cycle_YMD, - '${HH}':cycle_HH, - '${MEMDIR}':'', - '${GRDRESNAME}':grd}) %} - {% set file_path = COM_WAVE_GRID_RES_TMPL | replace_tmpl(tmpl_dict) %} - - "{{ file_path | relpath(ROTDIR) }}/{{ head }}*" + {% for file_path in WAVE_GRID_RES_COM_list %} + - "{{ file_path }}/{{ head }}*" {% endfor %} # Wave point output - - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}bull.tar" - - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}cbull.tar" - - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}spec.tar.gz" + - "{{ COMIN_WAVE_STATION }}/{{ head }}bull.tar" + - "{{ COMIN_WAVE_STATION }}/{{ head }}cbull.tar" + - "{{ COMIN_WAVE_STATION }}/{{ head }}spec.tar.gz" diff --git a/parm/archive/gdaswave_restart.yaml.j2 b/parm/archive/gdaswave_restart.yaml.j2 index d475e8862ca..3b57185ac0e 100644 --- a/parm/archive/gdaswave_restart.yaml.j2 +++ b/parm/archive/gdaswave_restart.yaml.j2 @@ -5,10 +5,5 @@ gdaswave_restart: # TODO explicitly name the wave restart files to archive # NOTE: there are 2 restart files, one is needed to be saved at the same time as restartb is saved # The other which is +6 from the cycle date is needed every cycle for downstream jobs - # If IAU is on, grab the beginning of the window. - {% if DOIAU %} - {% set offset_dt = current_cycle | add_to_datetime("+3H" | to_timedelta) %} - {% else %} - {% set offset_dt = current_cycle | add_to_datetime("+6H" | to_timedelta) %} - {% endif %} - - "{{ COMIN_WAVE_RESTART | relpath(ROTDIR) }}/{{ offset_dt | to_fv3time }}.restart.ww3.nc" + # offset_dt_fv3 is pre-calculated in Python: +3H if DOIAU, +6H otherwise (FV3 format: YYYYMMDD.HHMMSS) + - "{{ COMIN_WAVE_RESTART }}/{{ offset_dt_fv3 }}.restart.ww3.nc" diff --git a/parm/archive/gefs_arcdir.yaml.j2 b/parm/archive/gefs_arcdir.yaml.j2 index 8be329861ec..df265bfec70 100644 --- a/parm/archive/gefs_arcdir.yaml.j2 +++ b/parm/archive/gefs_arcdir.yaml.j2 @@ -1,4 +1,4 @@ -# Variables provided by archive_vars.py: +# Variables provided by archive_vrfy_vars.py: # - cycle_HH, cycle_YMDH, cycle_YMD, head # - VFYARC (archive directory) # - COMIN_ATMOS_ENSSTAT_1p00 (calculated in Python with MEMDIR='ensstat') diff --git a/parm/archive/gfs_arcdir.yaml.j2 b/parm/archive/gfs_arcdir.yaml.j2 index 70356347580..f6623f63cd9 100644 --- a/parm/archive/gfs_arcdir.yaml.j2 +++ b/parm/archive/gfs_arcdir.yaml.j2 @@ -1,4 +1,4 @@ -# Variables provided by archive_vars.py: +# Variables provided by archive_vrfy_vars.py: # - cycle_HH, cycle_YMDH, cycle_YMD, head # - VFYARC, GEFS_ARCH # - All COMIN_* paths diff --git a/parm/archive/gfs_downstream.yaml.j2 b/parm/archive/gfs_downstream.yaml.j2 index 6bc663cb5b4..40ef6d5e6e7 100644 --- a/parm/archive/gfs_downstream.yaml.j2 +++ b/parm/archive/gfs_downstream.yaml.j2 @@ -4,11 +4,11 @@ gfs_downstream: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_downstream.tar" required: {% if DO_GEMPAK and DO_BUFRSND %} - - "{{ COMIN_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.sfc.bufr" - - "{{ COMIN_ATMOS_GEMPAK | relpath(ROTDIR) }}/gfs_{{ cycle_YMDH }}.soundings.bufr" + - "{{ COMIN_ATMOS_GEMPAK }}/gfs_{{ cycle_YMDH }}.sfc.bufr" + - "{{ COMIN_ATMOS_GEMPAK }}/gfs_{{ cycle_YMDH }}.soundings.bufr" {% endif %} {% for i in range(1, NUM_SND_COLLECTIVES) %} - - "{{ COMIN_ATMOS_BUFR | relpath(ROTDIR) }}/gfs_collective{{ i }}.fil" + - "{{ COMIN_ATMOS_BUFR }}/gfs_collective{{ i }}.fil" {% endfor %} - - "{{ COMIN_ATMOS_BUFR | relpath(ROTDIR) }}/bufr.??????.{{ cycle_YMDH }}" - - "{{ COMIN_ATMOS_BUFR | relpath(ROTDIR) }}/gfs.t{{ cycle_HH }}z.soundings.tar.gz" + - "{{ COMIN_ATMOS_BUFR }}/bufr.??????.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_BUFR }}/gfs.t{{ cycle_HH }}z.soundings.tar.gz" diff --git a/parm/archive/gfs_flux.yaml.j2 b/parm/archive/gfs_flux.yaml.j2 index a6f16f0b36c..f477a7ff4c7 100644 --- a/parm/archive/gfs_flux.yaml.j2 +++ b/parm/archive/gfs_flux.yaml.j2 @@ -4,6 +4,6 @@ gfs_flux: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux.tar" required: {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sflux.f{{ '%03d' % fhr }}.grib2" - - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sflux.f{{ '%03d' % fhr }}.grib2.idx" + - "{{ COMIN_ATMOS_MASTER }}/{{ head }}sflux.f{{ '%03d' % fhr }}.grib2" + - "{{ COMIN_ATMOS_MASTER }}/{{ head }}sflux.f{{ '%03d' % fhr }}.grib2.idx" {% endfor %} diff --git a/parm/archive/gfs_flux_1p00.yaml.j2 b/parm/archive/gfs_flux_1p00.yaml.j2 index 697c35a62b0..8150a52bf56 100644 --- a/parm/archive/gfs_flux_1p00.yaml.j2 +++ b/parm/archive/gfs_flux_1p00.yaml.j2 @@ -4,6 +4,6 @@ gfs_flux_1p00: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_flux_1p00.tar" required: {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}.grib2" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}.grib2" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}flux.1p00.f{{ '%03d' % fhr }}.grib2.idx" {% endfor %} diff --git a/parm/archive/gfs_netcdfa.yaml.j2 b/parm/archive/gfs_netcdfa.yaml.j2 index 8745c8d8066..8381c87d2b0 100644 --- a/parm/archive/gfs_netcdfa.yaml.j2 +++ b/parm/archive/gfs_netcdfa.yaml.j2 @@ -5,27 +5,27 @@ gfs_netcdfa: required: {% if DO_JEDIATMVAR %} {% if ATMINC_GRID == 'gaussian' %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}analysis.atm.a006.nc" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}analysis.sfc.a006.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}analysis.atm.a006.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}analysis.sfc.a006.nc" {% else %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}jedi_analysis.atm.a006.nc" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}jedi_analysis.sfc.a006.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}jedi_analysis.atm.a006.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}jedi_analysis.sfc.a006.nc" {% endif %} {% else %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}analysis.atm.a006.nc" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}analysis.sfc.a006.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}analysis.atm.a006.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}analysis.sfc.a006.nc" {% endif %} - {% for iaufhr in IAUFHRS %} + {% for iaufhr in iaufhrs_str %} {% if DO_JEDIATMVAR and not (ATMINC_GRID == 'gaussian') %} {% for itile in range(6) %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}jedi_increment.atm.i{{ "%03d" % iaufhr }}.tile{{ itile+1 }}.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}jedi_increment.atm.i{{ iaufhr }}.tile{{ itile+1 }}.nc" {% endfor %} {% else %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}increment.atm.i{{ "%03d" % iaufhr }}.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}increment.atm.i{{ iaufhr }}.nc" {% endif %} {% endfor %} optional: {% if not DO_JEDIATMVAR %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}increment.dtf.i006.nc" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}increment.done.txt" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}increment.dtf.i006.nc" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}increment.done.txt" {% endif %} diff --git a/parm/archive/gfs_netcdfb.yaml.j2 b/parm/archive/gfs_netcdfb.yaml.j2 index d400363e776..c66c9f3820a 100644 --- a/parm/archive/gfs_netcdfb.yaml.j2 +++ b/parm/archive/gfs_netcdfb.yaml.j2 @@ -4,6 +4,6 @@ gfs_netcdfb: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_netcdfb.tar" required: {% for fhr in range(0, ARCH_GAUSSIAN_FHMAX + ARCH_GAUSSIAN_FHINC, ARCH_GAUSSIAN_FHINC) %} - - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.f{{ '%03d' % fhr }}.nc" - - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfc.f{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_HISTORY }}/{{ head }}atm.f{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_HISTORY }}/{{ head }}sfc.f{{ '%03d' % fhr }}.nc" {% endfor %} diff --git a/parm/archive/gfs_pgrb2b.yaml.j2 b/parm/archive/gfs_pgrb2b.yaml.j2 index d3e930f3080..db7ea934efb 100644 --- a/parm/archive/gfs_pgrb2b.yaml.j2 +++ b/parm/archive/gfs_pgrb2b.yaml.j2 @@ -4,16 +4,16 @@ gfs_pgrb2b: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_pgrb2b.tar" required: {% if MODE == "cycled" %} - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_b.0p25.analysis.grib2" - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_b.0p25.analysis.grib2.idx" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_b.1p00.analysis.grib2" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_b.1p00.analysis.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_b.0p25.analysis.grib2" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_b.0p25.analysis.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_b.1p00.analysis.grib2" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_b.1p00.analysis.grib2.idx" {% endif %} {% if ARCH_GAUSSIAN %} {% for fhr in range(0, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_b.0p25.f{{ '%03d' % fhr }}.grib2" - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_b.0p25.f{{ '%03d' % fhr }}.grib2.idx" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_b.1p00.f{{ '%03d' % fhr }}.grib2" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_b.1p00.f{{ '%03d' % fhr }}.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_b.0p25.f{{ '%03d' % fhr }}.grib2" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_b.0p25.f{{ '%03d' % fhr }}.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_b.1p00.f{{ '%03d' % fhr }}.grib2" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_b.1p00.f{{ '%03d' % fhr }}.grib2.idx" {% endfor %} {% endif %} diff --git a/parm/archive/gfs_restarta.yaml.j2 b/parm/archive/gfs_restarta.yaml.j2 index 8f6a0b6c10b..5279e0f0882 100644 --- a/parm/archive/gfs_restarta.yaml.j2 +++ b/parm/archive/gfs_restarta.yaml.j2 @@ -4,20 +4,14 @@ gfs_restarta: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfs_restarta.tar" required: {% if MODE == "cycled" %} - {% if DOHYBVAR and DOIAU %} - {% set anl_offset = "-3H" %} - {% else %} - {% set anl_offset = "0H" %} - {% endif %} - {% set anl_timedelta = anl_offset | to_timedelta %} - {% set anl_time = current_cycle | add_to_datetime(anl_timedelta) %} + {# anl_time_YMD and anl_time_HH are pre-calculated in Python: -3H if DOHYBVAR and DOIAU, 0H otherwise #} {% for i_tile in range(1, 7) %} - - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.sfcanl_data.tile{{ i_tile }}.nc" + - "{{ COMIN_ATMOS_RESTART }}/{{ anl_time_YMD }}.{{ anl_time_HH }}0000.sfcanl_data.tile{{ i_tile }}.nc" {% endfor %} {% elif MODE == "forecast-only" %} - - "{{ COMIN_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_ctrl.nc" + - "{{ COMIN_ATMOS_INPUT }}/gfs_ctrl.nc" {% for i_tile in range(1, 7) %} - - "{{ COMIN_ATMOS_INPUT | relpath(ROTDIR) }}/gfs_data.tile{{ i_tile }}.nc" - - "{{ COMIN_ATMOS_INPUT | relpath(ROTDIR) }}/sfc_data.tile{{ i_tile }}.nc" + - "{{ COMIN_ATMOS_INPUT }}/gfs_data.tile{{ i_tile }}.nc" + - "{{ COMIN_ATMOS_INPUT }}/sfc_data.tile{{ i_tile }}.nc" {% endfor %} {% endif %} diff --git a/parm/archive/gfsa.yaml.j2 b/parm/archive/gfsa.yaml.j2 index b64687b5b89..72581d01902 100644 --- a/parm/archive/gfsa.yaml.j2 +++ b/parm/archive/gfsa.yaml.j2 @@ -12,73 +12,73 @@ gfsa: {% endfor %} # UFS configuration - - "{{ COMIN_CONF | relpath(ROTDIR) }}/ufs.input.nml" + - "{{ COMIN_CONF }}/ufs.input.nml" {% if DO_OCN %} - - "{{ COMIN_CONF | relpath(ROTDIR) }}/MOM_parameter_doc.all" + - "{{ COMIN_CONF }}/MOM_parameter_doc.all" {% endif %} {% if MODE == "cycled" %} # Analysis GRIB2 (gridded) data - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_a.0p25.analysis.grib2" - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_a.0p25.analysis.grib2.idx" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_a.1p00.analysis.grib2" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_a.1p00.analysis.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_a.0p25.analysis.grib2" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_a.0p25.analysis.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_a.1p00.analysis.grib2" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_a.1p00.analysis.grib2.idx" {% if DO_VMINMON %} # Minimization monitor - - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.costs.txt" - - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.cost_terms.txt" - - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.gnorms.ieee_d" - - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/{{ cycle_YMDH }}.reduction.ieee_d" - - "{{ COMIN_ATMOS_MINMON | relpath(ROTDIR) }}/gnorm_data.txt" + - "{{ COMIN_ATMOS_MINMON }}/{{ cycle_YMDH }}.costs.txt" + - "{{ COMIN_ATMOS_MINMON }}/{{ cycle_YMDH }}.cost_terms.txt" + - "{{ COMIN_ATMOS_MINMON }}/{{ cycle_YMDH }}.gnorms.ieee_d" + - "{{ COMIN_ATMOS_MINMON }}/{{ cycle_YMDH }}.reduction.ieee_d" + - "{{ COMIN_ATMOS_MINMON }}/gnorm_data.txt" {% endif %} # State data {% if DO_JEDIATMVAR %} - - "{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}anlvar.atm.yaml" - - "{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}anlvar.fv3.atm.yaml" - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmos_analysis.ioda_hofx.tar.gz" + - "{{ COMIN_CONF }}/{{ head }}anlvar.atm.yaml" + - "{{ COMIN_CONF }}/{{ head }}anlvar.fv3.atm.yaml" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}atmos_analysis.ioda_hofx.tar.gz" {% else %} - - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat.txt" + - "{{ COMIN_ATMOS_ANALYSIS }}/{{ head }}gsistat.txt" {% endif %} {% if DO_AERO_ANL %} - - "{{ COMIN_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aero_analysis.ioda_hofx.tar.gz" + - "{{ COMIN_CHEM_ANALYSIS }}/{{ head }}aero_analysis.ioda_hofx.tar.gz" {% endif %} {% if DO_JEDIOCNVAR %} - - "{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}marine_analysis.ioda_hofx.tar.gz" + - "{{ COMIN_OCEAN_ANALYSIS }}/{{ head }}marine_analysis.ioda_hofx.tar.gz" {% endif %} {% if DO_PREP_OBS_AERO %} - - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs" - - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs" + - "{{ COMIN_OBS }}/{{ head }}aeroobs" + - "{{ COMIN_OBS }}/{{ head }}aerorawobs" {% endif %} # Snow analysis data {% if DO_JEDISNOWDA %} {% for itile in range(1,7) %} - - "{{ COMIN_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ current_cycle | to_fv3time }}.snow_increment.sfc_data.tile{{ itile }}.nc" + - "{{ COMIN_SNOW_ANALYSIS }}/{{ cycle_fv3time }}.snow_increment.sfc_data.tile{{ itile }}.nc" {% endfor %} - - "{{ COMIN_SNOW_ANALYSIS | relpath(ROTDIR) }}/{{ head }}snow_analysis.ioda_hofx.tar.gz" - - "{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}snowanlvar.yaml" + - "{{ COMIN_SNOW_ANALYSIS }}/{{ head }}snow_analysis.ioda_hofx.tar.gz" + - "{{ COMIN_CONF }}/{{ head }}snowanlvar.yaml" {% endif %} # BUFR inputs - - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}nsstbufr" - - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr" - - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}prepbufr.acft_profiles" + - "{{ COMIN_OBS }}/{{ head }}nsstbufr" + - "{{ COMIN_OBS }}/{{ head }}prepbufr" + - "{{ COMIN_OBS }}/{{ head }}prepbufr.acft_profiles" {% endif %} # Full cycle # Forecast GRIB2 products {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_a.0p25.f{{ '%03d' % fhr }}.grib2" - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_a.0p25.f{{ '%03d' % fhr }}.grib2.idx" - - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}log.f{{ '%03d' % fhr }}.txt" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_a.0p25.f{{ '%03d' % fhr }}.grib2" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_a.0p25.f{{ '%03d' % fhr }}.grib2.idx" + - "{{ COMIN_ATMOS_HISTORY }}/{{ head }}log.f{{ '%03d' % fhr }}.txt" {% endfor %} optional: # Cyclone tracking data; only present if there's something to track. - - "{{ COMIN_ATMOS_TRACK | relpath(ROTDIR) }}/avno.t{{ cycle_HH }}z.cyclone.trackatcfunix" - - "{{ COMIN_ATMOS_TRACK | relpath(ROTDIR) }}/avnop.t{{ cycle_HH }}z.cyclone.trackatcfunix" - - "{{ COMIN_ATMOS_GENESIS | relpath(ROTDIR) }}/trak.gfso.atcfunix.{{ cycle_YMDH }}" - - "{{ COMIN_ATMOS_GENESIS | relpath(ROTDIR) }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}" - - "{{ COMIN_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}" - - "{{ COMIN_ATMOS_GENESIS | relpath(ROTDIR) }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_TRACK }}/avno.t{{ cycle_HH }}z.cyclone.trackatcfunix" + - "{{ COMIN_ATMOS_TRACK }}/avnop.t{{ cycle_HH }}z.cyclone.trackatcfunix" + - "{{ COMIN_ATMOS_GENESIS }}/trak.gfso.atcfunix.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_GENESIS }}/trak.gfso.atcfunix.altg.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_GENESIS }}/storms.gfso.atcf_gen.{{ cycle_YMDH }}" + - "{{ COMIN_ATMOS_GENESIS }}/storms.gfso.atcf_gen.altg.{{ cycle_YMDH }}" diff --git a/parm/archive/gfsb.yaml.j2 b/parm/archive/gfsb.yaml.j2 index 22711d8222c..4b5bfd3e160 100644 --- a/parm/archive/gfsb.yaml.j2 +++ b/parm/archive/gfsb.yaml.j2 @@ -5,16 +5,16 @@ gfsb: required: {% if MODE == "cycled" %} # GRIB2 (subsampled) analysis data - - "{{ COMIN_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pres_a.0p50.analysis.grib2" - - "{{ COMIN_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pres_a.0p50.analysis.grib2.idx" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_a.1p00.analysis.grib2" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_a.1p00.analysis.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_0p50 }}/{{ head }}pres_a.0p50.analysis.grib2" + - "{{ COMIN_ATMOS_GRIB_0p50 }}/{{ head }}pres_a.0p50.analysis.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_a.1p00.analysis.grib2" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_a.1p00.analysis.grib2.idx" {% endif %} # GRIB2 orecast data {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COMIN_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pres_a.0p50.f{{ '%03d' % fhr }}.grib2" - - "{{ COMIN_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pres_a.0p50.f{{ '%03d' % fhr }}.grib2.idx" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_a.1p00.f{{ '%03d' % fhr }}.grib2" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_a.1p00.f{{ '%03d' % fhr }}.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_0p50 }}/{{ head }}pres_a.0p50.f{{ '%03d' % fhr }}.grib2" + - "{{ COMIN_ATMOS_GRIB_0p50 }}/{{ head }}pres_a.0p50.f{{ '%03d' % fhr }}.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_a.1p00.f{{ '%03d' % fhr }}.grib2" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_a.1p00.f{{ '%03d' % fhr }}.grib2.idx" {% endfor %} diff --git a/parm/archive/gfsocean_analysis.yaml.j2 b/parm/archive/gfsocean_analysis.yaml.j2 index d8a84c07c4a..fc58416636f 100644 --- a/parm/archive/gfsocean_analysis.yaml.j2 +++ b/parm/archive/gfsocean_analysis.yaml.j2 @@ -3,37 +3,32 @@ gfsocean_analysis: name: "GFSOCEAN_ANALYSIS" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfsocean_analysis.tar" required: - {% if DOIAU %} - {% set anl_delta = "-3H" | to_timedelta %} - {% else %} - {% set anl_delta = "0H" | to_timedelta %} - {% endif %} - {% set anl_time = current_cycle | add_to_datetime(anl_delta) %} + {# anl_time_YMD and anl_time_HH are pre-calculated in Python: -3H if DOIAU, 0H otherwise #} # analysis and analysis increments - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}jedi_analysis.a006.nc' - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}jedi_increment.i006.nc' - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/{{ head }}mom6_increment.i006.nc' - - '{{ COMIN_ICE_ANALYSIS | relpath(ROTDIR) }}/{{ anl_time | to_YMD }}.{{ anl_time | strftime("%H") }}0000.analysis.cice_model.res.nc' - - '{{ COMIN_ICE_ANALYSIS | relpath(ROTDIR) }}/{{ head }}jedi_analysis.a006.nc' - - '{{ COMIN_ICE_ANALYSIS | relpath(ROTDIR) }}/{{ head }}jedi_increment.i006.nc' - - '{{ COMIN_ICE_ANALYSIS | relpath(ROTDIR) }}/{{ head }}pp_increment.i006.nc' + - '{{ COMIN_OCEAN_ANALYSIS }}/{{ head }}jedi_analysis.a006.nc' + - '{{ COMIN_OCEAN_ANALYSIS }}/{{ head }}jedi_increment.i006.nc' + - '{{ COMIN_OCEAN_ANALYSIS }}/{{ head }}mom6_increment.i006.nc' + - '{{ COMIN_ICE_ANALYSIS }}/{{ anl_time_YMD }}.{{ anl_time_HH }}0000.analysis.cice_model.res.nc' + - '{{ COMIN_ICE_ANALYSIS }}/{{ head }}jedi_analysis.a006.nc' + - '{{ COMIN_ICE_ANALYSIS }}/{{ head }}jedi_increment.i006.nc' + - '{{ COMIN_ICE_ANALYSIS }}/{{ head }}pp_increment.i006.nc' # static background error - - '{{ COMIN_OCEAN_BMATRIX | relpath(ROTDIR) }}/{{ head }}bkgerr_parametric_stddev.nc' - - '{{ COMIN_ICE_BMATRIX | relpath(ROTDIR) }}/{{ head }}bkgerr_parametric_stddev.nc' + - '{{ COMIN_OCEAN_BMATRIX }}/{{ head }}bkgerr_parametric_stddev.nc' + - '{{ COMIN_ICE_BMATRIX }}/{{ head }}bkgerr_parametric_stddev.nc' # runtime configs - - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_setcorscales.yaml' - - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_chgres.yaml' - - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_diagb.yaml' - - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}gridgen.yaml' - - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_vtscales.yaml' - - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}bmat_fields_metadata.yaml' - - '{{ COMIN_CONF | relpath(ROTDIR) }}/var.yaml' + - '{{ COMIN_CONF }}/{{ head }}soca_setcorscales.yaml' + - '{{ COMIN_CONF }}/{{ head }}soca_chgres.yaml' + - '{{ COMIN_CONF }}/{{ head }}soca_diagb.yaml' + - '{{ COMIN_CONF }}/{{ head }}gridgen.yaml' + - '{{ COMIN_CONF }}/{{ head }}soca_vtscales.yaml' + - '{{ COMIN_CONF }}/{{ head }}bmat_fields_metadata.yaml' + - '{{ COMIN_CONF }}/var.yaml' # soca_parameters_diffusion_vt, and soca_parameters_diffusion_hz when present - - '{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}soca_parameters_diffusion_??.yaml' - - '{{ COMIN_CONF | relpath(ROTDIR) }}/soca_incpostproc.yaml' + - '{{ COMIN_CONF }}/{{ head }}soca_parameters_diffusion_??.yaml' + - '{{ COMIN_CONF }}/soca_incpostproc.yaml' optional: # obs space diags - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags/{{ head }}ocn.*.stats.csv' - - '{{ COMIN_OCEAN_ANALYSIS | relpath(ROTDIR) }}/diags/*.nc' + - '{{ COMIN_OCEAN_ANALYSIS }}/diags/{{ head }}ocn.*.stats.csv' + - '{{ COMIN_OCEAN_ANALYSIS }}/diags/*.nc' diff --git a/parm/archive/gfswave.yaml.j2 b/parm/archive/gfswave.yaml.j2 index 1ab3efebc76..92e548a37ac 100644 --- a/parm/archive/gfswave.yaml.j2 +++ b/parm/archive/gfswave.yaml.j2 @@ -5,38 +5,26 @@ gfswave: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gfswave.tar" required: # Wave GRIB2 regional forecast products - {% set WAVE_OUT_GRIDS_list = WAVE_OUT_GRIDS.split(' ') %} - {% for grd in WAVE_OUT_GRIDS_list %} - {% set tmpl_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':RUN, - '${YMD}':cycle_YMD, - '${HH}':cycle_HH, - '${MEMDIR}':'', - '${GRDRESNAME}':grd}) %} - {% set file_path = COM_WAVE_GRID_RES_TMPL | replace_tmpl(tmpl_dict) %} - - {% for fh in range(0, FHMAX_HF_WAV + FHOUT_HF_WAV, FHOUT_HF_WAV) %} - # NOTE This is as explicit as possible without major logic to parse wavepostGRD. - # Matches files of the form "gfs.tCCz...fHHH.grib2". - - "{{ file_path | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" - - "{{ file_path | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" - {% endfor %} - - # Global wave GRIB2 forecast products - {% for fh in range(FHMAX_HF_WAV + FHOUT_WAV_GFS, FHMAX_WAV_GFS + FHOUT_WAV_GFS, FHOUT_WAV_GFS) %} - - "{{ file_path | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" - - "{{ file_path | relpath(ROTDIR) }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" + {% for file_path in WAVE_GRID_RES_COM_list %} + {% for fh in range(0, FHMAX_HF_WAV + FHOUT_HF_WAV, FHOUT_HF_WAV) %} + - "{{ file_path }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" + - "{{ file_path }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" + {% endfor %} + # Global wave GRIB2 forecast products + {% for fh in range(FHMAX_HF_WAV + FHOUT_WAV_GFS, FHMAX_WAV_GFS + FHOUT_WAV_GFS, FHOUT_WAV_GFS) %} + - "{{ file_path }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2" + - "{{ file_path }}/{{ head }}*.*.f{{ '%03d' % fh }}.grib2.idx" + {% endfor %} {% endfor %} # Wave point output - - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}bull.tar" - - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}cbull.tar" - - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}spec.tar.gz" + - "{{ COMIN_WAVE_STATION }}/{{ head }}bull.tar" + - "{{ COMIN_WAVE_STATION }}/{{ head }}cbull.tar" + - "{{ COMIN_WAVE_STATION }}/{{ head }}spec.tar.gz" # Wave boundary point output {% if DOBNDPNT_WAVE %} - - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibpbull.tar" - - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibpcbull.tar" - - "{{ COMIN_WAVE_STATION | relpath(ROTDIR) }}/{{ head }}ibp.tar" + - "{{ COMIN_WAVE_STATION }}/{{ head }}ibpbull.tar" + - "{{ COMIN_WAVE_STATION }}/{{ head }}ibpcbull.tar" + - "{{ COMIN_WAVE_STATION }}/{{ head }}ibp.tar" {% endif %} - {% endfor %} diff --git a/parm/archive/ice_6hravg.yaml.j2 b/parm/archive/ice_6hravg.yaml.j2 index 1197abafb8c..6396535df84 100644 --- a/parm/archive/ice_6hravg.yaml.j2 +++ b/parm/archive/ice_6hravg.yaml.j2 @@ -2,7 +2,7 @@ ice_6hravg: name: "ICE_6HRAVG" target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_6hravg.tar" required: - - "{{ COMIN_ICE_HISTORY | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.ic.nc" + - "{{ COMIN_ICE_HISTORY }}/{{ RUN }}.t{{ cycle_HH }}z.ic.nc" {% for fhr in range(6, FHMAX_GFS + 6, 6) %} - - "{{ COMIN_ICE_HISTORY | relpath(ROTDIR) }}/{{ RUN }}.t{{ cycle_HH }}z.6hr_avg.f{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ICE_HISTORY }}/{{ RUN }}.t{{ cycle_HH }}z.6hr_avg.f{{ '%03d' % fhr }}.nc" {% endfor %} diff --git a/parm/archive/ice_grib2.yaml.j2 b/parm/archive/ice_grib2.yaml.j2 index f191165d794..407c1615e6b 100644 --- a/parm/archive/ice_grib2.yaml.j2 +++ b/parm/archive/ice_grib2.yaml.j2 @@ -6,13 +6,13 @@ ice_grib2: {% for fhr in range(FHOUT_ICE_GFS, FHMAX_GFS + FHOUT_ICE_GFS, FHOUT_ICE_GFS) %} {% set fhr3 = '%03d' % fhr %} {% if ICERES == 500 %} - - "{{ COMIN_ICE_GRIB | relpath(ROTDIR) }}/5p00/{{ RUN }}.t{{ cycle_HH }}z.5p00.f{{ fhr3 }}.grib2" - - "{{ COMIN_ICE_GRIB | relpath(ROTDIR) }}/5p00/{{ RUN }}.t{{ cycle_HH }}z.5p00.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_ICE_GRIB }}/5p00/{{ RUN }}.t{{ cycle_HH }}z.5p00.f{{ fhr3 }}.grib2" + - "{{ COMIN_ICE_GRIB }}/5p00/{{ RUN }}.t{{ cycle_HH }}z.5p00.f{{ fhr3 }}.grib2.idx" {% elif ICERES == 100 %} - - "{{ COMIN_ICE_GRIB | relpath(ROTDIR) }}/1p00/{{ RUN }}.t{{ cycle_HH }}z.1p00.f{{ fhr3 }}.grib2" - - "{{ COMIN_ICE_GRIB | relpath(ROTDIR) }}/1p00/{{ RUN }}.t{{ cycle_HH }}z.1p00.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_ICE_GRIB }}/1p00/{{ RUN }}.t{{ cycle_HH }}z.1p00.f{{ fhr3 }}.grib2" + - "{{ COMIN_ICE_GRIB }}/1p00/{{ RUN }}.t{{ cycle_HH }}z.1p00.f{{ fhr3 }}.grib2.idx" {% elif ICERES == 25 or ICERES == "025" %} - - "{{ COMIN_ICE_GRIB | relpath(ROTDIR) }}/0p25/{{ RUN }}.t{{ cycle_HH }}z.0p25.f{{ fhr3 }}.grib2" - - "{{ COMIN_ICE_GRIB | relpath(ROTDIR) }}/0p25/{{ RUN }}.t{{ cycle_HH }}z.0p25.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_ICE_GRIB }}/0p25/{{ RUN }}.t{{ cycle_HH }}z.0p25.f{{ fhr3 }}.grib2" + - "{{ COMIN_ICE_GRIB }}/0p25/{{ RUN }}.t{{ cycle_HH }}z.0p25.f{{ fhr3 }}.grib2.idx" {% endif %} {% endfor %} diff --git a/parm/archive/ice_native.yaml.j2 b/parm/archive/ice_native.yaml.j2 index 546650a0225..afe6d65bed8 100644 --- a/parm/archive/ice_native.yaml.j2 +++ b/parm/archive/ice_native.yaml.j2 @@ -3,5 +3,5 @@ ice_native: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ice_native_subset.tar" required: {% for fhr in range(FHOUT_ICE_GFS, FHMAX_GFS + FHOUT_ICE_GFS, FHOUT_ICE_GFS) %} - - "{{ COMIN_ICE_NETCDF | relpath(ROTDIR) }}/native/{{ RUN }}.t{{ cycle_HH }}z.native.f{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ICE_NETCDF }}/native/{{ RUN }}.t{{ cycle_HH }}z.native.f{{ '%03d' % fhr }}.nc" {% endfor %} diff --git a/parm/archive/master_enkf.yaml.j2 b/parm/archive/master_enkf.yaml.j2 index 8828ff0a2c9..ff886cd844d 100644 --- a/parm/archive/master_enkf.yaml.j2 +++ b/parm/archive/master_enkf.yaml.j2 @@ -1,7 +1,4 @@ # Set variables/lists needed to parse the enkf templates -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set cycle_YMDH = current_cycle | to_YMDH %} {% set head = RUN + ".t" + cycle_HH + "z." %} # Determine which data to archive @@ -13,101 +10,26 @@ datasets: {% endfilter %} {% else %} - # Archive individual member data - # First, construct individual member directories from templates - # COMIN_ATMOS_ANALYSIS_MEM, COMIN_ATMOS_HISTORY_MEM, and COMIN_ATMOS_RESTART_MEM - - # Declare to-be-filled lists of member COM directories - {% set COMIN_ATMOS_ANALYSIS_MEM_list = [] %} - {% set COMIN_ATMOS_RESTART_MEM_list = [] %} - {% set COMIN_ATMOS_HISTORY_MEM_list = [] %} - {% set COMIN_OCEAN_ANALYSIS_MEM_list = [] %} - {% set COMIN_OCEAN_LETKF_MEM_list = [] %} - {% set COMIN_OCEAN_RESTART_MEM_list = [] %} - {% set COMIN_OCEAN_HISTORY_MEM_list = [] %} - {% set COMIN_ICE_ANALYSIS_MEM_list = [] %} - {% set COMIN_ICE_LETKF_MEM_list = [] %} - {% set COMIN_ICE_RESTART_MEM_list = [] %} - {% set COMIN_ICE_HISTORY_MEM_list = [] %} - {% set COMIN_MED_RESTART_MEM_list = [] %} - - # Determine which ensemble members belong to this group - {% set first_group_mem = (ENSGRP - 1) * NMEM_EARCGRP + 1 %} - {% set last_group_mem = [ ENSGRP * NMEM_EARCGRP, nmem_ens ] | min %} - - # Construct member COM directories for the group - {% for mem in range(first_group_mem, last_group_mem + 1) %} - - # Declare a dict of search and replace terms to run on each template - {% set mem_char = 'mem%03d' | format(mem) %} - {% set tmpl_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':RUN, - '${YMD}':cycle_YMD, - '${HH}':cycle_HH, - '${MEMDIR}': mem_char }) %} - - {% set COMIN_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) %} - {% set COMIN_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) %} - {% set COMIN_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) %} - {% set COMIN_OCEAN_ANALYSIS_MEM = COM_OCEAN_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) %} - {% set COMIN_OCEAN_LETKF_MEM = COM_OCEAN_LETKF_TMPL | replace_tmpl(tmpl_dict) %} - {% set COMIN_OCEAN_HISTORY_MEM = COM_OCEAN_HISTORY_TMPL | replace_tmpl(tmpl_dict) %} - {% set COMIN_OCEAN_RESTART_MEM = COM_OCEAN_RESTART_TMPL | replace_tmpl(tmpl_dict) %} - {% set COMIN_ICE_ANALYSIS_MEM = COM_ICE_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) %} - {% set COMIN_ICE_LETKF_MEM = COM_ICE_LETKF_TMPL | replace_tmpl(tmpl_dict) %} - {% set COMIN_ICE_HISTORY_MEM = COM_ICE_HISTORY_TMPL | replace_tmpl(tmpl_dict) %} - {% set COMIN_ICE_RESTART_MEM = COM_ICE_RESTART_TMPL | replace_tmpl(tmpl_dict) %} - {% set COMIN_MED_RESTART_MEM = COM_MED_RESTART_TMPL | replace_tmpl(tmpl_dict) %} - - # Append the member COM directories - {% do COMIN_ATMOS_ANALYSIS_MEM_list.append(COMIN_ATMOS_ANALYSIS_MEM)%} - {% do COMIN_ATMOS_HISTORY_MEM_list.append(COMIN_ATMOS_HISTORY_MEM)%} - {% do COMIN_ATMOS_RESTART_MEM_list.append(COMIN_ATMOS_RESTART_MEM)%} - {% do COMIN_OCEAN_ANALYSIS_MEM_list.append(COMIN_OCEAN_ANALYSIS_MEM)%} - {% do COMIN_OCEAN_LETKF_MEM_list.append(COMIN_OCEAN_LETKF_MEM)%} - {% do COMIN_OCEAN_HISTORY_MEM_list.append(COMIN_OCEAN_HISTORY_MEM)%} - {% do COMIN_OCEAN_RESTART_MEM_list.append(COMIN_OCEAN_RESTART_MEM)%} - {% do COMIN_ICE_ANALYSIS_MEM_list.append(COMIN_ICE_ANALYSIS_MEM)%} - {% do COMIN_ICE_LETKF_MEM_list.append(COMIN_ICE_LETKF_MEM)%} - {% do COMIN_ICE_HISTORY_MEM_list.append(COMIN_ICE_HISTORY_MEM)%} - {% do COMIN_ICE_RESTART_MEM_list.append(COMIN_ICE_RESTART_MEM)%} - {% do COMIN_MED_RESTART_MEM_list.append(COMIN_MED_RESTART_MEM)%} - - {% endfor %} - - {% set tmpl_dict = ({ '${ROTDIR}':ROTDIR, - '${RUN}':RUN, - '${YMD}':cycle_YMD, - '${HH}':cycle_HH, - '${MEMDIR}': "ensstat"}) %} - {% set COMIN_CONF = COM_CONF_TMPL | replace_tmpl(tmpl_dict) %} - # Archive member data {% filter indent(width=4) %} {% include "enkf_grp.yaml.j2" %} {% endfilter %} - # Determine if restarts should be saved - {% set save_warm_start_forecast, save_warm_start_cycled = ( False, False ) %} - # Save the increments and restarts every ARCH_WARMICFREQ days # The ensemble increments (group a) should be saved on the ARCH_CYC # The forecasts are only run for enkfgdas RUNs, so skip for enkfgfs - {% if (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 and is_gdas %} - {% if ARCH_CYC == cycle_HH | int %} - {% filter indent(width=4) %} + # Note: All archive timing variables (archive_*, save_warm_start_*) are provided by Python via ArchiveTarVars._get_cycle_vars() + {% if archive_increments %} + {% filter indent(width=4) %} {% include "enkf_restarta_grp.yaml.j2" %} - {% endfilter %} - {% endif %} + {% endfilter %} {% endif %} # The ensemble ICs (group b) are restarts and always lag increments by assim_freq - {% set ics_offset = (assim_freq | string + "H") | to_timedelta %} - {% if (current_cycle | add_to_datetime(ics_offset) - SDATE).days % ARCH_WARMICFREQ == 0 and is_gdas %} - {% if (ARCH_CYC - assim_freq) % 24 == cycle_HH | int %} - {% filter indent(width=4) %} + # Note: archive_ics and archive_ics_at_cyc are provided by Python via ArchiveTarVars._get_cycle_vars() + {% if archive_ics %} + {% filter indent(width=4) %} {% include "enkf_restartb_grp.yaml.j2" %} - {% endfilter %} - {% endif %} + {% endfilter %} {% endif %} # End of individual member archiving diff --git a/parm/archive/master_enkfgdas.yaml.j2 b/parm/archive/master_enkfgdas.yaml.j2 deleted file mode 100644 index 7eeee4c0727..00000000000 --- a/parm/archive/master_enkfgdas.yaml.j2 +++ /dev/null @@ -1,8 +0,0 @@ -# Set variables specific to gdasenkf runs then parse the master_enkf template -{% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF, FHOUT_ENKF) %} -{% set do_calc_increment = DO_CALC_INCREMENT %} -{% set nmem_ens = NMEM_ENS %} -{% set restart_interval = restart_interval_enkfgdas %} -{% set is_gdas = True %} -{% set is_gfs = False %} -{% include "master_enkf.yaml.j2" %} diff --git a/parm/archive/master_enkfgfs.yaml.j2 b/parm/archive/master_enkfgfs.yaml.j2 deleted file mode 100644 index e944cdc6374..00000000000 --- a/parm/archive/master_enkfgfs.yaml.j2 +++ /dev/null @@ -1,8 +0,0 @@ -# Set variables specific to gfsenkf runs then parse the master_enkf template -{% set (fhmin, fhmax, fhout) = (FHMIN_ENKF, FHMAX_ENKF_GFS, FHOUT_ENKF_GFS) %} -{% set do_calc_increment = DO_CALC_INCREMENT_ENKF_GFS %} -{% set nmem_ens = NMEM_ENS_GFS %} -{% set restart_interval = restart_interval_enkfgfs %} -{% set is_gdas = False %} -{% set is_gfs = True %} -{% include "master_enkf.yaml.j2" %} diff --git a/parm/archive/master_gcafs.yaml.j2 b/parm/archive/master_gcafs.yaml.j2 index 39b919a43cc..a6fd47520af 100644 --- a/parm/archive/master_gcafs.yaml.j2 +++ b/parm/archive/master_gcafs.yaml.j2 @@ -1,8 +1,3 @@ -# Set variables/lists needed to parse the enkf templates -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set cycle_YMDH = current_cycle | to_YMDH %} - # Determine which data to archive datasets: gcafs: @@ -11,34 +6,34 @@ datasets: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/gcafs.tar" required: {% for fhr in range(0, ARCH_GAUSSIAN_FHMAX + ARCH_GAUSSIAN_FHINC, ARCH_GAUSSIAN_FHINC) %} - - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.f{{ '%03d' % fhr }}.nc" - - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfc.f{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_HISTORY }}/{{ head }}atm.f{{ '%03d' % fhr }}.nc" + - "{{ COMIN_ATMOS_HISTORY }}/{{ head }}sfc.f{{ '%03d' % fhr }}.nc" {% endfor %} {% if ARCH_GAUSSIAN %} {% for fhr in range(0, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_b.0p25.f{{ '%03d' % fhr }}.grib2" - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_b.0p25.f{{ '%03d' % fhr }}.grib2.idx" - - "{{ COMIN_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pres_b.0p50.f{{ '%03d' % fhr }}.grib2" - - "{{ COMIN_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pres_b.0p50.f{{ '%03d' % fhr }}.grib2.idx" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_b.1p00.f{{ '%03d' % fhr }}.grib2" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_b.1p00.f{{ '%03d' % fhr }}.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_b.0p25.f{{ '%03d' % fhr }}.grib2" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_b.0p25.f{{ '%03d' % fhr }}.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_0p50 }}/{{ head }}pres_b.0p50.f{{ '%03d' % fhr }}.grib2" + - "{{ COMIN_ATMOS_GRIB_0p50 }}/{{ head }}pres_b.0p50.f{{ '%03d' % fhr }}.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_b.1p00.f{{ '%03d' % fhr }}.grib2" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_b.1p00.f{{ '%03d' % fhr }}.grib2.idx" {% endfor %} {% endif %} - "{{ COMIN_CONF | relpath(ROTDIR) }}/ufs.input.nml" # Forecast GRIB2 products {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_a.0p25.f{{ '%03d' % fhr }}.grib2" - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_a.0p25.f{{ '%03d' % fhr }}.grib2.idx" - - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}log.f{{ '%03d' % fhr }}.txt" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_a.0p25.f{{ '%03d' % fhr }}.grib2" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_a.0p25.f{{ '%03d' % fhr }}.grib2.idx" + - "{{ COMIN_ATMOS_HISTORY }}/{{ head }}log.f{{ '%03d' % fhr }}.txt" {% endfor %} # GRIB2 forecast data {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %} - - "{{ COMIN_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pres_a.0p50.f{{ '%03d' % fhr }}.grib2" - - "{{ COMIN_ATMOS_GRIB_0p50 | relpath(ROTDIR) }}/{{ head }}pres_a.0p50.f{{ '%03d' % fhr }}.grib2.idx" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_a.1p00.f{{ '%03d' % fhr }}.grib2" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_a.1p00.f{{ '%03d' % fhr }}.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_0p50 }}/{{ head }}pres_a.0p50.f{{ '%03d' % fhr }}.grib2" + - "{{ COMIN_ATMOS_GRIB_0p50 }}/{{ head }}pres_a.0p50.f{{ '%03d' % fhr }}.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_a.1p00.f{{ '%03d' % fhr }}.grib2" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_a.1p00.f{{ '%03d' % fhr }}.grib2.idx" {% endfor %} # Archive the EXPDIR if requested {% if archive_expdir %} diff --git a/parm/archive/master_gcdas.yaml.j2 b/parm/archive/master_gcdas.yaml.j2 index ed694b9c4c3..0943b145875 100644 --- a/parm/archive/master_gcdas.yaml.j2 +++ b/parm/archive/master_gcdas.yaml.j2 @@ -1,6 +1,3 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set cycle_YMDH = current_cycle | to_YMDH %} {% set head = "gcdas.t" + cycle_HH + "z." %} datasets: @@ -14,10 +11,10 @@ datasets: - "logs/{{ cycle_YMDH }}/{{ RUN }}_aeroanlinit.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}_aeroanlvar.log" - "logs/{{ cycle_YMDH }}/{{ RUN }}_aeroanlfinal.log" - - "{{ COMIN_CHEM_ANALYSIS | relpath(ROTDIR) }}/{{ head }}aero_analysis.ioda_hofx.tar.gz" - - "{{ COMIN_CONF | relpath(ROTDIR) }}/{{ head }}aerovar.yaml" + - "{{ COMIN_CHEM_ANALYSIS }}/{{ head }}aero_analysis.ioda_hofx.tar.gz" + - "{{ COMIN_CONF }}/{{ head }}aerovar.yaml" {% for itile in range(1,7) %} - - "{{ COMIN_CHEM_ANALYSIS | relpath(ROTDIR) }}/aeroinc.{{ cycle_YMD }}.{{ cycle_HH }}0000.fv_tracer.res.tile{{ itile }}.nc" + - "{{ COMIN_CHEM_ANALYSIS }}/aeroinc.{{ cycle_YMD }}.{{ cycle_HH }}0000.fv_tracer.res.tile{{ itile }}.nc" {% endfor %} # Analysis Master GRIB2 data - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}master.analysis.grib2" @@ -43,8 +40,8 @@ datasets: - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}analysis.sfc.a006.nc" {% endif %} {% if DO_PREP_OBS_AERO %} - - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}aeroobs" - - "{{ COMIN_OBS | relpath(ROTDIR) }}/{{ head }}aerorawobs" + - "{{ COMIN_OBS }}/{{ head }}aeroobs" + - "{{ COMIN_OBS }}/{{ head }}aerorawobs" {% endif %} # Forecast and post logs - "logs/{{ cycle_YMDH }}/{{ RUN }}_fcst_seg0.log" @@ -56,36 +53,32 @@ datasets: - "logs/{{ cycle_YMDH }}/{{ RUN }}_atmos_upp_f{{ fhr3 }}.log" {% endif %} ## not WRITE_DOPOST # Forecast GRIB2 data - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_a.0p25.f{{ fhr3 }}.grib2" - - "{{ COMIN_ATMOS_GRIB_0p25 | relpath(ROTDIR) }}/{{ head }}pres_a.0p25.f{{ fhr3 }}.grib2.idx" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_a.1p00.f{{ fhr3 }}.grib2" - - "{{ COMIN_ATMOS_GRIB_1p00 | relpath(ROTDIR) }}/{{ head }}pres_a.1p00.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_a.0p25.f{{ fhr3 }}.grib2" + - "{{ COMIN_ATMOS_GRIB_0p25 }}/{{ head }}pres_a.0p25.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_a.1p00.f{{ fhr3 }}.grib2" + - "{{ COMIN_ATMOS_GRIB_1p00 }}/{{ head }}pres_a.1p00.f{{ fhr3 }}.grib2.idx" # Forecast GRIB2 fluxes - - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sflux.f{{ fhr3 }}.grib2" - - "{{ COMIN_ATMOS_MASTER | relpath(ROTDIR) }}/{{ head }}sflux.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_ATMOS_MASTER }}/{{ head }}sflux.f{{ fhr3 }}.grib2" + - "{{ COMIN_ATMOS_MASTER }}/{{ head }}sflux.f{{ fhr3 }}.grib2.idx" # FV3 log - - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}log.f{{ fhr3 }}.txt" + - "{{ COMIN_ATMOS_HISTORY }}/{{ head }}log.f{{ fhr3 }}.txt" # Raw netCDF forecasts - - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}atm.f{{ fhr3 }}.nc" - - "{{ COMIN_ATMOS_HISTORY | relpath(ROTDIR) }}/{{ head }}sfc.f{{ fhr3 }}.nc" + - "{{ COMIN_ATMOS_HISTORY }}/{{ head }}atm.f{{ fhr3 }}.nc" + - "{{ COMIN_ATMOS_HISTORY }}/{{ head }}sfc.f{{ fhr3 }}.nc" {% endfor %} + {# restart_prefixes are pre-calculated in Python #} # Now get the restart files. - {% for r_time in range(restart_interval_gdas, FHMAX + 1, restart_interval_gdas) %} - {% set r_timedelta = (r_time | string + "H") | to_timedelta %} - {% set r_dt = current_cycle | add_to_datetime(r_timedelta) %} - {% set r_YMD = r_dt | to_YMD %} - {% set r_HH = r_dt | strftime("%H") %} - {% set r_prefix = r_YMD + "." + r_HH + "0000" %} + {% for r_prefix in restart_prefixes %} {% for itile in range(1, 7) %} {% for datatype in ["fv_core.res", "fv_srf_wnd.res", "fv_tracer.res", "phy_data", "sfc_data"] %} - - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" + - "{{ COMIN_ATMOS_RESTART }}/{{ r_prefix }}.{{datatype}}.tile{{ itile }}.nc" {% if DO_CA %} - - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.ca_data.tile{{ itile }}.nc" + - "{{ COMIN_ATMOS_RESTART }}/{{ r_prefix }}.ca_data.tile{{ itile }}.nc" {% endif %} {% endfor %} {% endfor %} - - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.coupler.res" - - "{{ COMIN_ATMOS_RESTART | relpath(ROTDIR) }}/{{ r_prefix }}.fv_core.res.nc" + - "{{ COMIN_ATMOS_RESTART }}/{{ r_prefix }}.coupler.res" + - "{{ COMIN_ATMOS_RESTART }}/{{ r_prefix }}.fv_core.res.nc" {% endfor %} # Archive the EXPDIR if requested diff --git a/parm/archive/master_gdas.yaml.j2 b/parm/archive/master_gdas.yaml.j2 index da0dc52bd21..35474b43c67 100644 --- a/parm/archive/master_gdas.yaml.j2 +++ b/parm/archive/master_gdas.yaml.j2 @@ -1,8 +1,3 @@ -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set cycle_YMDH = current_cycle | to_YMDH %} -{% set head = "gdas.t" + cycle_HH + "z." %} - # Archive specific tarball type based on TARBALL_TYPE variable datasets: {% filter indent(width=4) %} diff --git a/parm/archive/master_gefs.yaml.j2 b/parm/archive/master_gefs.yaml.j2 index e33215a23c5..23149f82580 100644 --- a/parm/archive/master_gefs.yaml.j2 +++ b/parm/archive/master_gefs.yaml.j2 @@ -1,8 +1,3 @@ -# Set variables/lists needed to parse the gefs templates -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set cycle_YMDH = current_cycle | to_YMDH %} - datasets: # Archive the EXPDIR if requested {% if archive_expdir %} diff --git a/parm/archive/master_gfs.yaml.j2 b/parm/archive/master_gfs.yaml.j2 index 3144648f654..20df65e9443 100644 --- a/parm/archive/master_gfs.yaml.j2 +++ b/parm/archive/master_gfs.yaml.j2 @@ -1,8 +1,3 @@ -# Set variables/lists needed to parse the templates -{% set cycle_HH = current_cycle | strftime("%H") %} -{% set cycle_YMD = current_cycle | to_YMD %} -{% set cycle_YMDH = current_cycle | to_YMDH %} - # Archive specific tarball type based on TARBALL_TYPE variable datasets: {% filter indent(width=4) %} diff --git a/parm/archive/ocean_6hravg.yaml.j2 b/parm/archive/ocean_6hravg.yaml.j2 index b8fdcf0c53c..0d8a382def6 100644 --- a/parm/archive/ocean_6hravg.yaml.j2 +++ b/parm/archive/ocean_6hravg.yaml.j2 @@ -3,5 +3,5 @@ ocean_6hravg: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_6hravg.tar" required: {% for fhr in range(6, FHMAX_GFS + 6, 6) %} - - "{{ COMIN_OCEAN_HISTORY | relpath(ROTDIR) }}/gfs.t{{ cycle_HH }}z.6hr_avg.f{{ '%03d' % fhr }}.nc" + - "{{ COMIN_OCEAN_HISTORY }}/gfs.t{{ cycle_HH }}z.6hr_avg.f{{ '%03d' % fhr }}.nc" {% endfor %} diff --git a/parm/archive/ocean_grib2.yaml.j2 b/parm/archive/ocean_grib2.yaml.j2 index e1e513ce289..e6997a98578 100644 --- a/parm/archive/ocean_grib2.yaml.j2 +++ b/parm/archive/ocean_grib2.yaml.j2 @@ -5,13 +5,13 @@ ocean_grib2: {% for fhr in range(FHOUT_OCN_GFS, FHMAX_GFS + FHOUT_OCN_GFS, FHOUT_OCN_GFS) %} {% set fhr3 = '%03d' % fhr %} {% if OCNRES == 500 %} - - "{{ COMIN_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ RUN }}.t{{ cycle_HH }}z.5p00.f{{ fhr3 }}.grib2" - - "{{ COMIN_OCEAN_GRIB | relpath(ROTDIR) }}/5p00/{{ RUN }}.t{{ cycle_HH }}z.5p00.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_OCEAN_GRIB }}/5p00/{{ RUN }}.t{{ cycle_HH }}z.5p00.f{{ fhr3 }}.grib2" + - "{{ COMIN_OCEAN_GRIB }}/5p00/{{ RUN }}.t{{ cycle_HH }}z.5p00.f{{ fhr3 }}.grib2.idx" {% elif OCNRES == 100 %} - - "{{ COMIN_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ RUN }}.t{{ cycle_HH }}z.1p00.f{{ fhr3 }}.grib2" - - "{{ COMIN_OCEAN_GRIB | relpath(ROTDIR) }}/1p00/{{ RUN }}.t{{ cycle_HH }}z.1p00.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_OCEAN_GRIB }}/1p00/{{ RUN }}.t{{ cycle_HH }}z.1p00.f{{ fhr3 }}.grib2" + - "{{ COMIN_OCEAN_GRIB }}/1p00/{{ RUN }}.t{{ cycle_HH }}z.1p00.f{{ fhr3 }}.grib2.idx" {% elif OCNRES == 25 or OCNRES == "025" %} - - "{{ COMIN_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ RUN }}.t{{ cycle_HH }}z.0p25.f{{ fhr3 }}.grib2" - - "{{ COMIN_OCEAN_GRIB | relpath(ROTDIR) }}/0p25/{{ RUN }}.t{{ cycle_HH }}z.0p25.f{{ fhr3 }}.grib2.idx" + - "{{ COMIN_OCEAN_GRIB }}/0p25/{{ RUN }}.t{{ cycle_HH }}z.0p25.f{{ fhr3 }}.grib2" + - "{{ COMIN_OCEAN_GRIB }}/0p25/{{ RUN }}.t{{ cycle_HH }}z.0p25.f{{ fhr3 }}.grib2.idx" {% endif %} {% endfor %} diff --git a/parm/archive/ocean_native.yaml.j2 b/parm/archive/ocean_native.yaml.j2 index 51c039c06bb..f55178d7b9a 100644 --- a/parm/archive/ocean_native.yaml.j2 +++ b/parm/archive/ocean_native.yaml.j2 @@ -3,5 +3,5 @@ ocean_native: target: "{{ ATARDIR }}/{{ cycle_YMDH }}/ocean_native_subset.tar" required: {% for fhr in range(FHOUT_OCN_GFS, FHMAX_GFS + FHOUT_OCN_GFS, FHOUT_OCN_GFS) %} - - "{{ COMIN_OCEAN_NETCDF | relpath(ROTDIR) }}/native/{{ RUN }}.t{{ cycle_HH }}z.native.f{{ '%03d' % fhr }}.nc" + - "{{ COMIN_OCEAN_NETCDF }}/native/{{ RUN }}.t{{ cycle_HH }}z.native.f{{ '%03d' % fhr }}.nc" {% endfor %} diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index 389075472e2..5fef14ffe99 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -5,8 +5,7 @@ import shutil import tarfile from logging import getLogger -from typing import Any, Dict, List - +from typing import List from wxflow import (AttrDict, FileHandler, Hsi, Htar, Task, to_timedelta, chgrp, get_gid, logit, mkdir_p, parse_j2yaml, rm_p, rmdir, strftime, to_YMDH, which, chdir, ProcessError, save_as_yaml) @@ -20,12 +19,12 @@ class Archive(Task): """ @logit(logger, name="Archive") - def __init__(self, config: Dict[str, Any]) -> None: + def __init__(self, config: AttrDict) -> None: """Constructor for the Archive task Parameters ---------- - config : Dict[str, Any] + config : AttrDict Incoming configuration for the task from the environment Returns @@ -33,30 +32,21 @@ def __init__(self, config: Dict[str, Any]) -> None: None """ super().__init__(config) - - rotdir = self.task_config.ROTDIR + os.sep - - # Find all absolute paths in the environment and get their relative paths from ${ROTDIR} - path_dict = self._gen_relative_paths(rotdir) - - # Extend task_config with path_dict - self.task_config = AttrDict(**self.task_config, **path_dict) - # Boolean used for cleanup if the EXPDIR was archived self.archive_expdir = False @logit(logger) - def configure_vrfy(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any]): + def configure_vrfy(self, arch_dict: AttrDict) -> (AttrDict): """Determine which files will need to be created to archive to arcdir. Parameters ---------- - arch_dict : Dict[str, Any] + arch_dict : AttrDict Task specific keys, e.g. runtime options (DO_AERO_FCST, DO_ICE, etc) Return ------ - arcdir_set : Dict[str, Any] + arcdir_set : AttrDict Set of FileHandler instructions to copy files to the ARCDIR """ @@ -85,26 +75,25 @@ def configure_vrfy(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any]): arch_dict['path_exists'] = os.path.exists # Parse the input jinja yaml template - arcdir_set = Archive._construct_arcdir_set(arcdir_j2yaml, - arch_dict) + arcdir_set = parse_j2yaml(arcdir_j2yaml, arch_dict, allow_missing=True) # Collect datasets that need to be archived self.tar_cmd = "" - return arcdir_set + return AttrDict(arcdir_set) @logit(logger) - def configure_tars(self, arch_dict: Dict[str, Any]) -> (List[Dict[str, Any]]): + def configure_tars(self, arch_dict: AttrDict) -> (List[AttrDict]): """Determine which tarballs will need to be created. Parameters ---------- - arch_dict : Dict[str, Any] + arch_dict : AttrDict Task specific keys, e.g. runtime options (DO_AERO_FCST, DO_ICE, etc) Return ------ - atardir_sets : List[Dict[str, Any]] + atardir_sets : List[AttrDict] List of tarballs and instructions for creating them via tar or htar """ @@ -165,56 +154,181 @@ def configure_tars(self, arch_dict: Dict[str, Any]) -> (List[Dict[str, Any]]): self.chmod_cmd = os.chmod self.rm_cmd = rm_p else: - raise ValueError("FATAL ERROR: Invalid achiving method selected: {arch_dict.ARCHCOM_TO}") - - master_yaml = "master_" + arch_dict.RUN + ".yaml.j2" + raise ValueError(f"FATAL ERROR: Invalid archiving method selected: {arch_dict.ARCHCOM_TO}") # Determine if expdir archiving is requested this cycle (skip gfs/gdas ensembles) + # Construct master YAML filename based on RUN if "enkf" in arch_dict.RUN: arch_dict['archive_expdir'] = False + master_yaml = "master_enkf.yaml.j2" else: arch_dict['archive_expdir'] = self._archive_expdir(arch_dict) + master_yaml = f"master_{arch_dict.RUN}.yaml.j2" + master_yaml_path = os.path.join(archive_parm, master_yaml) + + # Check if this is EnKF member archiving (ENSGRP != 0) + ensgrp = arch_dict.get('ENSGRP', 0) + + if "enkf" in arch_dict.RUN and ensgrp != 0: + # For EnKF member archiving, render templates once per member + first_group_mem = arch_dict.get('first_group_mem') + last_group_mem = arch_dict.get('last_group_mem') + if first_group_mem is None or last_group_mem is None: + raise ValueError("EnKF member archiving requires first_group_mem and last_group_mem in arch_dict") + atardir_sets = self._parse_yaml_enkf_members(arch_dict, master_yaml_path, first_group_mem, last_group_mem) + + elif ( + ("enkf" in arch_dict.RUN and ensgrp == 0) or + arch_dict.RUN in ["gfs", "gefs", "gdas", "gcdas", "gcafs"] + ): + # Single-pass rendering for EnKF mean/spread and deterministic runs + parsed_sets = parse_j2yaml(master_yaml_path, arch_dict, allow_missing=False) + atardir_sets = self._process_additional_datasets(arch_dict, parsed_sets) + else: + raise ValueError(f"FATAL ERROR: Unsupported RUN type for archiving: {arch_dict.RUN}") - parsed_sets = parse_j2yaml(os.path.join(archive_parm, master_yaml), - arch_dict, - allow_missing=False) + # Save the tarball list as a YAML in case we are using globus + group = arch_dict.get("ENSGRP", -1) + self._create_datasets_yaml(atardir_sets, group) + + return atardir_sets + + @logit(logger) + def _process_additional_datasets(self, arch_dict: AttrDict, parsed_sets: AttrDict) -> List[AttrDict]: + """Process parsed YAML datasets into final archive format with EXPDIR handling. + + This method converts parsed datasets to the final atardir_sets format, + handling EXPDIR archiving logic, path conversion, fileset creation, + and rstprod checking. Used for: + - GFS/GDAS/GCAFS/GCDAS deterministic runs + - EnKF ensemble mean/spread (ENSGRP=0) + - EnKF member aggregation (called from _parse_yaml_enkf_members) + + Parameters + ---------- + arch_dict : AttrDict + Archive configuration dictionary + parsed_sets : AttrDict + Parsed YAML datasets from parse_j2yaml + + Returns + ------- + List[AttrDict] + List of datasets to archive + """ # Determine if we actually archiving the EXPDIR this cycle # This will notify the cleanup function to remove the temporary copy - if arch_dict.archive_expdir: + if arch_dict.get('archive_expdir', False): # Check that "expdir" is in the set of archives to create for dataset in parsed_sets.datasets.values(): - if dataset.name == "EXPDIR": + if dataset.get('name') == "EXPDIR": # If found, check if we should archive this cycle self.archive_expdir = True break # If requested, get workflow hashes/statuses/diffs for EXPDIR archiving - if self.archive_expdir and (arch_dict.ARCH_HASHES or arch_dict.ARCH_DIFFS): + if self.archive_expdir and (arch_dict.get('ARCH_HASHES') or arch_dict.get('ARCH_DIFFS')): self._pop_git_info(arch_dict) atardir_sets = [] for dataset in parsed_sets.datasets.values(): + # Convert COMIN paths from absolute to relative before creating fileset + dataset = self._convert_dataset_paths_to_relative(dataset, arch_dict['ROTDIR']) + dataset["fileset"] = Archive._create_fileset(dataset) dataset["has_rstprod"] = Archive._has_rstprod(dataset.fileset) - atardir_sets.append(dataset) + atardir_sets.append(AttrDict(dataset)) - # Save the tarball list as a YAML in case we are using globus - group = arch_dict.get("ENSGRP", -1) - self._create_datasets_yaml(atardir_sets, group) + return atardir_sets + + @logit(logger) + def _parse_yaml_enkf_members(self, arch_dict: AttrDict, master_yaml_path: str, + first_group_mem: int, last_group_mem: int) -> List[AttrDict]: + """Per-member template rendering for EnKF member archiving. + + This method renders templates once for each ensemble member, collecting + files from all members into the final datasets. + + Parameters + ---------- + arch_dict : AttrDict + Archive configuration dictionary (must have ENSGRP != 0 and member_vars key) + master_yaml_path : str + Full path to the master YAML template (e.g., /path/to/master_enkfgdas.yaml.j2) + first_group_mem : int + First member number in this archive group + last_group_mem : int + Last member number in this archive group + + Returns + ------- + List[AttrDict] + List of datasets to archive (aggregated across all members) + """ + + logger.info(f"Rendering templates for EnKF members {first_group_mem} to {last_group_mem}") + + # Dictionary to accumulate datasets across all members + # Key: dataset name (e.g., "ENKF_GRP", "ENKF_RESTARTA_GRP", "ENKF_RESTARTB_GRP") + # Value: dataset dict with accumulated file lists + accumulated_datasets = AttrDict() + + # Render template once per member + for mem in range(first_group_mem, last_group_mem + 1): + logger.debug(f"Rendering template for member {mem}") + + # Create member-specific dict combining arch_dict with member COM paths + member_vars = arch_dict.get(f"com_set_{mem:02d}") + if member_vars is None: + raise ValueError(f"Member COM paths for com_set_{mem:02d} not found in arch_dict.") + + # Create temporary AttrDict with member-specific variables + member_dict = AttrDict({**arch_dict, **member_vars}) + + # Parse template with member-specific variables + member_parsed_sets = parse_j2yaml( + master_yaml_path, + member_dict, + allow_missing=False) + + # Accumulate datasets + for dataset_key, dataset in member_parsed_sets.datasets.items(): + dataset_name = dataset.get('name') + + if dataset_name not in accumulated_datasets: + # First time seeing this dataset - initialize it + accumulated_datasets[dataset_name] = { + 'name': dataset_name, + 'target': dataset.get('target'), + 'required': [], + 'optional': [] + } + + # Append this member's files to the accumulated dataset + if 'required' in dataset: + accumulated_datasets[dataset_name].required.extend(dataset['required']) + if 'optional' in dataset: + accumulated_datasets[dataset_name].optional.extend(dataset['optional']) + + # Convert accumulated datasets to parsed_sets format and process with standard method + member_parsed_sets = AttrDict({'datasets': accumulated_datasets}) + atardir_sets = self._process_additional_datasets(arch_dict, member_parsed_sets) + + logger.info(f"Accumulated {len(atardir_sets)} datasets from {last_group_mem - first_group_mem + 1} members") return atardir_sets @logit(logger) - def execute_store_products(self, arcdir_set: Dict[str, Any]) -> None: + def execute_store_products(self, arcdir_set: AttrDict) -> None: """Perform local archiving of data products to ARCDIR. Parameters ---------- - arcdir_set : Dict[str, Any] + arcdir_set : AttrDict FileHandler instructions to populate ARCDIR with Return @@ -226,12 +340,12 @@ def execute_store_products(self, arcdir_set: Dict[str, Any]) -> None: FileHandler(arcdir_set).sync() @logit(logger) - def execute_backup_dataset(self, atardir_set: Dict[str, Any]) -> None: + def execute_backup_dataset(self, atardir_set: AttrDict) -> None: """Create a backup tarball from a yaml dict. Parameters ---------- - atardir_set: Dict[str, Any] + atardir_set: AttrDict Dict defining set of files to backup and the target tarball. Return @@ -260,7 +374,7 @@ def execute_backup_dataset(self, atardir_set: Dict[str, Any]) -> None: @staticmethod @logit(logger) - def _create_fileset(atardir_set: Dict[str, Any]) -> List: + def _create_fileset(atardir_set: AttrDict) -> List: """ Collect the list of all available files from the parsed yaml dict. Globs are expanded and if required files are missing, an error is @@ -272,7 +386,7 @@ def _create_fileset(atardir_set: Dict[str, Any]) -> List: Parameters ---------- - atardir_set: Dict + atardir_set: AttrDict Contains full paths for required and optional files to be archived. """ @@ -284,8 +398,8 @@ def _create_fileset(atardir_set: Dict[str, Any]) -> List: # Check that all required files are present and add them to the list of files to archive if "required" in atardir_set: - if atardir_set.required is not None: - for item in atardir_set.required: + if atardir_set['required'] is not None: + for item in atardir_set['required']: glob_set = glob.glob(item) if len(glob_set) == 0: raise FileNotFoundError(f"FATAL ERROR: Required file, directory, or glob {item} not found!") @@ -294,8 +408,8 @@ def _create_fileset(atardir_set: Dict[str, Any]) -> List: # Check for optional files and add found items to the list of files to archive if "optional" in atardir_set: - if atardir_set.optional is not None: - for item in atardir_set.optional: + if atardir_set['optional'] is not None: + for item in atardir_set['optional']: glob_set = glob.glob(item) if len(glob_set) == 0: logger.warning(f"WARNING: optional file/glob {item} not found!") @@ -333,7 +447,7 @@ def _has_rstprod(fileset: List) -> bool: return False @logit(logger) - def _protect_rstprod(self, atardir_set: Dict[str, Any]) -> None: + def _protect_rstprod(self, atardir_set: AttrDict) -> None: """ Changes the group of the target tarball to rstprod and the permissions to 640. If this fails for any reason, attempt to delete the file before exiting. @@ -377,66 +491,61 @@ def _create_tarball(target: str, fileset: List) -> None: for filename in fileset: tarball.add(filename) - @logit(logger) - def _gen_relative_paths(self, root_path: str) -> Dict[str, Any]: - """Generate a dict of paths in self.task_config relative to root_path - - Parameters - ---------- - root_path : str - Path to base all relative paths off of - - Return - ------ - rel_path_dict : Dict - Dictionary of paths relative to root_path. Members will be named - based on the dict names in self.config. For COM paths, the names will - follow COMIN_ --> _dir. For all other directories, the - names will follow --> _dir. - """ - - rel_path_dict = {} - for key, value in self.task_config.items(): - if isinstance(value, str): - if root_path in value: - rel_path = value.replace(root_path, "") - rel_key = (key[4:] if key.startswith("COMIN_") else key).lower() + "_dir" - rel_path_dict[rel_key] = rel_path - - return rel_path_dict - @staticmethod @logit(logger) - def _construct_arcdir_set(arcdir_j2yaml, arch_dict) -> Dict: - """Construct the list of files to send to the ARCDIR and Fit2Obs - directories from a template. + def _convert_dataset_paths_to_relative(dataset: AttrDict, rotdir: str) -> AttrDict: + """Convert all COMIN paths in a dataset from absolute to relative paths. - TODO Copying Fit2Obs data doesn't belong in archiving should be - moved elsewhere. + This method processes the 'required' and 'optional' lists in a rendered + dataset and converts any paths that start with ROTDIR to relative paths. + This ensures tarball contents use portable relative paths. Parameters ---------- - arcdir_j2yaml: str - The filename of the ARCDIR jinja template to parse. - - arch_dict: Dict - The context dictionary to parse arcdir_j2yaml with. + dataset : AttrDict + Dataset dictionary from parsed YAML with 'required' and 'optional' lists + rotdir : str + ROTDIR path to strip from absolute paths - Return - ------ - arcdir_set : Dict - FileHandler dictionary (i.e. with top level "mkdir" and "copy" keys) - containing all directories that need to be created and what data - files need to be copied to the ARCDIR and the Fit2Obs directory. + Returns + ------- + AttrDict + Dataset with all COMIN paths converted to relative paths + + Notes + ----- + This method is called AFTER YAML rendering to convert paths that were + rendered with absolute COMIN variables into relative paths suitable for + tar archiving. + + Examples + -------- + >>> dataset = AttrDict({ + ... 'required': ['/data/rotdir/gfs.20251218/00/atmos/file1.nc'], + ... 'optional': ['/data/rotdir/gfs.20251218/00/atmos/file2.nc'] + ... }) + >>> Archive._convert_dataset_paths_to_relative(dataset, '/data/rotdir') + {'required': ['gfs.20251218/00/atmos/file1.nc'], + 'optional': ['gfs.20251218/00/atmos/file2.nc']} """ + rotdir_prefix = rotdir if rotdir.endswith(os.sep) else rotdir + os.sep - # Get the FileHandler dictionary for creating directories and copying - # to the ARCDIR and VFYARC directories. - arcdir_set = parse_j2yaml(arcdir_j2yaml, - arch_dict, - allow_missing=True) + # Convert required paths + if 'required' in dataset and dataset['required'] is not None: + dataset['required'] = [ + path.replace(rotdir_prefix, '') if rotdir_prefix in path else path + for path in dataset['required'] + ] - return arcdir_set + # Convert optional paths + if 'optional' in dataset and dataset['optional'] is not None: + dataset['optional'] = [ + path.replace(rotdir_prefix, '') if rotdir_prefix in path else path + for path in dataset['optional'] + ] + + logger.debug(f"Converted dataset '{dataset.get('name', 'UNKNOWN')}' paths to relative") + return dataset @staticmethod @logit(logger) @@ -508,7 +617,7 @@ def replace_string_from_to_file(filename_in, filename_out, search_str, replace_s return @logit(logger) - def _archive_expdir(self, arch_dict: Dict[str, Any]) -> bool: + def _archive_expdir(self, arch_dict: AttrDict) -> bool: """ This function checks if the EXPDIR should be archived this RUN/cycle and returns the temporary path in the ROTDIR where the EXPDIR will be @@ -516,7 +625,7 @@ def _archive_expdir(self, arch_dict: Dict[str, Any]) -> bool: Parameters ---------- - arch_dict: Dict + arch_dict: AttrDict Dictionary with required parameters, including the following: current_cycle: Datetime @@ -565,7 +674,7 @@ def _archive_expdir(self, arch_dict: Dict[str, Any]) -> bool: return False @logit(logger) - def _pop_git_info(self, arch_dict: Dict[str, Any]) -> Dict[str, Any]: + def _pop_git_info(self, arch_dict: AttrDict) -> None: """ This function checks the configuration options ARCH_HASHES and ARCH_DIFFS and ARCH_EXPDIR_FREQ to determine if the git hashes and/or diffs should be @@ -574,7 +683,7 @@ def _pop_git_info(self, arch_dict: Dict[str, Any]) -> Dict[str, Any]: Parameters ---------- - arch_dict: Dict + arch_dict: AttrDict Dictionary with required parameters, including the following: EXPDIR: str @@ -639,7 +748,7 @@ def _pop_git_info(self, arch_dict: Dict[str, Any]) -> Dict[str, Any]: return - def _arch_warm_start_increments(self, arch_dict: Dict[str, Any]) -> bool: + def _arch_warm_start_increments(self, arch_dict: AttrDict) -> bool: """ This method determines if warm restart increments are to be archived based on the configuration settings ARCH_CYC (integer cycle number to archive on) and @@ -665,7 +774,7 @@ def _arch_warm_start_increments(self, arch_dict: Dict[str, Any]) -> bool: # Otherwise, do not archive warm restarts return False - def _arch_warm_restart_ics(self, arch_dict: Dict[str, Any]) -> bool: + def _arch_warm_restart_ics(self, arch_dict: AttrDict) -> bool: """ This method determines if warm ICs are to be archived based on the configuration settings ARCH_CYC (integer cycle number to archive on) and @@ -673,16 +782,18 @@ def _arch_warm_restart_ics(self, arch_dict: Dict[str, Any]) -> bool: """ # Get the variables need to determine if warm restart ICs should be archived + cycle_HH = int(strftime(arch_dict.current_cycle, "%H")) SDATE = arch_dict.SDATE RUN = arch_dict.RUN - assim_freq = arch_dict.assim_freq + assim_freq = int(arch_dict.assim_freq) + arch_cyc_val = int(arch_dict.ARCH_CYC) # The GDAS and EnKFGDAS ICs always lag the forecast increments by assim_freq hours if "gdas" in RUN: - arch_cyc = (arch_dict.ARCH_CYC - assim_freq) % 24 + arch_cyc = (arch_cyc_val - assim_freq) % 24 else: - arch_cyc = arch_dict.ARCH_CYC + arch_cyc = arch_cyc_val if cycle_HH != arch_cyc: # Not the right cycle hour @@ -696,7 +807,7 @@ def _arch_warm_restart_ics(self, arch_dict: Dict[str, Any]) -> bool: # Otherwise, do not archive warm restarts return False - def _arch_restart(self, arch_dict: Dict) -> bool: + def _arch_restart(self, arch_dict: AttrDict) -> bool: """ This method determines if warm restarts or warm ICs are to be archived based on the tar_type and the booleans arch_increments and arch_warm_ics. diff --git a/ush/python/pygfs/utils/archive_tar_vars.py b/ush/python/pygfs/utils/archive_tar_vars.py new file mode 100644 index 00000000000..647f436bcb8 --- /dev/null +++ b/ush/python/pygfs/utils/archive_tar_vars.py @@ -0,0 +1,863 @@ +#!/usr/bin/env python3 +""" +Archive Variables Utility Module + +Overview +-------- +This module provides utility functions to collect variables needed by YAML templates +for archiving verification (vrfy) data for GFS and GEFS systems. File set +generation logic (loops, conditionals, path construction) is handled by the YAML +templates themselves. + +Architecture +------------ +Python provides VARIABLES -> YAML templates build FILE SETS + +Python Code Responsibilities: + - Compute cycle-specific variables (cycle_HH, cycle_YMDH, cycle_YMD, head) + - Calculate COM directory paths for ENKF system with grid loops (0p25, 0p50, 1p00) + - Extract configuration keys (RUN, DO_* flags, FHMAX*, etc.) + - Provide complete arch_dict to YAML templates + +YAML Template Responsibilities (parm/archive/master_*.yaml.j2): + - Build file sets with source -> destination mappings + - Handle loops (forecast hours, grids, basins) + - Apply conditionals (DO_* flags, MODE, RUN type) + +Key Functions +------------- +get_all_yaml_vars(config_dict): + Main entry point - collects all variables for YAML templates + +add_config_vars(config_dict): + Extracts configuration keys and COM* variables (created in job scripts) + +Design Note +----------- +This is NOT a Task class - it's a utility module with functions that operate on +config_dict dictionaries. + +Logging +------- +All public operational functions are decorated with @logit(logger). +""" +import os +from logging import getLogger +from wxflow import AttrDict, logit, to_YMD, to_YMDH, add_to_datetime, to_timedelta, to_fv3time + +logger = getLogger(__name__.split('.')[-1]) + + +class ArchiveTarVars: + """ + Utility class for collecting archive tar variables. + + This class provides variables for YAML templates that handle archiving + for three systems: + - GFS: Global Forecast System + - GEFS: Global Ensemble Forecast System + - GCAFS: Global Chemistry and Aerosol Forecast System + - GDAS: Global Data Assimilation Systems (GDAS and GCDAS) + - EnKF: Ensemble Kalman Filter systems (EnKFGFS, EnKFGDAS) + + The YAML templates (parm/archive/master_*.yaml.j2) contain all file set + generation logic. This class only provides the variables they need. + """ + + @staticmethod + @logit(logger) + def get_all_yaml_vars(config_dict: AttrDict) -> AttrDict: + """ + Collect all variables needed for YAML templates. + + This method provides only the VARIABLES needed by the YAML templates + (cycle vars, COM paths, config keys). The YAML templates handle all + file set generation logic (loops, conditionals, path construction). + + Parameters + ---------- + config_dict : AttrDict + Configuration dictionary from Archive.task_config + + Returns + ------- + AttrDict + Dictionary containing variables for Jinja2 templates: + - cycle_HH, cycle_YMDH, cycle_YMD, head: Cycle-specific variables + - COMIN_*, COMOUT_*, COM_*: All COM directory paths (from job scripts) + - Config keys: RUN, PSLOT, ROTDIR, DO_* flags, FHMAX*, etc. + + Notes + ----- + File set generation (mkdir lists, copy operations) is handled entirely + by the YAML templates. This method only provides the variables they need. + COM paths are created in the job scripts (JGLOBAL_ARCHIVE_VRFY and + JGLOBAL_ENKF_ARCHIVE_VRFY) and passed through config_dict. + """ + + # Build arch_dict with variables for Jinja2 templates + arch_dict = AttrDict() + + # Add config variables (config keys, COM* variables from job scripts) + arch_dict.update(ArchiveTarVars.add_config_vars(config_dict)) + + # Add YAML-specific cycle variables (analysis/restart times, archive flags) + arch_dict.update(ArchiveTarVars._get_all_cyc_vars(config_dict)) + + # Add tarball-specific variables if TARBALL_TYPE is defined + tarball_type = config_dict.get('TARBALL_TYPE', '') + if tarball_type: + arch_dict.update(ArchiveTarVars.get_tarball_specific_vars(config_dict, tarball_type)) + + if config_dict.get('RUN') in ['enkfgfs', 'enkfgdas']: + # EnKF systems: Handle ensemble member-specific paths + ensgrp = config_dict.get('ENSGRP', 0) + if ensgrp == 0: + # ENSGRP=0: Ensemble mean/spread (enkf.yaml.j2) + arch_dict.update(ArchiveTarVars.get_enkf_com_paths(config_dict)) + else: + # ENSGRP=!0: Individual member groups + arch_dict.update(ArchiveTarVars._create_enkf_mem_com_sets( + config_dict, + arch_dict['first_group_mem'], + arch_dict['last_group_mem'] + )) + elif config_dict.get('RUN') in ['gfs', 'gdas']: + # GFS/GDAS systems: COMIN variables already set in job scripts + # For wave tarballs, collect all COMIN_WAVE_GRID_* paths from config_dict + if tarball_type in ['gfswave', 'gdaswave']: + arch_dict['WAVE_GRID_RES_COM_list'] = [v for k, v in config_dict.items() if k.startswith('COMIN_WAVE_GRID_')] + elif config_dict.get('RUN') == 'gcafs': + # GCAFS system: COMIN variables already set in job scripts + logger.info("GCAFS system: COMIN variables already set in job scripts") + elif config_dict.get('RUN') == 'gcdas': + logger.info("GCDAS system: COMIN variables already set in job scripts") + else: + logger.info(f"Unknown RUN type '{config_dict.get('RUN')}', no additional COM paths added") + + logger.info(f"Collected {len(arch_dict)} variables for YAML templates") + logger.debug(f"arch_dict keys: {list(arch_dict.keys())}") + + return arch_dict + + @staticmethod + @logit(logger) + def add_config_vars(config_dict: AttrDict) -> AttrDict: + + """ + Collect configuration variables for archive tar operations. + + This method extracts all required configuration keys for + archiving operations, including ensemble-specific parameters. + + Parameters + ---------- + config_dict : AttrDict + Configuration dictionary from Archive.task_config + + Configuration keys extracted (if present): + - Basic: ATARDIR, current_cycle, IAUFHRS, RUN, PDY, PSLOT + - Archive control: DO_ARCHCOM, ARCHCOM_TO, ROTDIR, PARMgfs, ARCDIR, SDATE, MODE + - Ensemble: ENSGRP, NMEM_EARCGRP, NMEM_ENS, NMEM_ENS_GFS + - EnKF operations: DO_CALC_INCREMENT_ENKF_GFS, DO_JEDIATMENS, lobsdiag_forenkf + - Forecast: FHMIN_ENKF, FHMAX_ENKF_GFS, FHOUT_ENKF_GFS, FHMAX_ENKF, FHOUT_ENKF + - EnKF settings: ENKF_SPREAD, DOIAU_ENKF, IAU_OFFSET, IAUFHRS_ENKF + - Restart: restart_interval_enkfgdas, restart_interval_enkfgfs + - Hybrid/DA: DOHYBVAR, DOIAU, DO_CA, DO_CALC_INCREMENT, assim_freq + - Archive timing: ARCH_CYC, ARCH_WARMICFREQ, ARCH_FCSTICFREQ + - Ocean/Ice: DOHYBVAR_OCN, DOLETKF_OCN + - Other: DO_JEDISNOWDA, NET, DO_GSISOILDA, DO_LAND_IAU + + COM variable prefixes collected: + - COM_, COMIN_, COMOUT_ + + Returns + ------- + AttrDict + Dictionary containing all archive variables + + Notes + ----- + Missing keys will be silently skipped (not added to config_dict). + This method is used for all archive operations (GFS, GDAS, EnKF, etc.). + """ + config_vars = AttrDict() + + # Common configuration keys (present in both exglobal_enkf_earc_tars.py and exglobal_archive_tars.py) + config_keys = [ + # Basic configuration + 'ATARDIR', 'current_cycle', 'RUN', 'PDY', 'PSLOT', 'NET', 'MODE', + 'PARMgfs', 'ROTDIR', 'SDATE', + # Archive control + 'DO_ARCHCOM', 'ARCHCOM_TO', 'ARCDIR', + # Data assimilation + 'DOHYBVAR', 'DOIAU', 'DO_CA', 'assim_freq', 'IAUFHRS', + 'DO_JEDISNOWDA', 'DO_GSISOILDA', 'DO_LAND_IAU', + # Ocean/Ice DA + 'DOHYBVAR_OCN', 'DOLETKF_OCN', 'NMEM_ENS', + # Archive timing and control + 'ARCH_CYC', 'ARCH_WARMICFREQ', 'ARCH_FCSTICFREQ', + ] + + # Add system-specific keys based on RUN type + if 'enkf' in config_dict.get('RUN', ''): + # EnKF-specific keys (only in exglobal_enkf_earc_tars.py) + config_keys.extend([ + # Ensemble configuration + 'ENSGRP', 'NMEM_EARCGRP', 'NMEM_ENS_GFS', + # EnKF-specific operations + 'DO_CALC_INCREMENT_ENKF_GFS', 'DO_JEDIATMENS', 'lobsdiag_forenkf', 'DO_CALC_INCREMENT', + # EnKF forecast configuration + 'FHMIN_ENKF', 'FHMAX_ENKF_GFS', 'FHOUT_ENKF_GFS', 'FHMAX_ENKF', 'FHOUT_ENKF', + # EnKF settings + 'ENKF_SPREAD', 'DOIAU_ENKF', 'IAU_OFFSET', 'IAUFHRS_ENKF', + # EnKF restart intervals + 'restart_interval_enkfgdas', 'restart_interval_enkfgfs', + ]) + else: + # Archive-specific keys (only in exglobal_archive_tars.py) + config_keys.extend([ + # Forecast configuration + 'FHMIN', 'FHMAX', 'FHOUT', + 'FHMIN_GFS', 'FHMAX_GFS', 'FHOUT_GFS', 'FHOUT_HF_GFS', 'FHMAX_HF_GFS', + 'FHOUT_OCN', 'FHOUT_ICE', 'FHOUT_OCN_GFS', 'FHOUT_ICE_GFS', + 'FHOUT_WAV', 'FHOUT_WAV_GFS', 'FHOUT_HF_WAV', 'FHMAX_WAV', 'FHMAX_HF_WAV', 'FHMAX_WAV_GFS', + # Monitoring and verification + 'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_FIT2OBS', 'FHMAX_FITS', + # Model components + 'DO_OCN', 'DO_ICE', 'DO_WAVE', 'DO_PREP_OBS_AERO', 'WRITE_DOPOST', + # Data assimilation + 'DO_JEDIATMVAR', 'DO_JEDIOCNVAR', 'DO_AERO_ANL', 'DO_AERO_FCST', 'ATMINC_GRID', + # Restart intervals + 'restart_interval_gdas', 'restart_interval_gfs', + # Archive control + 'ARCH_GAUSSIAN', 'ARCH_GAUSSIAN_FHMAX', 'ARCH_GAUSSIAN_FHINC', + 'ARCH_EXPDIR', 'ARCH_EXPDIR_FREQ', 'ARCH_HASHES', 'ARCH_DIFFS', + # Grid and resolution + 'OCNRES', 'ICERES', 'waveGRD', 'WAVE_OUT_GRIDS', + # Other + 'DO_BUFRSND', 'NUM_SND_COLLECTIVES', 'DOBNDPNT_WAVE', + 'OFFSET_START_HOUR', 'EXPDIR', 'EDATE', 'HOMEgfs', + 'DO_GEMPAK', 'DATASETS_YAML', 'TARBALL_TYPE', + ]) + + # Extract keys if they exist in config_dict + for key in config_keys: + if key in config_dict: + config_vars[key] = config_dict[key] + else: + logger.debug(f"Config key '{key}' not found in config_dict; skipping.") + + # Import COM* directory and template variables created by job scripts + for key in config_dict.keys(): + if key.startswith(("COM_", "COMIN_", "COMOUT_")): + config_vars[key] = config_dict.get(key) + + logger.info(f"Collected {len(config_vars)} archive tar variables") + logger.debug(f"Archive variables: {list(config_vars.keys())}") + + return config_vars + + @staticmethod + @logit(logger) + def _get_all_cyc_vars(config_dict: AttrDict) -> AttrDict: + """Compute common cycle variables for all archive YAML templates. + + This method computes basic cycle variables used across all archive systems + (GFS, GDAS, EnKF, GCAFS). System-specific variables are computed in their + respective methods. + + Parameters + ---------- + config_dict : AttrDict + Configuration dictionary from Archive.task_config + Required keys: current_cycle, assim_freq + + Returns + ------- + AttrDict + Dictionary containing common cycle variables: + - cycle_HH, cycle_YMDH, cycle_YMD: Current cycle time components + - assim_freq: Assimilation frequency as string + Plus system-specific variables if RUN is 'enkf*' + """ + current_cycle = config_dict.current_cycle + assim_freq = config_dict.get('assim_freq', 6) + + vars_out = AttrDict() + # Basic cycle variables (common to all systems) + vars_out['cycle_HH'] = current_cycle.strftime("%H") + vars_out['cycle_YMDH'] = to_YMDH(current_cycle) + vars_out['cycle_YMD'] = to_YMD(current_cycle) + vars_out['cycle_fv3time'] = to_fv3time(current_cycle) + + # Assimilation frequency + vars_out['assim_freq'] = str(assim_freq) + + # Padded IAU forecast hours for templates + if 'IAUFHRS' in config_dict: + vars_out['iaufhrs_str'] = [f"{h:03d}" for h in config_dict['IAUFHRS']] + + # Add EnKF-specific variables if RUN contains 'enkf' + if 'enkf' in config_dict.get('RUN', ''): + vars_out.update(ArchiveTarVars._get_enkf_specific_cyc_vars(config_dict, current_cycle)) + + # Add GCDAS-specific variables if RUN is 'gcdas' + if config_dict.get('RUN', '') == 'gcdas': + vars_out.update(ArchiveTarVars._get_gcdas_specific_cyc_vars(config_dict, current_cycle)) + + return vars_out + + @staticmethod + @logit(logger) + def _get_enkf_specific_cyc_vars(config_dict: AttrDict, current_cycle) -> AttrDict: + """Compute EnKF-specific cycle variables. + + This method computes variables specific to EnKF ensemble systems including + forecast configuration, member ranges, and restart prefixes. + + Parameters + ---------- + config_dict : AttrDict + Configuration dictionary from Archive.task_config + Required keys: RUN + Optional keys: FHMIN_ENKF, FHMAX_ENKF, FHOUT_ENKF, DO_CALC_INCREMENT, + DO_CALC_INCREMENT_ENKF_GFS, NMEM_ENS, NMEM_ENS_GFS, + restart_interval_enkfgdas, restart_interval_enkfgfs, + ENSGRP, NMEM_EARCGRP + current_cycle : datetime + Current cycle time + + Returns + ------- + AttrDict + Dictionary containing EnKF-specific variables: + - fhmin, fhmax, fhout: Forecast time configuration + - do_calc_increment: Whether to calculate increments + - nmem_ens: Number of ensemble members + - restart_interval: Restart interval for this system + - is_gdas, is_gfs: System type flags + - enkf_epos_ngrps: Number of ensemble groups (ENSGRP=0 only) + - first_group_mem, last_group_mem: Member range for this group + - restart_prefixes: List of restart time prefixes + """ + enkf_vars = AttrDict() + + # EnKF-specific analysis and restart times (using DOIAU_ENKF) + doiau_enkf = config_dict.get('DOIAU_ENKF', False) + dohybvar_ocn = config_dict.get('DOHYBVAR_OCN', False) + + # Analysis time (for surface analysis restart files) + anl_delta = to_timedelta("-3H") if doiau_enkf else to_timedelta("0H") + anl_time = add_to_datetime(current_cycle, anl_delta) + enkf_vars['anl_YMD'] = to_YMD(anl_time) + enkf_vars['anl_HH'] = anl_time.strftime("%H") + + # Restart hour calculations (when DOHYBVAR_OCN is true) + # Two different logic blocks for different templates: + # 1. enkf_restarta: conditional on DOIAU_ENKF (+3H if true, +6H if false) + # 2. enkf_restartb: always +3H (not conditional) + if dohybvar_ocn: + # For enkf_restarta_grp.yaml.j2 (conditional logic) + rst_delta_a = to_timedelta("+3H") if doiau_enkf else to_timedelta("+6H") + rst_time_a = add_to_datetime(current_cycle, rst_delta_a) + enkf_vars['rst_HH_restarta'] = rst_time_a.strftime("%H") + + # For enkf_restartb_grp.yaml.j2 (always +3H) + rst_delta_b = to_timedelta("+3H") + rst_time_b = add_to_datetime(current_cycle, rst_delta_b) + enkf_vars['rst_HH_restartb'] = rst_time_b.strftime("%H") + + # Forecast output frequency + enkf_vars['fhout'] = config_dict.get('FHOUT_ENKF', 3) + enkf_vars['fhmin'] = config_dict.get('FHMIN_ENKF', 0) + enkf_vars['fhmax'] = config_dict.get('FHMAX_ENKF', 0) + # System-specific configuration + if config_dict.get('RUN', '') == 'enkfgfs': + enkf_vars['do_calc_increment'] = config_dict.get('DO_CALC_INCREMENT_ENKF_GFS', False) + enkf_vars['nmem_ens'] = config_dict.get('NMEM_ENS_GFS', None) + enkf_vars['restart_interval'] = config_dict.get('restart_interval_enkfgfs', None) + enkf_vars['is_gdas'] = False + enkf_vars['is_gfs'] = True + elif config_dict.get('RUN', '') == 'enkfgdas': + enkf_vars['do_calc_increment'] = config_dict.get('DO_CALC_INCREMENT', False) + enkf_vars['nmem_ens'] = config_dict.get('NMEM_ENS') + enkf_vars['restart_interval'] = config_dict.get('restart_interval_enkfgdas', None) + enkf_vars['is_gdas'] = True + enkf_vars['is_gfs'] = False + else: + logger.warning( + f"RUN='{config_dict.get('RUN', '')}' does not match a supported EnKF type ('enkfgfs' or 'enkfgdas'). " + ) + + # ENSGRP-specific calculations + ensgrp = config_dict.get('ENSGRP', 0) + if ensgrp == 0: + enkf_vars['enkf_epos_ngrps'] = len(range(enkf_vars['fhmin'], enkf_vars['fhmax'] + enkf_vars['fhout'], enkf_vars['fhout'])) + else: + nmem_earcgrp = config_dict.get('NMEM_EARCGRP') + if nmem_earcgrp and enkf_vars['nmem_ens']: + enkf_vars['first_group_mem'] = (ensgrp - 1) * nmem_earcgrp + 1 + enkf_vars['last_group_mem'] = min(ensgrp * nmem_earcgrp, enkf_vars['nmem_ens']) + + # Pre-compute all restart time prefixes for YAML templates using helper method + if enkf_vars.get('is_gdas') and enkf_vars.get('restart_interval') and enkf_vars.get('fhmax'): + enkf_vars['restart_prefixes'] = ArchiveTarVars._calculate_restart_prefixes( + current_cycle, + enkf_vars['restart_interval'], + enkf_vars['fhmax'] + ) + else: + enkf_vars['restart_prefixes'] = [] + + # Archive timing logic for EnKF systems (from master_enkf.yaml.j2) + # Both archive groups require: is_gdas AND SDATE AND specific day/cycle conditions + sdate = config_dict.get('SDATE') + arch_warmicfreq = config_dict.get('ARCH_WARMICFREQ', 1) + arch_cyc = config_dict.get('ARCH_CYC', 0) + assim_freq = config_dict.get('assim_freq', 6) + + # Archive timing booleans - increments (group a) + # Logic: (current_cycle - SDATE).days % ARCH_WARMICFREQ == 0 AND is_gdas AND ARCH_CYC == cycle_HH + enkf_vars['archive_increments'] = False + current_cycle_days = (current_cycle - sdate).days + enkf_vars['archive_increments'] = ( + (current_cycle_days % arch_warmicfreq == 0) and + enkf_vars.get('is_gdas', False) and + (arch_cyc == int(current_cycle.strftime("%H"))) + ) + + # Archive timing booleans - ICs (group b) + # Logic: (ics_offset_cycle - SDATE).days % ARCH_WARMICFREQ == 0 AND is_gdas AND (ARCH_CYC - assim_freq) % 24 == cycle_HH + enkf_vars['archive_ics'] = False + ics_offset_cycle = add_to_datetime(current_cycle, to_timedelta(f"+{assim_freq}H")) + ics_offset_days = (ics_offset_cycle - sdate).days + enkf_vars['archive_ics'] = ( + (ics_offset_days % arch_warmicfreq == 0) and + enkf_vars.get('is_gdas', False) and + ((arch_cyc - assim_freq) % 24 == int(current_cycle.strftime("%H"))) + ) + + # Warm start flags (placeholders for future use) + enkf_vars['save_warm_start_forecast'] = False + enkf_vars['save_warm_start_cycled'] = False + + return enkf_vars + + @staticmethod + @logit(logger) + def _get_gcdas_specific_cyc_vars(config_dict: AttrDict, current_cycle) -> AttrDict: + """Compute GCDAS-specific cycle variables. + + This method computes variables specific to GCDAS (Global Chemistry Data + Assimilation System) including restart prefixes for archiving. + + Parameters + ---------- + config_dict : AttrDict + Configuration dictionary from Archive.task_config + Optional keys: restart_interval_gdas, FHMAX + current_cycle : datetime + Current cycle time + + Returns + ------- + AttrDict + Dictionary containing GCDAS-specific variables: + - restart_prefixes: List of restart time prefixes + """ + gcdas_vars = AttrDict() + + # GCDAS restart prefixes calculation + restart_interval = config_dict.get('restart_interval_gdas', 6) + fhmax = config_dict.get('FHMAX', 9) + gcdas_vars['restart_prefixes'] = ArchiveTarVars._calculate_restart_prefixes( + current_cycle, restart_interval, fhmax + ) + + logger.info(f"Calculated {len(gcdas_vars['restart_prefixes'])} restart prefixes for GCDAS " + f"(interval={restart_interval}H, FHMAX={fhmax}H)") + + return gcdas_vars + + @staticmethod + @logit(logger) + def get_enkf_com_paths(config_dict: AttrDict) -> AttrDict: + """Extract absolute COMIN paths for EnKF ensemble mean/spread (ENSGRP=0). + + This method extracts absolute COM paths from config_dict. The paths will be + converted to relative paths AFTER YAML rendering in archive.py. + + Parameters + ---------- + config_dict : AttrDict + Configuration dictionary from Archive.task_config + Required keys: ROTDIR + Optional keys: COMIN_*, COMOUT_* variables (created by job scripts) + + Returns + ------- + AttrDict + Dictionary with absolute COMIN paths for ensemble statistics: + - Keys match enkf.yaml.j2 template variable names + - Values are absolute paths (conversion to relative happens after YAML rendering) + + Notes + ----- + This method should ONLY be called when ENSGRP == 0 (ensemble mean archiving). + For individual member archiving (ENSGRP != 0), use get_enkf_member_com_paths(). + + Paths remain absolute at this stage. Relative path conversion happens in + archive.py after YAML rendering. + + Examples + -------- + >>> # Job script creates: COMIN_ATMOS_HISTORY_ENSSTAT=/path/to/ROTDIR/enkfgdas.20211221/00/atmos/ensstat + >>> com_paths = ArchiveTarVars.get_enkf_com_paths(config) + >>> com_paths['COMIN_ATMOS_HISTORY_ENSSTAT'] + '/path/to/ROTDIR/enkfgdas.20211221/00/atmos/ensstat' + """ + com_paths = AttrDict() + com_vars = [ + 'COMIN_ATMOS_HISTORY', + 'COMIN_ATMOS_HISTORY_ENSSTAT', + 'COMIN_ATMOS_ANALYSIS_ENSSTAT', + 'COMIN_SNOW_ANALYSIS_ENSSTAT', + 'COMIN_OCEAN_ANALYSIS_ENSSTAT', + 'COMIN_ICE_ANALYSIS_ENSSTAT', + 'COMIN_CONF', + ] + for var_name in com_vars: + if var_name in config_dict: + abs_path = config_dict[var_name] + com_paths[var_name] = abs_path + logger.debug(f"Extracted {var_name}: {abs_path}") + logger.info(f"Extracted {len(com_paths)} absolute ensemble statistics COM paths (will convert to relative after YAML rendering)") + return com_paths + + @staticmethod + @logit(logger) + def get_enkf_member_com_paths(config_dict: AttrDict, member: int) -> AttrDict: + """Generate absolute COM paths for a single ensemble member. + + This method creates absolute COM paths for a specific ensemble member. + The paths will be converted to relative paths AFTER YAML rendering in archive.py. + + Parameters + ---------- + config_dict : AttrDict + Configuration dictionary from Archive.task_config + Required keys: ROTDIR, COM_*_TMPL variables + member : int + Member number (e.g., 1, 2, 3, ..., NMEM_ENS) + + Returns + ------- + AttrDict + Dictionary with absolute COM paths for this specific member: + - COMIN_ATMOS_ANALYSIS_MEM: Absolute path to member analysis directory + - COMIN_ATMOS_HISTORY_MEM: Absolute path to member history directory + - COMIN_ATMOS_RESTART_MEM: Absolute path to member restart directory + - COMIN_OCEAN_ANALYSIS_MEM: Absolute path to member ocean analysis + - COMIN_OCEAN_LETKF_MEM: Absolute path to member ocean LETKF + - COMIN_OCEAN_HISTORY_MEM: Absolute path to member ocean history + - COMIN_OCEAN_RESTART_MEM: Absolute path to member ocean restart + - COMIN_ICE_ANALYSIS_MEM: Absolute path to member ice analysis + - COMIN_ICE_LETKF_MEM: Absolute path to member ice LETKF + - COMIN_ICE_HISTORY_MEM: Absolute path to member ice history + - COMIN_ICE_RESTART_MEM: Absolute path to member ice restart + - COMIN_MED_RESTART_MEM: Absolute path to member mediator restart + - member_num: Member number (padded to 3 digits, e.g., "001") + Paths are absolute (conversion to relative happens after YAML rendering). + + Notes + ----- + This method is called during per-member template rendering in configure_tars. + The singular variable names (COMIN_*_MEM) are used in simplified YAML templates + that no longer contain member loops. + + Paths remain absolute at this stage. Relative path conversion happens in + archive.py after YAML rendering. + + Examples + -------- + >>> # Generate variables for member 5 + >>> member_vars = ArchiveTarVars.get_enkf_single_member_vars(config, 5) + >>> member_vars['COMIN_ATMOS_RESTART_MEM'] + '/path/to/ROTDIR/enkfgdas.20211221/00/atmos/mem005' + >>> member_vars['member_num'] + '005' + """ + # Create member-specific cycle dictionary + cycle_dict = ArchiveTarVars._create_cycle_dicts(config_dict) + cycle_dict['${MEMDIR}'] = f"mem{member:03d}" + + # Define template mappings (singular key -> template key) + template_mappings = [ + ('COMIN_ATMOS_ANALYSIS_MEM', 'COM_ATMOS_ANALYSIS_TMPL'), + ('COMIN_ATMOS_HISTORY_MEM', 'COM_ATMOS_HISTORY_TMPL'), + ('COMIN_ATMOS_RESTART_MEM', 'COM_ATMOS_RESTART_TMPL'), + ('COMIN_OCEAN_ANALYSIS_MEM', 'COM_OCEAN_ANALYSIS_TMPL'), + ('COMIN_OCEAN_LETKF_MEM', 'COM_OCEAN_LETKF_TMPL'), + ('COMIN_OCEAN_HISTORY_MEM', 'COM_OCEAN_HISTORY_TMPL'), + ('COMIN_OCEAN_RESTART_MEM', 'COM_OCEAN_RESTART_TMPL'), + ('COMIN_ICE_ANALYSIS_MEM', 'COM_ICE_ANALYSIS_TMPL'), + ('COMIN_ICE_LETKF_MEM', 'COM_ICE_LETKF_TMPL'), + ('COMIN_ICE_HISTORY_MEM', 'COM_ICE_HISTORY_TMPL'), + ('COMIN_ICE_RESTART_MEM', 'COM_ICE_RESTART_TMPL'), + ('COMIN_MED_RESTART_MEM', 'COM_MED_RESTART_TMPL'), + ] + + # Generate absolute COM paths for this member + member_vars = {} + for var_key, template_key in template_mappings: + if config_dict.get(template_key): + # Replace template variables to create absolute path + abs_path = config_dict[template_key] + for var, value in cycle_dict.items(): + abs_path = abs_path.replace(var, value) + member_vars[var_key] = abs_path + + logger.debug(f"Generated {len(member_vars)} absolute COM paths for member {member} (will convert to relative after YAML rendering)") + return member_vars + + @staticmethod + @logit(logger) + def _create_enkf_mem_com_sets(config_dict: AttrDict, first_group_mem: int, last_group_mem: int) -> AttrDict: + """Generate COM path sets for a group of ensemble members. + + Parameters + ---------- + config_dict : AttrDict + Configuration dictionary with COM templates + first_group_mem : int + First member number in this archive group + last_group_mem : int + Last member number in this archive group + + Returns + ------- + AttrDict + Dictionary mapping com_set_NNN keys to member-specific COM paths + """ + mem_var_set = {} + for member in range(first_group_mem, last_group_mem + 1): + mem_var_set[f"com_set_{member:02d}"] = ArchiveTarVars.get_enkf_member_com_paths(config_dict, member) + return mem_var_set + + @staticmethod + @logit(logger) + def _create_cycle_dicts(config_dict: AttrDict) -> AttrDict: + """Create cycle directories for template substitution + + Parameters + ---------- + rotdir : str + ROTDIR path + run : str + RUN type + + Returns + ------- + AttrDict + Dictionary containing current_cycle_dict and previous_cycle_dict + """ + return { + '${ROTDIR}': config_dict['ROTDIR'], + '${RUN}': config_dict['RUN'], + '${YMD}': to_YMD(config_dict['current_cycle']), + '${HH}': config_dict['current_cycle'].strftime("%H"), + } + + @staticmethod + @logit(logger) + def get_tarball_specific_vars(config_dict: AttrDict, tarball_type: str) -> AttrDict: + """ + Calculate tarball-specific template variables. + + This method computes variables that are specific to certain tarballs, + such as offset times for wave restart files and analysis times that depend + on IAU settings. + + Parameters + ---------- + config_dict : AttrDict + Archive configuration dictionary with current_cycle and optional DOIAU, DOHYBVAR + tarball_type : str + Type of tarball being created (e.g., 'gdaswave_restart', 'gdas_restarta') + + Returns + ------- + AttrDict + Dictionary of tarball-specific variables, which may include: + - offset_dt: datetime offset for wave restart files based on IAU + - anl_time: analysis time adjusted for IAU/hybrid var + - prefix: formatted datetime prefix for restart files (YYYYMMDD.HHMMSS) + - offset_YMD, offset_HH: formatted date/hour components + - r_prefix_list: list of restart prefixes for restart intervals + + Notes + ----- + Time offset logic by tarball type: + + gdaswave_restart, gdaswave: + - If DOIAU: +3H (IAU window beginning) or +6H (standard) + + gdas_restarta, gfs_restarta, gdasocean_analysis, gfsocean_analysis: + - If DOHYBVAR and DOIAU: -3H (IAU window beginning) + - Otherwise: 0H (current cycle) + + gdas_restartb: + - If DOIAU: -3H offset_dt with formatted prefix + - Always includes center time prefix + - Calculates r_prefix_list for restart intervals + + Examples + -------- + >>> config_dict = AttrDict({ + ... 'current_cycle': datetime(2025, 12, 18, 0), + ... 'DOIAU': True + ... }) + >>> ArchiveTarVars.get_tarball_specific_vars(config_dict, 'gdaswave_restart') + {'offset_dt': datetime(2025, 12, 18, 3)} + """ + tarball_vars = AttrDict() + current_cycle = config_dict['current_cycle'] + + if tarball_type in ['gdaswave_restart']: + # Wave restart offset time calculation + # If IAU is enabled, use +3H offset (beginning of IAU window) + # Otherwise, use +6H offset (standard forecast time) + doiau = config_dict.get('DOIAU', False) + offset_hours = 3 if doiau else 6 + + offset_dt = add_to_datetime(current_cycle, to_timedelta(f"+{offset_hours}H")) + # Return formatted string for direct use in YAML + tarball_vars['offset_dt_fv3'] = to_fv3time(offset_dt) + + logger.info(f"Calculated offset_dt_fv3 for {tarball_type}: {to_fv3time(offset_dt)} " + f"(DOIAU={doiau}, offset={offset_hours}H)") + + elif tarball_type in ['gdaswave']: + # Wave data always uses +6H offset + offset_dt = add_to_datetime(current_cycle, to_timedelta("+6H")) + # Return formatted string for direct use in YAML + tarball_vars['offset_dt_fv3'] = to_fv3time(offset_dt) + + logger.info(f"Calculated offset_dt_fv3 for {tarball_type}: {to_fv3time(offset_dt)} (+6H)") + + elif tarball_type in ['gdas_restarta', 'gfs_restarta']: + # Atmosphere restart analysis time calculation + # If hybrid var with IAU: use -3H (beginning of IAU window) + # Otherwise: use 0H (current cycle time) + dohybvar = config_dict.get('DOHYBVAR', False) + doiau = config_dict.get('DOIAU', False) + + if dohybvar and doiau: + anl_offset = "-3H" + else: + anl_offset = "0H" + + anl_time = add_to_datetime(current_cycle, to_timedelta(anl_offset)) + # Return formatted strings for direct use in YAML + tarball_vars['anl_time_YMD'] = to_YMD(anl_time) + tarball_vars['anl_time_HH'] = anl_time.strftime("%H") + + logger.info(f"Calculated anl_time for {tarball_type}: {to_YMD(anl_time)}.{anl_time.strftime('%H')}0000 " + f"(DOHYBVAR={dohybvar}, DOIAU={doiau}, offset={anl_offset})") + + elif tarball_type in ['gdasocean_analysis', 'gfsocean_analysis']: + # Ocean/ice analysis time calculation + # If IAU: use -3H (beginning of IAU window) + # Otherwise: use 0H (current cycle time) + doiau = config_dict.get('DOIAU', False) + + if doiau: + anl_offset = "-3H" + else: + anl_offset = "0H" + + anl_time = add_to_datetime(current_cycle, to_timedelta(anl_offset)) + # Return formatted strings for direct use in YAML + tarball_vars['anl_time_YMD'] = to_YMD(anl_time) + tarball_vars['anl_time_HH'] = anl_time.strftime("%H") + + logger.info(f"Calculated anl_time for {tarball_type}: {to_YMD(anl_time)}.{anl_time.strftime('%H')}0000 " + f"(DOIAU={doiau}, offset={anl_offset})") + + elif tarball_type == 'gdas_restartb': + # Restart B has multiple time calculations + doiau = config_dict.get('DOIAU', False) + + # If IAU is on, calculate offset time (-3H) and its prefix + if doiau: + offset_dt = add_to_datetime(current_cycle, to_timedelta("-3H")) + offset_YMD = to_YMD(offset_dt) + offset_HH = offset_dt.strftime("%H") + offset_prefix = f"{offset_YMD}.{offset_HH}0000" + + tarball_vars['offset_dt'] = offset_dt + tarball_vars['offset_YMD'] = offset_YMD + tarball_vars['offset_HH'] = offset_HH + tarball_vars['offset_prefix'] = offset_prefix + + logger.debug(f"Calculated offset_prefix for gdas_restartb: {offset_prefix} (DOIAU=True)") + + # Always calculate center time prefix + cycle_YMD = to_YMD(current_cycle) + cycle_HH = current_cycle.strftime("%H") + tarball_vars['center_prefix'] = f"{cycle_YMD}.{cycle_HH}0000" + + # Calculate restart interval prefixes using helper method + restart_interval = config_dict.get('restart_interval_gdas', 6) + fhmax = config_dict.get('FHMAX', 9) + tarball_vars['r_prefix_list'] = ArchiveTarVars._calculate_restart_prefixes( + current_cycle, restart_interval, fhmax + ) + + logger.info(f"Calculated {len(tarball_vars['r_prefix_list'])} restart prefixes for gdas_restartb " + f"(interval={restart_interval}H, FHMAX={fhmax}H)") + + else: + logger.warning(f"Tarball type '{tarball_type}' does not have specific variable calculations") + + return tarball_vars + + @staticmethod + @logit(logger) + def _calculate_restart_prefixes(current_cycle: 'datetime', restart_interval: int, fhmax: int) -> list: + """Calculate restart time prefixes for a given interval and forecast max. + + This is a reusable helper for calculating restart file prefixes that appear + at regular intervals (e.g., every 6 hours) throughout a forecast period. + + Parameters + ---------- + current_cycle : datetime + Current cycle time + restart_interval : int + Interval in hours between restart files + fhmax : int + Maximum forecast hour + + Returns + ------- + list + List of restart prefixes in format "YYYYMMDD.HHMMSS" + + Examples + -------- + >>> from datetime import datetime + >>> cycle = datetime(2025, 12, 18, 0) + >>> ArchiveTarVars._calculate_restart_prefixes(cycle, 6, 12) + ['20251218.060000', '20251218.120000'] + """ + restart_prefixes = [] + for r_time in range(restart_interval, fhmax + 1, restart_interval): + r_dt = add_to_datetime(current_cycle, to_timedelta(f"+{r_time}H")) + r_prefix = f"{to_YMD(r_dt)}.{r_dt.strftime('%H')}0000" + restart_prefixes.append(r_prefix) + + logger.debug(f"Calculated {len(restart_prefixes)} restart prefixes " + f"(interval={restart_interval}H, FHMAX={fhmax}H)") + return restart_prefixes diff --git a/ush/python/pygfs/utils/archive_vars.py b/ush/python/pygfs/utils/archive_vrfy_vars.py similarity index 95% rename from ush/python/pygfs/utils/archive_vars.py rename to ush/python/pygfs/utils/archive_vrfy_vars.py index 182d918e996..621e0012d08 100644 --- a/ush/python/pygfs/utils/archive_vars.py +++ b/ush/python/pygfs/utils/archive_vrfy_vars.py @@ -41,7 +41,6 @@ """ import os from logging import getLogger -from typing import Any, Dict from wxflow import AttrDict, logit, to_YMD, to_YMDH logger = getLogger(__name__.split('.')[-1]) @@ -63,7 +62,7 @@ class ArchiveVrfyVars: @staticmethod @logit(logger) - def get_all_yaml_vars(config_dict: AttrDict) -> Dict[str, Any]: + def get_all_yaml_vars(config_dict: AttrDict) -> AttrDict: """Collect all variables needed for YAML templates. This method provides only the VARIABLES needed by the YAML templates @@ -76,7 +75,7 @@ def get_all_yaml_vars(config_dict: AttrDict) -> Dict[str, Any]: Returns ------- - Dict[str, Any] + AttrDict Dictionary containing variables for Jinja2 templates: - cycle_HH, cycle_YMDH, cycle_YMD, head: Cycle-specific variables - COMIN_*, COMOUT_*, COM_*: All COM directory paths (from job scripts) @@ -98,7 +97,7 @@ def get_all_yaml_vars(config_dict: AttrDict) -> Dict[str, Any]: @staticmethod @logit(logger) - def add_config_vars(config_dict: AttrDict) -> Dict[str, Any]: + def add_config_vars(config_dict: AttrDict) -> AttrDict: """Collect configuration keys and COM* variables for archive operations. Formats resolution variables (OCNRES, ICERES) to 3 digits and extracts @@ -111,7 +110,7 @@ def add_config_vars(config_dict: AttrDict) -> Dict[str, Any]: Returns ------- - Dict[str, Any] + AttrDict Dictionary with config keys and all COM_*, COMIN_*, COMOUT_* variables """ general_dict = {} @@ -152,7 +151,7 @@ def add_config_vars(config_dict: AttrDict) -> Dict[str, Any]: @staticmethod @logit(logger) - def _get_cycle_vars(config_dict: AttrDict) -> Dict[str, Any]: + def _get_cycle_vars(config_dict: AttrDict) -> AttrDict: """Calculate cycle-specific variables. Parameters @@ -162,7 +161,7 @@ def _get_cycle_vars(config_dict: AttrDict) -> Dict[str, Any]: Returns ------- - Dict[str, Any] + AttrDict Dictionary containing: - cycle_HH: Cycle hour (e.g., '00', '06') - cycle_YMDH: Full cycle timestamp (YYYYMMDDHH)