Skip to content

Preprocess and assimilate the GHCN snow data #4386

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
jiaruidong2017 wants to merge 10 commits into from
Closed

Preprocess and assimilate the GHCN snow data #4386

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
d1d6612
Preprocess and assimilate the GHCN snow data
jiaruidong2017 Dec 27, 2025
805ee0a
Add GHCN preprocessing into the snow ensemble analysis
jiaruidong2017 Dec 27, 2025
1cda206
Add an option for manually setting a run to fail or warn
jiaruidong2017 Dec 31, 2025
3933d6e
Merge branch 'develop' into feature/snow-ghcn2
jiaruidong2017 Dec 31, 2025
f2c6847
Merge PR #4387 into this PR
jiaruidong2017 Jan 2, 2026
a52892f
Rename the environment variabe to be more specific
jiaruidong2017 Jan 2, 2026
3a0faa7
Merge branch 'develop' into feature/snow-ghcn2
jiaruidong2017 Jan 5, 2026
8a8a72e
Update the GDASApp commit hash
jiaruidong2017 Jan 5, 2026
82a104a
Update config.esnowanl.j2 to include the required parameters
jiaruidong2017 Jan 5, 2026
f374702
Add DO_GHCN in the script for snow ensemble analysis
jiaruidong2017 Jan 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ ush/global_cycle_driver.sh
ush/gsi_satbias2ioda_all.sh
ush/jediinc2fv3.py
ush/imsfv3_scf2ioda.py
ush/ghcn_snod2ioda.py
ush/bufr_snocvr_snomad.py
ush/atparse.bash
ush/bufr2ioda_insitu*
Expand Down
10 changes: 7 additions & 3 deletions dev/parm/config/gfs/config.esnowanl.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,19 @@ echo "BEGIN: config.esnowanl"
# Get task specific resources
source "${EXPDIR}/config.resources" esnowanl

export TASK_CONFIG_YAML="${PARMgfs}/gdas/snow/snow_ens_config.yaml.j2"
export OBS_LIST_YAML="${PARMgfs}/gdas/snow/snow_obs_list.yaml.j2"

# Name of the executable that applies increment to bkg and its namelist template
export APPLY_INCR_EXE="${EXECgfs}/gdas_apply_incr.x"
export ENS_APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/ens_apply_incr_nml.j2"

export TASK_CONFIG_YAML="${PARMgfs}/gdas/snow/snow_ens_config.yaml.j2"
export OBS_LIST_YAML="${PARMgfs}/gdas/snow/snow_obs_list.yaml.j2"
export ims_scf_obs_suffix="asc" # asc-ascii; nc-netcdf
export fail_on_missing_snowobs=False # False: just warn; True: fail & exit

export PREP_SNOCVR_SNOMAD_YAML="${PARMgfs}/gdas/snow/prep/prep_snocvr_snomad.yaml.j2"
export OBSBUILDER="${USHgfs}/bufr_snocvr_snomad.py"
export PREP_GHCN_YAML="${PARMgfs}/gdas/snow/prep/prep_ghcn.yaml.j2"
export GHCN2IODACONV="${USHgfs}/ghcn_snod2ioda.py"

export io_layout_x="{{ IO_LAYOUT_X }}"
export io_layout_y="{{ IO_LAYOUT_Y }}"
Expand Down
4 changes: 4 additions & 0 deletions dev/parm/config/gfs/config.snowanl.j2
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,13 @@ export APPLY_INCR_NML_TMPL="${PARMgfs}/gdas/snow/apply_incr_nml.j2"

export TASK_CONFIG_YAML="${PARMgfs}/gdas/snow/snow_det_config.yaml.j2"
export OBS_LIST_YAML="${PARMgfs}/gdas/snow/snow_obs_list.yaml.j2"
export ims_scf_obs_suffix="asc" # asc-ascii; nc-netcdf
export fail_on_missing_snowobs=False # False: just warn; True: fail & exit

export PREP_SNOCVR_SNOMAD_YAML="${PARMgfs}/gdas/snow/prep/prep_snocvr_snomad.yaml.j2"
export OBSBUILDER="${USHgfs}/bufr_snocvr_snomad.py"
export PREP_GHCN_YAML="${PARMgfs}/gdas/snow/prep/prep_ghcn.yaml.j2"
export GHCN2IODACONV="${USHgfs}/ghcn_snod2ioda.py"

export io_layout_x="{{ IO_LAYOUT_X }}"
export io_layout_y="{{ IO_LAYOUT_Y }}"
Expand Down
4 changes: 4 additions & 0 deletions dev/scripts/exglobal_snow_analysis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,10 @@
if snow_anl.task_config.DO_IMS_SCF:
snow_anl.execute('scf_to_ioda')

# Process GHCN (if applicable)
if snow_anl.task_config.DO_GHCN:
snow_anl.prepare_GHCN()

# Execute JEDI snow analysis
snow_anl.execute('snowanlvar')

Expand Down
4 changes: 4 additions & 0 deletions dev/scripts/exglobal_snowens_analysis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,10 @@
if snow_ens_anl.task_config.DO_IMS_SCF:
snow_ens_anl.execute('scf_to_ioda')

# Process GHCN (if applicable)
if snow_ens_anl.task_config.DO_GHCN:
snow_ens_anl.prepare_GHCN()

# Execute JEDI snow analysis
snow_ens_anl.execute('snowanlvar')

Expand Down
2 changes: 1 addition & 1 deletion sorc/gdas.cd
Open in desktop
Submodule gdas.cd updated 14 files
+0 −4 .gitmodules
+0 −1 bundle/CMakeLists.txt
+8 −8 ci/driver.sh
+4 −3 ci/run_ci.sh
+1 −1 parm/jcb-gdas
+1 −0 parm/snow/jcb-base.yaml.j2
+9 −0 parm/snow/prep/prep_ghcn.yaml.j2
+1 −1 parm/snow/snow_stage_ims_scf2ioda.yaml.j2
+1 −1 sorc/da-utils
+0 −1 sorc/land-imsproc
+1 −1 sorc/soca
+2 −0 test/gw-ci/CMakeLists.txt
+200 −0 ush/snow/ghcn_snod2ioda.py
+54 −2 utils/land/gdas_fv3jedi_calc_scf_to_ioda.cc
2 changes: 1 addition & 1 deletion sorc/link_workflow.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then
${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/gsi_satbias2ioda_all.sh" .
${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/snow/bufr_snocvr_snomad.py" .
${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/build/bin/imsfv3_scf2ioda.py" .
${LINK_OR_COPY} "${HOMEgfs}/sorc/gdas.cd/ush/snow/ghcn_snod2ioda.py" .
fi

#------------------------------
Expand Down Expand Up @@ -439,7 +440,6 @@ fi
# GDASApp executables
if [[ -d "${HOMEgfs}/sorc/gdas.cd/install" ]]; then
cp -f "${HOMEgfs}/sorc/gdas.cd/install/bin"/gdas* ./
cp -f "${HOMEgfs}/sorc/gdas.cd/install/bin/calcfIMS.exe" ./gdas_calcfIMS.x
cp -f "${HOMEgfs}/sorc/gdas.cd/install/bin/apply_incr.exe" ./gdas_apply_incr.x
fi

Expand Down
109 changes: 105 additions & 4 deletions ush/python/pygfs/task/snow_analysis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,14 +48,46 @@ def __init__(self, config: Dict[str, Any]):
super().__init__(config)

_res = int(self.task_config['CASE'][1:])
_fail_on_missing = str(self.task_config.fail_on_missing_snowobs[0]).lower() == "true" \
if isinstance(self.task_config.fail_on_missing_snowobs, list) \
else bool(self.task_config.fail_on_missing_snowobs)

# if 00z, do SCF preprocessing
_ims_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}imssnow96.asc')
_ims_file = os.path.join(
self.task_config.COMIN_OBS,
f'{self.task_config.OPREFIX}imssnow96.{self.task_config.ims_scf_obs_suffix}'
)
logger.info(f"Checking for IMS file: {_ims_file}")
if self.task_config.cyc == 0 and os.path.exists(_ims_file):
_DO_IMS_SCF = True
_DO_IMS_SCF = False
if self.task_config.cyc == 0:
if os.path.exists(_ims_file):
_DO_IMS_SCF = True
else:
if _fail_on_missing:
raise FileNotFoundError(
f"IMS obs file required but not found: {_ims_file}"
)
else:
logger.warning(f"IMS obs file missing: {_ims_file}")
else:
logger.info("Not 00z cycle — Skipping IMS preprocessing.")

# if 00z, do GHCN preprocessing
_ghcn_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}ghcn_snow.csv')
logger.info(f"Checking for GHCN csv file: {_ghcn_file}")
_DO_GHCN = False
if self.task_config.cyc == 0:
if os.path.exists(_ghcn_file):
_DO_GHCN = True
else:
if _fail_on_missing:
raise FileNotFoundError(
f"GHCN obs file required but not found: {_ghcn_file}"
)
else:
logger.warning(f"GHCN obs file missing: {_ghcn_file}")
else:
_DO_IMS_SCF = False
logger.info("Not 00z cycle — Skipping GHCN preprocessing.")

# Extend task_config with variables repeatedly used across this class
self.task_config.update(AttrDict(
Expand All @@ -68,6 +100,7 @@ def __init__(self, config: Dict[str, Any]):
'snow_prepobs_path': os.path.join(self.task_config.DATA, 'prep'),
'ims_file': _ims_file,
'DO_IMS_SCF': _DO_IMS_SCF, # Boolean to decide if IMS snow cover processing is done
'DO_GHCN': _DO_GHCN, # Boolean to decide if GHCN processing is done
}
))

Expand Down Expand Up @@ -222,6 +255,74 @@ def prepare_SNOCVR_SNOMAD(self) -> None:
logger.info(f"Copy {output_file} successfully generated")
FileHandler(prep_snocvr_snomad_config.netcdf).sync()

@logit(logger)
def prepare_GHCN(self) -> None:
"""Prepare the GHCN data for a global snow analysis

This method will prepare GHCN data for a global snow analysis using JEDI.
This includes:
- creating GHCN snowdepth data in IODA format.

Parameters
----------
Analysis: parent class for GDAS task

Returns
----------
None
"""

# Read and render the prep_ghcn.yaml.j2
logger.info(f"Reading {self.task_config.PREP_GHCN_YAML}")
prep_ghcn_config = parse_j2yaml(self.task_config.PREP_GHCN_YAML, self.task_config)
logger.debug(f"{self.task_config.PREP_GHCN_YAML}:\n{pformat(prep_ghcn_config)}")

# Define these locations in gdas/snow/prep/prep_ghcn.yaml.j2
logger.info("Copying GHCN obs to DATA")
FileHandler(prep_ghcn_config.stage).sync()

# Execute ioda converter to create the GHCN obs data in IODA format
logger.info("Create GHCN obs data in IODA format")

csv_file = f'{self.task_config.OPREFIX}ghcn_snow.csv'
station_file = f'ghcnd-stations.txt'
output_file = f'{self.task_config.OPREFIX}ghcn_snow.nc'
if os.path.exists(f"{os.path.join(self.task_config.DATA, output_file)}"):
rm_p(output_file)
if not os.path.isfile(csv_file):
logger.warning(f"WARNING: GHCN obs file not found.")
return

logger.info("Link GHCN2IODACONV into DATA/")
exe_src = self.task_config.GHCN2IODACONV
exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src))
if os.path.exists(exe_dest):
rm_p(exe_dest)
os.symlink(exe_src, exe_dest)

exe = Executable(exe_dest)
exe.add_default_arg(["-i", f"{os.path.join(self.task_config.DATA, csv_file)}"])
exe.add_default_arg(["-o", f"{os.path.join(self.task_config.DATA, output_file)}"])
exe.add_default_arg(["-f", f"{os.path.join(self.task_config.DATA, station_file)}"])
exe.add_default_arg(["-d", f"{to_YMDH(self.task_config.current_cycle)}"])
try:
logger.debug(f"Executing {exe}")
exe()
except OSError:
logger.exception(f"Failed to execute {exe}")
raise
except Exception as err:
logger.exception(f"An error occured during execution of {exe}")
raise WorkflowException(f"An error occured during execution of {exe}") from err

# Ensure the IODA snow depth GHCN file is produced by the IODA converter
# If so, copy to DATA/prep/
if not os.path.isfile(f"{os.path.join(self.task_config.DATA, output_file)}"):
logger.warning(f"{output_file} not produced - continuing without it.")
else:
logger.info(f"Copy {output_file} successfully generated")
FileHandler(prep_ghcn_config.ghcn2ioda).sync()

@logit(logger)
def add_increments(self) -> None:
"""Executes the program "apply_incr.exe" to create analysis "sfc_data" files by adding increments to backgrounds
Expand Down
107 changes: 103 additions & 4 deletions ush/python/pygfs/task/snowens_analysis.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,14 +51,46 @@ def __init__(self, config: Dict[str, Any]):
super().__init__(config)

_res = int(self.task_config['CASE_ENS'][1:])
_fail_on_missing = str(self.task_config.fail_on_missing_snowobs[0]).lower() == "true" \
if isinstance(self.task_config.fail_on_missing_snowobs, list) \
else bool(self.task_config.fail_on_missing_snowobs)

# if 00z, do SCF preprocessing
_ims_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}imssnow96.asc')
_ims_file = os.path.join(
self.task_config.COMIN_OBS,
f'{self.task_config.OPREFIX}imssnow96.{self.task_config.ims_scf_obs_suffix}'
)
logger.info(f"Checking for IMS file: {_ims_file}")
if self.task_config.cyc == 0 and os.path.exists(_ims_file):
_DO_IMS_SCF = True
_DO_IMS_SCF = False
if self.task_config.cyc == 0:
if os.path.exists(_ims_file):
_DO_IMS_SCF = True
else:
if _fail_on_missing:
raise FileNotFoundError(
f"IMS obs file required but not found: {_ims_file}"
)
else:
logger.warning(f"IMS obs file missing: {_ims_file}")
else:
logger.info("Not 00z cycle — Skipping IMS preprocessing.")

# if 00z, do GHCN preprocessing
_ghcn_file = os.path.join(self.task_config.COMIN_OBS, f'{self.task_config.OPREFIX}ghcn_snow.csv')
logger.info(f"Checking for GHCN csv file: {_ghcn_file}")
_DO_GHCN = False
if self.task_config.cyc == 0:
if os.path.exists(_ghcn_file):
_DO_GHCN = True
else:
if _fail_on_missing:
raise FileNotFoundError(
f"GHCN obs file required but not found: {_ghcn_file}"
)
else:
logger.warning(f"GHCN obs file missing: {_ghcn_file}")
else:
_DO_IMS_SCF = False
logger.info("Not 00z cycle — Skipping GHCN preprocessing.")

# Extend task_config with variables repeatedly used across this class
self.task_config.update(AttrDict(
Expand All @@ -69,8 +101,10 @@ def __init__(self, config: Dict[str, Any]):
'npz': self.task_config.LEVS - 1,
'CASE': self.task_config.CASE_ENS,
'snow_bkg_path': os.path.join('.', 'bkg', 'ensmean/'),
'snow_prepobs_path': os.path.join(self.task_config.DATA, 'prep'),
'ims_file': _ims_file,
'DO_IMS_SCF': _DO_IMS_SCF, # Boolean to decide if IMS snow cover processing is done
'DO_GHCN': _DO_GHCN, # Boolean to decide if GHCN processing is done
}
))

Expand Down Expand Up @@ -226,6 +260,71 @@ def prepare_SNOCVR_SNOMAD(self) -> None:
logger.info(f"Copy {output_file} successfully generated")
FileHandler(prep_snocvr_snomad_config.netcdf).sync()

@logit(logger)
def prepare_GHCN(self) -> None:
"""Prepare the GHCN data for a global snow analysis
This method will prepare GHCN data for a global snow analysis using JEDI.
This includes:
- creating GHCN snowdepth data in IODA format.
Parameters
----------
Analysis: parent class for GDAS task
Returns
----------
None
"""

# Read and render the prep_ghcn.yaml.j2
logger.info(f"Reading {self.task_config.PREP_GHCN_YAML}")
prep_ghcn_config = parse_j2yaml(self.task_config.PREP_GHCN_YAML, self.task_config)
logger.debug(f"{self.task_config.PREP_GHCN_YAML}:\n{pformat(prep_ghcn_config)}")

# Define these locations in gdas/snow/prep/prep_ghcn.yaml.j2
logger.info("Copying GHCN obs to DATA")
FileHandler(prep_ghcn_config.stage).sync()

# Execute ioda converter to create the GHCN obs data in IODA format
logger.info("Create GHCN obs data in IODA format")

csv_file = f'{self.task_config.OPREFIX}ghcn_snow.csv'
station_file = f'ghcnd-stations.txt'
output_file = f'{self.task_config.OPREFIX}ghcn_snow.nc'
if os.path.exists(f"{os.path.join(self.task_config.DATA, output_file)}"):
rm_p(output_file)
if not os.path.isfile(csv_file):
logger.warning(f"WARNING: GHCN obs file not found.")
return

logger.info("Link GHCN2IODACONV into DATA/")
exe_src = self.task_config.GHCN2IODACONV
exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src))
if os.path.exists(exe_dest):
rm_p(exe_dest)
os.symlink(exe_src, exe_dest)

exe = Executable(exe_dest)
exe.add_default_arg(["-i", f"{os.path.join(self.task_config.DATA, csv_file)}"])
exe.add_default_arg(["-o", f"{os.path.join(self.task_config.DATA, output_file)}"])
exe.add_default_arg(["-f", f"{os.path.join(self.task_config.DATA, station_file)}"])
exe.add_default_arg(["-d", f"{to_YMDH(self.task_config.current_cycle)}"])
try:
logger.debug(f"Executing {exe}")
exe()
except OSError:
logger.exception(f"Failed to execute {exe}")
raise
except Exception as err:
logger.exception(f"An error occured during execution of {exe}")
raise WorkflowException(f"An error occured during execution of {exe}") from err

# Ensure the IODA snow depth GHCN file is produced by the IODA converter
# If so, copy to DATA/prep/
if not os.path.isfile(f"{os.path.join(self.task_config.DATA, output_file)}"):
logger.warning(f"{output_file} not produced - continuing without it.")
else:
logger.info(f"Copy {output_file} successfully generated")
FileHandler(prep_ghcn_config.ghcn2ioda).sync()

@logit(logger)
def add_increments(self) -> None:
"""Executes the program "apply_incr.exe" to create analysis "sfc_data" files by adding increments to backgrounds
Expand Down
Loading