Enable EnKF-only for atmosphere

dev/job_cards/rocoto/prep.sh

@@ -150,6 +150,19 @@ export COMINgfs=${COMIN_ATMOS_HISTORY_GFS}
 
 export COMSP=${COMSP:-"${COMIN_OBS}/${RUN_local}.t${cyc}z."}
 
+if [[ ${DOENKFONLY_ATM:-"NO"} == "YES" ]]; then
+    MEMDIR="ensstat" RUN="enkf${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx COMIN_ATMOS_HISTORY_ENS_STAT_PREV:COM_ATMOS_HISTORY_TMPL
+    MEMDIR="mem001" RUN="enkf${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx COMIN_ATMOS_HISTORY_ENS_MEM001_PREV:COM_ATMOS_HISTORY_TMPL
+    RUN="gdas" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx COMOUT_ATMOS_HISTORY_DET_PREV:COM_ATMOS_HISTORY_TMPL
+    mkdir -p "${COMOUT_ATMOS_HISTORY_DET_PREV}"
+    ln -sf "${COMIN_ATMOS_HISTORY_ENS_MEM001_PREV}/enkfgdas.t${gcyc}z.log.f003.txt" "${COMOUT_ATMOS_HISTORY_DET_PREV}/gdas.t${gcyc}z.log.f003.txt"
+    ln -sf "${COMIN_ATMOS_HISTORY_ENS_MEM001_PREV}/enkfgdas.t${gcyc}z.log.f006.txt" "${COMOUT_ATMOS_HISTORY_DET_PREV}/gdas.t${gcyc}z.log.f006.txt"
+    ln -sf "${COMIN_ATMOS_HISTORY_ENS_MEM001_PREV}/enkfgdas.t${gcyc}z.log.f009.txt" "${COMOUT_ATMOS_HISTORY_DET_PREV}/gdas.t${gcyc}z.log.f009.txt"
+    ln -sf "${COMIN_ATMOS_HISTORY_ENS_STAT_PREV}/enkfgdas.t${gcyc}z.ensmean.atm.f003.nc" "${COMOUT_ATMOS_HISTORY_DET_PREV}/gdas.t${gcyc}z.atm.f003.nc"
+    ln -sf "${COMIN_ATMOS_HISTORY_ENS_STAT_PREV}/enkfgdas.t${gcyc}z.ensmean.atm.f006.nc" "${COMOUT_ATMOS_HISTORY_DET_PREV}/gdas.t${gcyc}z.atm.f006.nc"
+    ln -sf "${COMIN_ATMOS_HISTORY_ENS_STAT_PREV}/enkfgdas.t${gcyc}z.ensmean.atm.f009.nc" "${COMOUT_ATMOS_HISTORY_DET_PREV}/gdas.t${gcyc}z.atm.f009.nc"
+fi
+
 # Create or Copy prepbufr, prepbufr.acft_profiles, nsstbufr files
 # Do not fail on external errors
 if [[ ${MAKE_PREPBUFR:-"YES"} == "YES" ]]; then

dev/job_cards/rocoto/prepatmanlbias.sh

@@ -0,0 +1,18 @@
+#! /usr/bin/env bash
+
+###############################################################
+# Source UFSDA workflow modules
+source "${HOMEgfs}/dev/ush/load_modules.sh" ufsda
+status=$?
+if [[ ${status} -ne 0 ]]; then
+    exit "${status}"
+fi
+
+export job="prepatmanlbias"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/dev/jobs/JGLOBAL_ATM_PREP_ANL_BIAS"
+status=$?
+exit "${status}"

dev/jobs/JGLOBAL_ATMENS_ANALYSIS_INITIALIZE

@@ -19,8 +19,13 @@ GDUMP="gdas"
 RUN=${GDUMP} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
     COMIN_OBS:COM_OBS_TMPL
 
-RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
-    COMIN_ATMOS_ANALYSIS_PREV:COM_ATMOS_ANALYSIS_TMPL
+if [[ ${DOENKFONLY_ATM:-"NO"} == "YES" ]]; then
+    MEMDIR='ensstat' RUN="enkf${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
+        COMIN_ATMOS_ANALYSIS_PREV:COM_ATMOS_ANALYSIS_TMPL
+else
+    RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
+        COMIN_ATMOS_ANALYSIS_PREV:COM_ATMOS_ANALYSIS_TMPL
+fi
 
 ###############################################################
 # Run relevant script

dev/jobs/JGLOBAL_ATM_PREP_ANL_BIAS

@@ -0,0 +1,129 @@
+#! /usr/bin/env bash
+export WIPE_DATA="NO"
+export DATA=${DATA:-${DATAROOT}/${RUN}fetch_${cyc}}
+source "${HOMEgfs}/ush/jjob_header.sh" -e "prepatmanlbias" -c "base prepatmanlbias"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+# Ignore possible spelling error (nothing is misspelled)
+# shellcheck disable=SC2153
+GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
+export gPDY=${GDATE:0:8}
+export gcyc=${GDATE:8:2}
+export gyy=${GDATE:0:4}
+export gmm=${GDATE:4:2}
+export gdd=${GDATE:6:2}
+export GDUMP="gdas"
+
+# Generate COM variables from templates
+MEMDIR="ensstat" RUN="enkf${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
+    COMOUT_ATMOS_ANALYSIS_PREV:COM_ATMOS_ANALYSIS_TMPL
+
+# Process bias corrections files grabbed from HPSS in task gdas_fetchatmanlbias
+BIASDIR=${DATA}/${GDUMP}.${gPDY}/${gcyc}/atmos
+BIASFILES="${GDUMP}.t${gcyc}z.abias ${GDUMP}.t${gcyc}z.abias_air  ${GDUMP}.t${gcyc}z.abias_int ${GDUMP}.t${gcyc}z.abias_pc ${GDUMP}.t${gcyc}z.radstat"
+
+ABIAS_SAT=${BIASDIR}/${GDUMP}.t${gcyc}z.abias
+ABIASPC_SAT=${BIASDIR}/${GDUMP}.t${gcyc}z.abias_pc
+ABIAS_SAT_JEDI_TAR=${COMOUT_ATMOS_ANALYSIS_PREV}/${GDUMP}.t${gcyc}z.rad_varbc_params.tar
+ABIAS_AIR_JEDI_TAR=${COMOUT_ATMOS_ANALYSIS_PREV}/${GDUMP}.t${gcyc}z.air_varbc_params.tar
+
+SATBIAS2IODAX=${SATBIAS2IODAX:-${EXECgfs}/satbias2ioda.x}
+SATBIAS2IODAY=${HOMEgfs}/sorc/gdas.cd/ush/satbias_converter.yaml.tmpl
+OUTPUT=${DATA}/output
+
+if [[ ! -d "${COMOUT_ATMOS_ANALYSIS_PREV}" ]]; then mkdir -p "${COMOUT_ATMOS_ANALYSIS_PREV}"; fi
+if [[ ! -d "${OUTPUT}" ]]; then mkdir -p "${OUTPUT}"; fi
+
+if [[ ${DO_JEDIATMENS} == "YES" ]]; then
+    # JEDI run
+    cd "${DATA}" || exit
+    if [[ ! -d "testrun/varbc" ]]; then mkdir -p "testrun/varbc"; fi
+    /bin/ln -sf "${ABIAS_SAT}" "./satbias_in"
+    /bin/ln -sf "${ABIASPC_SAT}" "./satbias_pc"
+    grep -i "NaN" satbias_in && echo "Stop. There are NaN in ${ABIAS}." && exit 1
+
+    # Get instruments from satbias_in
+    obsclass=$(grep "_" satbias_in | awk '{print $2}' | uniq)
+
+    # Loop over instruments. Covert GSI abias to JEDI format
+    for instrument in ${obsclass}; do
+        echo "${instrument}"
+        /bin/cp -f "${SATBIAS2IODAY}" satbias_converter.yaml
+        sed -i -e "s/INSTRUMENT/${instrument}/g" satbias_converter.yaml
+
+        ${SATBIAS2IODAX} satbias_converter.yaml
+
+        export err=$?
+        if [[ ${err} -ne 0 ]]; then
+            err_exit "satbias2iodas.x failed for ${instrument}, ABORT!"
+        fi
+
+        /bin/rm -f testrun/varbc/*nc
+        /bin/rm -f testrun/varbc/*txt
+
+        cd ./testrun/varbc/
+        grep "${instrument}" ../../satbias_in | awk '{print $2" "$3" "$4}' > \
+            "${OUTPUT}"/gdas.t"${gcyc}"z."${instrument}".tlapse.txt
+        /bin/cp -p satbias_"${instrument}".nc4 "${OUTPUT}"/gdas.t"${gcyc}"z."${instrument}".satbias.nc
+        /bin/mv satbias_"${instrument}".nc4 "${OUTPUT}"/gdas.t"${gcyc}"z."${instrument}".satbias_cov.nc
+        cd "${DATA}"
+        /bin/rm -rf satbias_converter.yaml
+    done
+
+    if [[ ${err} -eq 0 ]]; then
+        cd "${OUTPUT}" || exit
+        for file in ${BIASFILES}; do
+            cpreq "${BIASDIR}/${file}" "./enkf${file}.txt"
+            err=$?
+            if [[ ${err} -ne 0 ]]; then
+                err_exit "Error copying operational anl bias correction files"
+            fi
+        done
+        # TODO: Temporary solution and need to process abias_air for JEDI when ready
+        tar -cvf "${ABIAS_AIR_JEDI_TAR}" "./enkf${GDUMP}.t${gcyc}z.abias_air.txt"
+        rm -rf "./enkf${GDUMP}.t${gcyc}z.abias_air.txt"
+        tar -cvf "${ABIAS_SAT_JEDI_TAR}" ./*.nc ./*.txt
+        err=$?
+    fi
+
+else
+    # GSI run
+    for file in ${BIASFILES}; do
+        cpreq "${BIASDIR}/${file}" "${COMOUT_ATMOS_ANALYSIS_PREV}/enkf${file}.txt"
+        err=$?
+        if [[ ${err} -ne 0 ]]; then
+            err_exit "Error copying operational anl bias correction files"
+        fi
+    done
+fi
+
+if [[ ${err} -ne 0 ]]; then
+    err_exit "Error executing ${EXSCRIPT}"
+fi
+set_trace
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]]; then
+    cat "${pgmout}"
+fi
+
+###################################
+# Remove temp directories
+###################################
+if [[ "${KEEPDATA}" != "YES" ]]; then
+    rm -rf "${DATA}"
+fi
+
+exit 0

dev/jobs/JGLOBAL_ENKF_SELECT_OBS

@@ -65,17 +65,31 @@ if [[ ${DONST} == "YES" ]]; then
 fi
 export PREPQCPF="${COMIN_OBS}/${OPREFIX}prepbufr.acft_profiles"
 
-# Deterministic analysis and increment files
-export SFCANL="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}analysis.sfc.a006.nc"
-export DTFANL="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}increment.dtf.i006.nc"
-export ATMANL="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}analysis.atm.a006.nc"
-export ATMINC="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}increment.atm.i006.nc"
-
-# Guess Bias correction coefficients related to control
-export GBIAS=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias.txt
-export GBIASPC=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_pc.txt
-export GBIASAIR=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_air.txt
-export GRADSTAT=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}radstat.tar
+if [[ ${DOENKFONLY_ATM:-"NO"} == "YES" ]]; then
+    # Ensemble analysis and increment files
+    export SFCANL="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}analysis.sfc.a006.nc"
+    export DTFANL="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.dtf.i006.nc"
+    export ATMANL="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}analysis.atm.a006.nc"
+    export ATMINC="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}increment.atm.i006.nc"
+
+    # Guess Bias correction coefficients related to ensemble
+    export GBIAS="${COMIN_ATMOS_ANALYSIS_PREV}/${GPREFIX}abias.txt"
+    export GBIASPC="${COMIN_ATMOS_ANALYSIS_PREV}/${GPREFIX}abias_pc.txt"
+    export GBIASAIR="${COMIN_ATMOS_ANALYSIS_PREV}/${GPREFIX}abias_air.txt"
+    export GRADSTAT="${COMIN_ATMOS_ANALYSIS_PREV}/${GPREFIX}radstat.tar"
+else
+    # Deterministic analysis and increment files
+    export SFCANL="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}analysis.sfc.a006.nc"
+    export DTFANL="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}increment.dtf.i006.nc"
+    export ATMANL="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}analysis.atm.a006.nc"
+    export ATMINC="${COMOUT_ATMOS_ANALYSIS_DET}/${APREFIX_DET}increment.atm.i006.nc"
+
+    # Guess Bias correction coefficients related to control
+    export GBIAS="${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias.txt"
+    export GBIASPC="${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_pc.txt"
+    export GBIASAIR="${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_air.txt"
+    export GRADSTAT="${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}radstat.tar"
+fi
 
 # Bias correction coefficients related to ensemble mean
 export ABIAS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias.txt"

dev/jobs/JGLOBAL_FETCH

@@ -1,5 +1,5 @@
 #! /usr/bin/env bash
-
+export DATA=${DATA:-${DATAROOT}/${RUN}fetch_${cyc}}
 source "${HOMEgfs}/ush/jjob_header.sh" -e "fetch" -c "base fetch"
 
 # Execute fetching

dev/parm/config/gfs/config.atmensanl.j2

@@ -16,5 +16,6 @@ export io_layout_x="{{ IO_LAYOUT_X }}"
 export io_layout_y="{{ IO_LAYOUT_Y }}"
 
 export TASK_CONFIG_YAML="${PARMgfs}/gdas/atm/atm_ens_config.yaml.j2"
+export OBS_DIST_LOCALIZATIONS_YAML="${PARMgfs}/gdas/atm/atm_ens_obs_dist_localizations.yaml.j2"
 
 echo "END: config.atmensanl"

dev/parm/config/gfs/config.base.j2

@@ -398,6 +398,7 @@ export BUILD_GSINFO_DIR="${PARMgfs}/gsinfo"
 export NMEM_ENS_GFS="{{ NMEM_ENS_GFS }}"
 export NMEM_ENS_GFS_OFFSET="{{ NMEM_ENS_GFS_OFFSET }}"
 export DO_CALC_INCREMENT_ENKF_GFS="NO"
+export DOENKFONLY_ATM="{{ DOENKFONLY_ATM }}"
 
 # EnKF output frequency
 if [[ "${DOHYBVAR}" = "YES" ]]; then
@@ -521,6 +522,17 @@ if [[ "${machine}" == "URSA" || "${machine}" == "GAEAC6" || "${machine}" == "ORI
     export DO_METP=NO
 fi
 
+if [[ "${DOENKFONLY_ATM:-NO}" == "YES" ]] ; then
+    export RECENTER_ENKF="NO"   # Turn off recentering ensemble analysis
+    export DO_VERFOZN="NO"      # Ozone data assimilation monitoring
+    export DO_VERFRAD="NO"      # Radiance data assimilation monitoring
+    export DO_VMINMON="NO"      # GSI minimization monitoring
+    export DO_METP="NO"   	# Run METPLUS jobs - set METPLUS settings in config.metp
+    export DO_FIT2OBS="NO"      # Run fit to observations package
+    export DO_TRACKER="NO"	# Hurricane track verification
+    export DO_GENESIS="NO"      # Cyclone genesis verification
+fi
+
 # If starting ICs that are not at cycle hour
 export OFFSET_START_HOUR=0
 

dev/parm/config/gfs/config.esfc

@@ -26,6 +26,11 @@ if [[ "${DO_JEDIATMENS}" == "YES" || "${DO_JEDIATMVAR}" == "YES" ]]; then
    export DONST="NO"
 fi
 
+# Turn off NST if DOENKFONLY_ATM is YES
+if [[ "${DOENKFONLY_ATM:-NO}" == "YES" ]] ; then
+   export DONST="NO"
+fi
+
 if [[ "${RUN/enkf}" == "gfs" ]]; then  
     echo "turning off gsi soilda for gfs run"
     DO_GSISOILDA="NO"

dev/parm/config/gfs/config.fetch

@@ -7,13 +7,31 @@ echo "BEGIN: config.fetch"
 # Get task specific resources
 source "${EXPDIR}/config.resources" fetch
 
-# Determine start type
-if [[ "${EXP_WARM_START}" == ".false." ]]; then
-   ic_type="cold"
+if [[ "${PDY}${cyc}" -gt "${SDATE}" ]]; then
+    # fetch based on the second and following cycle
+    if [[ ${DOENKFONLY_ATM} == "YES" ]]; then
+        # determine GDAS version based on date
+       if [[ "${PDY}${cyc}" -ge "2022112900" ]]; then
+           gdas_version="v16.3"
+       elif [[ "${PDY}${cyc}" -ge "2022062700" ]]; then
+           gdas_version="v16.2"
+       else
+           gdas_version="prod"
+       fi
+       export gdas_version
+       export FETCH_YAML_TMPL="${PARMgfs}/fetch/${NET}_${APP}_gdas-anl-bias.yaml.j2"
+       export KEEPDATA="YES"
+    fi
 else
-   ic_type="warm"
-fi
+    # fetch based on first cycle
+    # Determine start type
+    if [[ "${EXP_WARM_START}" == ".false." ]]; then
+        ic_type="cold"
+    else
+       ic_type="warm"
+    fi
 
-export FETCH_YAML_TMPL="${PARMgfs}/fetch/${NET}_${APP}_${ic_type}_${MODE}.yaml.j2"
+    export FETCH_YAML_TMPL="${PARMgfs}/fetch/${NET}_${APP}_${ic_type}_${MODE}.yaml.j2"
+fi
 
 echo "END: config.fetch"

dev/parm/config/gfs/config.prep.j2

@@ -15,6 +15,11 @@ export PROCESS_TROPCY=${PROCESS_TROPCY:-NO}
 export TROPCYQCRELOSH="${SCRgfs}/exglobal_atmos_tropcy_qc_reloc.sh"
 export COMINsyn=${COMINsyn:-$(compath.py "${envir}/com/gfs/${gfs_ver}")/syndat}
 
+# If DOENKFONLY_ATM="YES", skip PROCESS_TROPCY
+if [[ "${DOENKFONLY_ATM:-NO}" == "YES" ]] ; then
+   export PROCESS_TROPCY="NO"
+fi
+
 # Allow users to control the generation or use of either operational or
 # their processed prepbufr, prepbufr.acft_profiles, nsstbufr files
 export MAKE_PREPBUFR="YES"  # Generate prepbufr, etc. files by executing obsproc

dev/parm/config/gfs/config.prepatmanlbias

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+########## config.prepatmiodaobs ##########
+# Atm Operational Anl Bias Correction specific
+
+echo "BEGIN: config.prepatmanlbias"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" prepatmanlbias
+
+echo "END: config.prepatmanlbias"

dev/parm/config/gfs/config.resources

@@ -12,7 +12,7 @@ if (( $# != 1 )); then
     echo "Must specify an input task argument to set resource variables!"
     echo "argument can be any one of the following:"
     echo "stage_ic aerosol_init fetch"
-    echo "prep prep_sfc"
+    echo "prep prep_sfc prepatmanlbias"
     echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal"
     echo "atmensanlinit atmensanlobs atmensanlsol atmensanlletkf atmensanlfv3inc atmensanlfinal ecen_fv3jedi analcalc_fv3jedi"
     echo "snowanl esnowanl"
@@ -115,6 +115,13 @@ case ${step} in
     memory="${mem_node_max}"
     ;;
 
+  "prepatmanlbias")
+    walltime="00:30:00"
+    ntasks=1
+    threads_per_task=1
+    tasks_per_node=1
+    ;;
+
   "aerosol_init")
     walltime="00:05:00"
     ntasks=1

dev/parm/config/gfs/yaml/defaults.yaml

@@ -24,6 +24,7 @@ base:
   FHMAX_ENKF_GFS: 12
   DOHYBVAR_OCN: "NO"
   DOLETKF_OCN: "NO"
+  DOENKFONLY_ATM: "NO"
   DO_TEST_MODE: "NO"
   NMEM_ENS_GFS: 30
   NMEM_ENS_GFS_OFFSET: 20

dev/scripts/exglobal_enkf_earc_tars.py

@@ -28,7 +28,8 @@ def main():
             'DOHYBVAR', 'DOIAU_ENKF', 'IAU_OFFSET', 'DOIAU', 'DO_CA',
             'DO_CALC_INCREMENT', 'assim_freq', 'ARCH_CYC', 'DO_JEDISNOWDA',
             'ARCH_WARMICFREQ', 'ARCH_FCSTICFREQ', 'DOHYBVAR_OCN',
-            'DOLETKF_OCN', 'IAUFHRS_ENKF', 'NET', 'NMEM_ENS_GFS', 'DO_GSISOILDA', 'DO_LAND_IAU']
+            'DOLETKF_OCN', 'IAUFHRS_ENKF', 'NET', 'NMEM_ENS_GFS', 'DO_GSISOILDA', 'DO_LAND_IAU',
+            'DOENKFONLY_ATM']
 
     archive_dict = AttrDict()
     for key in keys: