Changes in Release/3.0.3

bin/ExternalAnalysisToMPAS.csh

@@ -98,6 +98,7 @@ ln -sfv ${ExternalAnalysesDir}/${externalanalyses__UngribPrefix}* ./
 ## link MPAS mesh graph info
 rm ./x${ArgRatio}.${ArgNCells}.graph.info*
 ln -sfv $GraphInfoDir/x${ArgRatio}.${ArgNCells}.graph.info* .
+echo $GraphInfoDir/x${ArgRatio}.${ArgNCells}.graph.info
 
 ## Link MPAS invariant field
 if ( $ArgType == "Outer" ) then

bin/GetGDASAnalysisFromFTP.csh

@@ -45,7 +45,8 @@ set fhour = 000
 
 echo "Getting GDAS atm and sfc analyses from the NCEP FTP"
 # url for GDAS data
-set gdas_ftp = https://www.ftp.ncep.noaa.gov/data/nccf/com/gfs/prod/gdas.${ccyymmdd}/${hh}/atmos
+#set gdas_ftp = https://www.ftp.ncep.noaa.gov/data/nccf/com/gfs/prod/gdas.${ccyymmdd}/${hh}/atmos
+set gdas_ftp = https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gdas.${ccyymmdd}/${hh}/atmos
 set gdasAnaInfix = (atm sfc sfluxgrb)
 
 foreach anaInfix ($gdasAnaInfix)

bin/GetGFSAnalysisFromFTP.csh

@@ -43,7 +43,8 @@ set hh = `echo ${thisValidDate} | cut -c 9-10`
 set res = 0p25
 set fhour = 000
 # url for GFS data
-set gfs_ftp = https://www.ftp.ncep.noaa.gov/data/nccf/com/gfs/prod/gfs.${ccyymmdd}/${hh}/atmos
+#set gfs_ftp = https://www.ftp.ncep.noaa.gov/data/nccf/com/gfs/prod/gfs.${ccyymmdd}/${hh}/atmos
+set gfs_ftp = https://nomads.ncep.noaa.gov/pub/data/nccf/com/gfs/prod/gfs.${ccyymmdd}/${hh}/atmos
 set gribFile = gfs.t${hh}z.pgrb2.${res}.f${fhour}
 
 set WorkDir = ${ExperimentDirectory}/`echo "$ArgWorkDir" \

bin/GetGFSAnalysisFromRDA.csh

@@ -71,7 +71,7 @@ rm -rf GRIBFILE.*
 
 echo "Getting GFS analysis from RDA"
 # RDA GFS forecasts directory
-set GFSgribdirRDA = /glade/campaign/collections/rda/data/ds084.1
+set GFSgribdirRDA = /glade/campaign/collections/rda/data/d084001
 
 if ( ! -e ${GFSgribdirRDA}/${gribFile} ) then
    set preVyymmdd = `echo ${prevValidDate} | cut -c 1-8`

bin/GetObs.csh

@@ -54,9 +54,9 @@ cd ${self_WorkDir}
 # ================================================================================================
 
 set dataRoot = /glade/campaign/collections
-set defaultBUFRDirectory = $dataRoot//rda/data/ds735.0
-set satwndBUFRDirectory = $dataRoot/rda/data/ds351.0
-set PrepBUFRDirectory = $dataRoot/rda/data/ds337.0
+set defaultBUFRDirectory = $dataRoot/rda/data/d735000
+set satwndBUFRDirectory = $dataRoot/rda/data/d351000
+set PrepBUFRDirectory = $dataRoot/rda/data/d337000
 
 foreach inst ( ${convertToIODAObservations} )
   if ( "${observations__resource}" == "GladeRDAOnline" ) then
@@ -68,13 +68,6 @@ foreach inst ( ${convertToIODAObservations} )
           echo "Source file: ${satwndBUFRDirectory}/bufr/${ccyy}/${THIS_FILE}"
           cp -p ${satwndBUFRDirectory}/bufr/${ccyy}/${THIS_FILE} .
        endif
-       # link the GDAS observation error table
-       if ( -e ${GDASObsErrtable} ) then
-          ln -sf ${GDASObsErrtable} obs_errtable
-       else
-          echo "ERROR: ${GDASObsErrtable} does NOT exist" > ./FAIL
-          exit 1
-       endif
     # for prepbufr observations
     else if ( ${inst} == prepbufr ) then
        setenv THIS_FILE prepbufr.gdas.${ccyymmdd}.t${hh}z.nr.48h
@@ -88,14 +81,6 @@ foreach inst ( ${convertToIODAObservations} )
              cp -p ${PrepBUFRDirectory}/prepnr/${ccyy}/${THIS_FILE} .
           endif
        endif
-       # use obs errors embedded in prepbufr file
-       if ( -e obs_errtable ) then
-          rm -f obs_errtable
-       endif
-       # use external obs error table
-       #if ( -e ${GDASObsErrtable} ) then
-       #   ln -sf ${GDASObsErrtable} obs_errtable
-       #endif
     # for all other observations
     else
        # set the specific file to be extracted from the tar file
@@ -155,7 +140,8 @@ foreach inst ( ${convertToIODAObservations} )
   else if ( "${observations__resource}" == "NCEPFTPOnline" ) then
     echo "Getting ${inst} from the NCEP FTP"
     # url for GDAS data
-    set gdas_ftp = https://ftpprd.ncep.noaa.gov/data/nccf/com/obsproc/prod/gdas.${ccyymmdd}
+    #set gdas_ftp = https://ftpprd.ncep.noaa.gov/data/nccf/com/obsproc/prod/gdas.${ccyymmdd}
+    set gdas_ftp = https://nomads.ncep.noaa.gov/pub/data/nccf/com/obsproc/prod/gdas.${ccyymmdd}
     # set name for the observation type
     if ( ${inst} == prepbufr ) then
       set THIS_FILE = gdas.t${hh}z.${inst}.nr

bin/ObsToIODA.csh

@@ -5,7 +5,7 @@
 # This software is licensed under the terms of the Apache Licence Version 2.0
 # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
 
-#Convert CISL RDA archived NCEP BUFR files to IODA-v2 format based on Jamie Bresch (NCAR/MMM) script rda_obs2ioda.csh
+#Convert CISL RDA archived NCEP BUFR files to IODA-v3 format based on Jamie Bresch (NCAR/MMM) script rda_obs2ioda.csh
 
 # Process arguments
 # =================
@@ -79,7 +79,7 @@ foreach gdasfile ( *"gdas."* )
    if ( ${gdasfile} =~ *"cris"* && ${ccyy} >= '2021' ) then
      ln -sf ${CRTMTABLES}/cris-fsr431_npp.SpcCoeff.bin  ./cris_npp.SpcCoeff.bin
      ln -sf ${CRTMTABLES}/cris-fsr431_n20.SpcCoeff.bin  ./cris_n20.SpcCoeff.bin
-     #ln -sf ${CRTMTABLES}/cris-fsr431_n21.SpcCoeff.bin  ./cris_n21.SpcCoeff.bin
+     ln -sf ${CRTMTABLES}/cris-fsr431_n21.SpcCoeff.bin  ./cris_n21.SpcCoeff.bin
    else if ( ${gdasfile} =~ *"cris"* && ${ccyy} < '2021' ) then
      ln -sf ${CRTMTABLES}/cris399_npp.SpcCoeff.bin  ./cris_npp.SpcCoeff.bin
      ln -sf ${CRTMTABLES}/cris399_n20.SpcCoeff.bin  ./cris_n20.SpcCoeff.bin
@@ -89,7 +89,7 @@ foreach gdasfile ( *"gdas."* )
      ln -sf ${CRTMTABLES}/iasi616_metop-c.SpcCoeff.bin  ./iasi_metop-c.SpcCoeff.bin
    endif
 
-   # Run the obs2ioda executable to convert files from BUFR to IODA-v2
+   # Run the obs2ioda executable to convert files from BUFR to IODA-v3
    # ==================
    rm ./${obs2iodaEXE}
    ln -sfv ${obs2iodaBuildDir}/${obs2iodaEXE} ./
@@ -102,6 +102,10 @@ foreach gdasfile ( *"gdas."* )
      #./${obs2iodaEXE} ${SPLIThourly} ${gdasfile} >&! $log
      ./${obs2iodaEXE} ${gdasfile} >&! $log
    else if ( ${gdasfile} =~ *"prepbufr"* ) then
+     # use obs errors embedded in prepbufr file
+     if ( -e obs_errtable ) then
+       rm -f obs_errtable
+     endif
      set inst = `echo "$gdasfile" | cut -d'.' -f1`
      # run obs2ioda for preburf with additional QC as in GSI
      ./${obs2iodaEXE} ${gdasfile} >&! $log
@@ -114,6 +118,15 @@ foreach gdasfile ( *"gdas."* )
      mv -f sfc_obs_${thisCycleDate}.h5 ../sfc_obs_${thisCycleDate}.h5
      cd ..
      rm -rf sfc
+   else if ( ${gdasfile} =~ *"satwnd"* ) then
+     # link the GDAS observation error table
+     if ( -e ${GDASObsErrtable} ) then
+       ln -sf ${GDASObsErrtable} obs_errtable
+     else
+       echo "ERROR: ${GDASObsErrtable} does NOT exist" > ./FAIL
+       exit 1
+     endif
+     ./${obs2iodaEXE} ${gdasfile} >&! $log
    else
      ./${obs2iodaEXE} ${gdasfile} >&! $log
    endif
@@ -122,7 +135,7 @@ foreach gdasfile ( *"gdas."* )
    # ============
    grep "all done!" $log
    if ( $status != 0 ) then
-     echo "$0 (ERROR): Pre-processing observations to IODA-v2 failed" > ./FAIL-converter_${inst}
+     echo "$0 (ERROR): Pre-processing observations to IODA-v3 failed" > ./FAIL-converter_${inst}
      exit 1
    endif
 
@@ -132,163 +145,33 @@ foreach gdasfile ( *"gdas."* )
 end # gdasfile loop
 
 # need to change to mainScriptDir in order for environmentJEDI.csh to be sourced
-cd ${mainScriptDir}
-source config/environmentJEDI.csh
-cd -
-
-# upgrade from IODA v1 to v2 (convert char to string for /MetaData/stationIdentification & /MetaData/variable_names)
-
-set V1toV2 = ( \
-    aircraft \
-    satwind \
-    satwnd \
-    sfc \
-    sondes \
-    ascat \
-    profiler \
-)
-
-if ( "${convertToIODAObservations}" =~ *"prepbufr"* || "${convertToIODAObservations}" =~ *"satwnd"* ) then
-  # Run the ioda-upgrade executable to upgrade to get string station_id and string variable_names
-  # ==================
-
-  #foreach exec ($iodaUpgradeEXE1 $iodaUpgradeEXE2)
-  #  rm ./${exec}
-  #  ln -sfv ${iodaUpgradeBuildDir}/${exec} ./
-  #end
-  rm -f $iodaUpgradeEXE1
-  ln -sfv ${iodaUpgradeBuildDir}/${iodaUpgradeEXE1} ./
-
-  foreach ty ( ${V1toV2} )
-    echo 'begin ioda-upgrade-v1-to-v2 ' $ty
-    if ( -f ${ty}_obs_${thisValidDate}.h5 ) then
-      set ty_obs = ${ty}_obs_${thisValidDate}.h5
-      set ty_obs_base = `echo "$ty_obs" | cut -d'.' -f1`
-
-      set log = logs/log-upgradeV1-to-V2_${ty}
-      rm $log
-
-      ./${iodaUpgradeEXE1} ${ty_obs} ${ty_obs_base}_tmp.h5 >&! $log
-      rm -f ${ty_obs}
-      mv ${ty_obs_base}_tmp.h5 ${ty_obs}
-
-      # Check status
-      # ============
-      grep "Success!" $log
-      if ( $status != 0 ) then
-        echo "$0 (ERROR): ${exec} failed for $ty" > ./FAIL-upgradeV1-to-V2_${ty}
-        exit 1
-      endif
-    endif
-  end
+#cd ${mainScriptDir}
+#source config/environmentJEDI.csh
+#cd -
 
-endif
+if ( "${convertToIODAObservations}" =~ *"cris"* ) then
 
-# upgrade for IODA v2 to v3
-
-set V2toV3 = ( $V1toV2 \
-    amsua_n15 \
-    amsua_n18 \
-    amsua_n19 \
-    amsua_aqua \
-    amsua_metop-a \
-    amsua_metop-b \
-    amsua_metop-c \
-    gnssro \
-    mhs_n18 \
-    mhs_n19 \
-    mhs_metop-a \
-    mhs_metop-b \
-    mhs_metop-c \
-    iasi_metop-a \
-    iasi_metop-b \
-    iasi_metop-c \
+# Name update
+set NameUpdate = ( \
     cris_npp \
     cris_n20 \
-    #cris_n21 \
+    cris_n21 \
 )
 
-set iodaUpgradeV3Config = ${ConfigDir}/jedi/obsProc/ObsSpaceV2-to-V3.yaml
-rm -f $iodaUpgradeEXE2
-ln -sfv ${iodaUpgradeBuildDir}/${iodaUpgradeEXE2} ./
-
-foreach ty ( ${V2toV3} )
-  echo 'begin ioda-upgrade-v2-to-v3 ' $ty
+foreach ty ( ${NameUpdate} )
+  echo 'begin NameUpdate' $ty
   if ( -f ${ty}_obs_${thisValidDate}.h5 ) then
-
-    set ty_obs = ${ty}_obs_${thisValidDate}.h5
-    set ty_obs_base = `echo "$ty_obs" | cut -d'.' -f1`
-
-    set log = logs/log-upgradeV2-to-V3_${ty}
-    rm $log
-
-    ./${iodaUpgradeEXE2} ${ty_obs} ${ty_obs_base}_tmp.h5 $iodaUpgradeV3Config >&! $log
-    rm -f ${ty_obs}
-    mv ${ty_obs_base}_tmp.h5 $ty_obs
-
-    # Check status
-    # ============
-    grep "Success!" $log
-    if ( $status != 0 ) then
-      echo "$0 (ERROR): ${exec} failed for $ty" > ./FAIL-upgradeV2-to-V3_${ty}
-      exit 1
-    endif
-
-  endif
-end
-
-# upgrade for sensorScanPosition
-
-set ScanPositionUpdate = ( \
-    amsua_n15 \
-    amsua_n18 \
-    amsua_n19 \
-    amsua_aqua \
-    amsua_metop-a \
-    amsua_metop-b \
-    amsua_metop-c \
-    mhs_n18 \
-    mhs_n19 \
-    mhs_metop-a \
-    mhs_metop-b \
-    mhs_metop-c \
-    iasi_metop-a \
-    iasi_metop-b \
-    iasi_metop-c \
-    cris_npp \
-    cris_n20 \
-    #cris_n21 \
-)
-
-ln -fs ${pyDir}/update_sensorScanPosition.py .
-ln -fs ${pyDir}/fix_float2int.py .
-
-foreach ty ( ${ScanPositionUpdate} )
-  echo 'begin ScanPositionUpdate' $ty
-
-  if ( -f ${ty}_obs_${thisValidDate}.h5 ) then
-
-    set ty_obs = ${ty}_obs_${thisValidDate}.h5
-    setenv fname ${ty_obs}
-
-    set log = logs/log-update_sensorScanPosition_${ty}
-    rm $log
-
-    python update_sensorScanPosition.py
-
-    if ( ${ty} =~ *"cris"* && ${ccyy} >= 2021 ) then
+    #if ( ${ty} =~ *"cris"* && ${ccyy} >= 2021 ) then
+    if ( ${ccyy} >= 2021 ) then
        if ( ${ty} == "cris_npp" ) set tyy = "cris-fsr_npp"
        if ( ${ty} == "cris_n20" ) set tyy = "cris-fsr_n20"
-       #if ( ${ty} == "cris_n21" ) set tyy = "cris-fsr_n21"
-       mv -f ${ty_obs}.modified ${tyy}_obs_${thisValidDate}.h5
-    else
-       mv -f ${ty_obs}.modified ${ty_obs}
+       if ( ${ty} == "cris_n21" ) set tyy = "cris-fsr_n21"
+       mv -f ${ty}_obs_${thisValidDate}.h5 ${tyy}_obs_${thisValidDate}.h5
     endif
-
   endif
-
-  echo 'end of ScanPositionUpdate' $ty
+  echo 'end of NameUpdate' $ty
 end
+endif
 
 date