Updates for building with containers and with JEDI/LETKF-inline

.gitmodules

@@ -7,22 +7,28 @@
   url = https://github.com/NOAA-EMC/wxflow
 [submodule "sorc/gfs_utils.fd"]
   path = sorc/gfs_utils.fd
-  url = https://github.com/NOAA-EMC/gfs-utils
+  url = https://github.com/NOAA-EPIC/gfs-utils
+  branch = feature/jedi-inline
 [submodule "sorc/ufs_utils.fd"]
   path = sorc/ufs_utils.fd
-  url = https://github.com/ufs-community/UFS_UTILS.git
+  url = https://github.com/NOAA-EPIC/UFS_UTILS.git
+  branch = feature/jedi-inline
 [submodule "sorc/verif-global.fd"]
   path = sorc/verif-global.fd
   url = https://github.com/NOAA-EMC/EMC_verif-global.git
 [submodule "sorc/gsi_enkf.fd"]
   path = sorc/gsi_enkf.fd
-  url = https://github.com/NOAA-EMC/GSI.git
+  url = https://github.com/NOAA-EPIC/GSI.git
+  branch = feature/jedi-inline
 [submodule "sorc/gdas.cd"]
   path = sorc/gdas.cd
-  url = https://github.com/NOAA-EMC/GDASApp.git
+  url = https://github.com/NOAA-EPIC/GDASApp.git
+  branch = feature/jedi-inline
 [submodule "sorc/gsi_utils.fd"]
   path = sorc/gsi_utils.fd
-  url = https://github.com/NOAA-EMC/GSI-Utils.git
+  url = https://github.com/NOAA-EPIC/GSI-Utils.git
+  branch = feature/jedi-inline
 [submodule "sorc/gsi_monitor.fd"]
   path = sorc/gsi_monitor.fd
-  url = https://github.com/NOAA-EMC/GSI-Monitor.git
+  url = https://github.com/NOAA-EPIC/GSI-Monitor.git
+  branch = feature/jedi-inline

get-files.pl

@@ -0,0 +1,40 @@
+#!/usr/bin/perl
+
+  $count = 0;
+  $untracked = 0;
+  $directory = ".";
+  @lines = split(/\n/,`cat @ARGV[0]`);
+  foreach(@lines) {
+    $line = $_;
+    if($line =~ /Entering/) {
+      @words = split(/\'/,$line);
+      $directory = @words[1];
+    }
+    if($line =~ /modified:/) {
+      $line =~ s/\s+//g;
+      @words = split(/:/,$line);
+      $change = @words[1];
+      if($change !~ /content/) {
+        print("$directory/$change\n");
+      }
+    }
+    if($line =~ /Untracked/) {
+      $untracked = 1;
+      $count = 0;
+    } else {
+      $count++;
+    }
+    if(($untracked == 1) && ($count > 1)) {
+       @words=split(/\s+/,$line);
+       $file = @words[1];
+       if($file =~ /changes/){
+         $untracked = 0;
+       } else {
+         if($file ne "") {
+           print("$directory/$file\n");
+         }
+       }
+    }
+
+  }
+

modulefiles/module_base.container.lua

@@ -0,0 +1,57 @@
+help([[
+Load environment to run GFS on Hera
+]])
+
+local spack_mod_path=(os.getenv("spack_mod_path") or "None")
+prepend_path("MODULEPATH", spack_mod_path)
+
+setenv("CC", "icc" ) 
+setenv("CXX","icpc" ) 
+setenv("F77","ifort" ) 
+setenv("F90","ifort" ) 
+setenv("FC", "ifort" ) 
+load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None")))
+load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None")))
+--load(pathJoin("python", (os.getenv("python_ver") or "None")))
+
+--load(pathJoin("hpss", (os.getenv("hpss_ver") or "None")))
+--load(pathJoin("gempak", (os.getenv("gempak_ver") or "None")))
+--load(pathJoin("ncl", (os.getenv("ncl_ver") or "None")))
+load(pathJoin("jasper", (os.getenv("jasper_ver") or "None")))
+load(pathJoin("libpng", (os.getenv("libpng_ver") or "None")))
+load(pathJoin("cdo", (os.getenv("cdo_ver") or "None")))
+--load(pathJoin("R", (os.getenv("R_ver") or "None")))
+--load(pathJoin("perl", (os.getenv("perl_ver") or "None")))
+
+load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None")))
+load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None")))
+load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None")))
+
+load(pathJoin("nco", (os.getenv("nco_ver") or "None")))
+load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None")))
+load(pathJoin("grib-util", (os.getenv("grib_util_ver") or "None")))
+load(pathJoin("g2tmpl", (os.getenv("g2tmpl_ver") or "None")))
+load(pathJoin("gsi-ncdiag", (os.getenv("gsi_ncdiag_ver") or "None")))
+load(pathJoin("crtm", (os.getenv("crtm_ver") or "None")))
+load(pathJoin("bufr", (os.getenv("bufr_ver") or "None")))
+load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
+load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None")))
+load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
+load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
+load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
+load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None")))
+load(pathJoin("py-python-dateutil", (os.getenv("py_python_dateutil_ver") or "None")))
+load(pathJoin("met", (os.getenv("met_ver") or "None")))
+load(pathJoin("metplus", (os.getenv("metplus_ver") or "None")))
+load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
+
+--setenv("WGRIB2","wgrib2")
+
+-- Stop gap fix for wgrib with spack-stack 1.8.0
+-- TODO Remove this when spack-stack issue #1097 is resolved
+--setenv("WGRIB","wgrib")
+--setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
+
+--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
+
+whatis("Description: GFS run environment")

sorc/build_all.sh

@@ -1,6 +1,6 @@
 #! /usr/bin/env bash
 
-set +x
+set -x
 #------------------------------------
 # Exception handling is now included.
 #
@@ -145,7 +145,7 @@ build_opts["ww3prepost"]="${_wave_opt} ${_verbose_opt} ${_build_ufs_opt} ${_buil
 
 # Optional DA builds
 if [[ "${_build_ufsda}" == "YES" ]]; then
-   if [[ "${MACHINE_ID}" != "orion" && "${MACHINE_ID}" != "hera" && "${MACHINE_ID}" != "hercules" && "${MACHINE_ID}" != "wcoss2" && "${MACHINE_ID}" != "noaacloud" && "${MACHINE_ID}" != "gaea" ]]; then
+   if [[ "${MACHINE_ID}" != "container" && "${MACHINE_ID}" != "orion" && "${MACHINE_ID}" != "hera" && "${MACHINE_ID}" != "hercules" && "${MACHINE_ID}" != "wcoss2" && "${MACHINE_ID}" != "noaacloud" && "${MACHINE_ID}" != "gaea" ]]; then
       echo "NOTE: The GDAS App is not supported on ${MACHINE_ID}.  Disabling build."
    else
       build_jobs["gdas"]=8

sorc/build_gdas.sh

@@ -3,11 +3,12 @@ set -eux
 
 OPTIND=1
 _opts="-f "  # forces a clean build
-while getopts ":j:dv" option; do
+while getopts ":j:idv" option; do
   case "${option}" in
     d) _opts+="-c -DCMAKE_BUILD_TYPE=Debug " ;;
     j) BUILD_JOBS=${OPTARG};;
     v) _opts+="-v ";;
+    i) _opts+="-i -a ";;
     :)
       echo "[${BASH_SOURCE[0]}]: ${option} requires an argument"
       usage

versions/build.container.ver

@@ -0,0 +1,10 @@
+export stack_intel_ver=2021.10.0
+export stack_impi_ver=2021.12.1
+export spack_env=global-workflow-env
+source "${HOMEgfs:-}/versions/run.container.ver"
+export spack_mod_path="/opt/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core"
+export I_MPI_CC=$CC
+export I_MPI_CXX=$CXX
+export I_MPI_F77=$FC
+export I_MPI_FC=$FC
+export I_MPI_F90=$FC

versions/run.container.ver

@@ -0,0 +1,43 @@
+export spack_stack_ver=1.8.0
+export python_ver=3.11.6
+
+export jasper_ver=2.0.32
+export libpng_ver=1.6.37
+export cdo_ver=2.3.0
+export nco_ver=5.1.6
+export esmf_ver=8.6.1
+export fms_ver=2024.02
+export mapl_ver=2.46.3-esmf-8.6.1
+export gftl_shared_ver=1.9.0
+
+export hdf5_ver=1.14.3
+export netcdf_c_ver=4.9.2
+export netcdf_fortran_ver=4.6.1
+
+export bufr_ver=12.1.0
+export gsi_ncdiag_ver=1.1.2
+export g2tmpl_ver=1.13.0
+export g2_ver=3.5.1 
+export cmake_ver=3.27.9
+export crtm_ver=2.4.0.1
+export wgrib2_ver=3.1.1
+export grib_util_ver=1.4.0
+export prod_util_ver=2.1.1
+export py_netcdf4_ver=1.5.8
+export py_pyyaml_ver=6.0
+export py_jinja2_ver=3.1.2
+export parallelio_ver=2.6.2
+export py_pandas_ver=1.5.3
+export py_python_dateutil_ver=2.8.2
+export py_f90nml_ver=1.4.3
+export ip_ver=5.0.0
+
+export met_ver=11.1.1
+export metplus_ver=5.1.0
+export py_xarray_ver=2023.7.0
+
+export obsproc_run_ver=1.1.2
+export prepobs_run_ver=1.0.1
+
+export ens_tracker_ver=feature-GFSv17_com_reorg
+export fit2obs_ver=1.1.1