Skip to content

Update ufs-bundle to be buildable using entirely public repositories #76

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mark-a-potts wants to merge 21 commits into
base: develop
Choose a base branch
from
Open

Update ufs-bundle to be buildable using entirely public repositories #76

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
eb24a03
updates to get things building/linking
mark-a-potts Jan 25, 2022
142975a
using two different versions of fv3-jedi depending on UFS_APP
mark-a-potts Jan 26, 2022
2c2a90f
small change to ioda for expat linking
mark-a-potts Jan 27, 2022
6e4c053
fix for building in container
mark-a-potts Jan 27, 2022
81895ee
Merge branch 'feature/soca' of https://github.com/JCSDA/ufs-jedi-bund…
mark-a-potts Jan 27, 2022
603371d
public version
mark-a-potts Feb 11, 2022
30e2ab5
files to allow build without access to private repos
mark-a-potts Feb 12, 2022
099c4bb
updates for public
mark-a-potts Feb 14, 2022
83f4d4d
moved ioda and fv3-jedi-lm to NOAA-EMC
mark-a-potts Feb 14, 2022
c514ebf
removed patch command
mark-a-potts Feb 14, 2022
b0d6334
Added readme with instructions on how to build on Orion and Hera
mark-a-potts Feb 14, 2022
1bc6e12
changed fv3-jedi to NOAA-EMC
mark-a-potts Feb 14, 2022
13cba52
Merge branch 'feature/public' of https://github.com/jcsda/ufs-jedi-bu…
mark-a-potts Feb 14, 2022
de291f7
updates for build with ufs
mark-a-potts Mar 9, 2022
1772cc3
Updates to build ufs-bundle with ONLY public repos
mark-a-potts Feb 6, 2025
a9114b7
Deleted outdated README and removed stdout from patch
mark-a-potts Feb 6, 2025
a3359b5
reverted patch
mark-a-potts Feb 20, 2025
feed812
Merge remote-tracking branch 'origin/develop' into feature/public
mark-a-potts Feb 20, 2025
b783f3a
updated patch
mark-a-potts Jun 5, 2025
c5a8b98
moved oops and fv3-jedi to noaa-epic for sharing
mark-a-potts Jun 5, 2025
bd701f5
updated fv3-jedi-data branch
mark-a-potts Jun 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 13 additions & 14 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,26 +42,25 @@ find_package(FMS 2022.04 REQUIRED COMPONENTS R4 R8)

# Core JEDI repositories
# ----------------------
#ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH feature/ufs-stateset)
ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT oops GIT "https://github.com/noaa-epic/oops.git" BRANCH feature/Rtranspose UPDATE )
ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda/vader.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda/saber.git" BRANCH develop UPDATE )


if(UFS_APP MATCHES "^(ATMAERO)$" OR UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(NG-GODAS)$")
ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda/GSW-Fortran.git" BRANCH develop UPDATE )
endif()

# Use last known working tag until issues with new pure cmake build are resolved
ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/CRTMv3.git" TAG v3.1.0-skylabv7 ) # BRANCH develop UPDATE )

option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON)
ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda/ioda.git" BRANCH develop UPDATE )

option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON)
ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/jcsda-internal/ufo-data.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" BRANCH develop UPDATE )

# Options for building with certain models
# ----------------------------------------
Expand All @@ -82,7 +81,7 @@ set(FV3_PRECISION DOUBLE CACHE STRING "Precision of FV3 core (SINGLE, DOUBLE)")

# fv3-jedi linear model
# ---------------------
ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda/fv3-jedi-linearmodel.git" BRANCH develop UPDATE )
message(INFO "CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}")
include_directories(${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_INCLUDEDIR})
include_directories(${DEPEND_LIB_ROOT}/include_r8)
Expand Down Expand Up @@ -118,7 +117,7 @@ set(UFS_TAG "develop" )
if(UFS_APP MATCHES "^(NG-GODAS)$")
set(patch_command ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/ng-godas.patch <SOURCE_DIR> && cd <SOURCE_DIR> && git apply ng-godas.patch)
else()
set(patch_command "")
set(patch_command ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/ufs-wm.patch <SOURCE_DIR> && cd <SOURCE_DIR> && git apply ufs-wm.patch)
endif()

ExternalProject_Add(ufs-weather-model
Expand All @@ -131,7 +130,7 @@ ExternalProject_Add(ufs-weather-model
UPDATE_DISCONNECTED ON
INSTALL_DIR ${DEPEND_LIB_ROOT}
# DH* 20230316 turn off OpenMP for now ...
CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS=-fPIC -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v17_coupled_p8,FV3_GFS_v15p2,FV3_GFS_v17_p8 -DCMAKE_EXE_LINKER_FLAGS=${UFS_CMAKE_EXE_LINKER_FLAGS} -DCMAKE_C_COMPILER=${MPI_C_COMPILER} -DCMAKE_CXX_COMPILER=${MPI_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${MPI_Fortran_COMPILER} -DJEDI_DRIVER=ON -DCMAKE_PREFIX_PATH=${CMAKE_CURRENT_BINARY_DIR}/gsw ${UFS_CMAKE_BUILD_TYPE_FLAG} -DOPENMP=OFF -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT}
CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS=-fPIC -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v17_coupled_p8,FV3_GFS_v15p2,FV3_GFS_v17_p8 -DCMAKE_EXE_LINKER_FLAGS=${UFS_CMAKE_EXE_LINKER_FLAGS} -DCMAKE_C_COMPILER=${MPI_C_COMPILER} -DCMAKE_CXX_COMPILER=${MPI_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${MPI_Fortran_COMPILER} -DJEDI_DRIVER=ON -DCMAKE_PREFIX_PATH=${CMAKE_CURRENT_BINARY_DIR}/gsw ${UFS_CMAKE_BUILD_TYPE_FLAG} -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT}
# *DH
INSTALL_COMMAND make install
BUILD_ALWAYS TRUE
Expand Down Expand Up @@ -182,13 +181,13 @@ set_target_properties( mom6 PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CM
# Additional dependencies that we can't know about because ufs-weather-model is an external project
target_link_libraries( fv3atm INTERFACE fms )

ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.3.0 )
ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda/femps.git" TAG 1.3.0 )
if(UFS_APP MATCHES "^(ATMAERO)$" OR UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(ATM)$")
option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON)
ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH develop UPDATE )
ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/public UPDATE )
ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/noaa-epic/fv3-jedi.git" BRANCH feature/Rtranspose UPDATE )
elseif(UFS_APP MATCHES "^(NG-GODAS)$")
ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom_update UPDATE )
ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda/soca.git" BRANCH feature/ufs_dom_update UPDATE )
add_dependencies(soca ufs-weather-model)
else()
message(FATAL_ERROR "ufs-bundle unknown UFS_APP ${UFS_APP}")
Expand Down
81 changes: 81 additions & 0 deletions patches/ufs-wm.patch
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
diff --git a/FV3/fv3/atmos_cubed_sphere/model/fv_control.F90 b/FV3/fv3/atmos_cubed_sphere/model/fv_control.F90
index bf111af..574e233 100644
--- a/FV3/fv3/atmos_cubed_sphere/model/fv_control.F90
+++ b/FV3/fv3/atmos_cubed_sphere/model/fv_control.F90
@@ -457,6 +457,7 @@ module fv_control_mod
allocate(grids_on_this_pe(ngrids))
grids_on_this_pe(:) = .false.

+ grid_pes(1) = npes
do n=1,ngrids

if (ngrids == 1 .or. grid_pes(n) == 0) then
diff --git a/FV3/fv3/module_fcst_grid_comp.F90 b/FV3/fv3/module_fcst_grid_comp.F90
index 27cdf95..49825d7 100644
--- a/FV3/fv3/module_fcst_grid_comp.F90
+++ b/FV3/fv3/module_fcst_grid_comp.F90
@@ -575,7 +575,7 @@ if (rc /= ESMF_SUCCESS) write(0,*) 'rc=',rc,__FILE__,__LINE__; if(ESMF_LogFoundE
type(ESMF_DistGrid) :: distgrid
integer :: jsc, jec, isc, iec, nlev
type(domain2D) :: domain
- integer :: n, fcstNpes, tmpvar, k
+ integer :: n, fcstNpes, tmpvar, k, ii
logical :: freq_restart, fexist
integer, allocatable, dimension(:) :: isl, iel, jsl, jel
integer, allocatable, dimension(:,:,:) :: deBlockList
@@ -600,7 +600,7 @@ if (rc /= ESMF_SUCCESS) write(0,*) 'rc=',rc,__FILE__,__LINE__; if(ESMF_LogFoundE
Time_restart, Time_step_restart
type(time_type) :: iautime
integer :: io_unit, calendar_type_res, date_res(6), date_init_res(6)
-
+ integer,allocatable :: pelistZero(:)
integer,allocatable :: grid_number_on_all_pets(:)
logical,allocatable :: is_moving_on_all_pets(:), is_moving(:)
character(len=7) :: nest_suffix
@@ -865,8 +865,9 @@ if (rc /= ESMF_SUCCESS) write(0,*) 'rc=',rc,__FILE__,__LINE__; if(ESMF_LogFoundE

pelist => null()
call atmos_model_get_nth_domain_info(n, layout, nx, ny, pelist)
- call ESMF_VMBroadcast(vm, bcstData=layout, count=2, rootPet=pelist(1), rc=rc); ESMF_ERR_ABORT(rc)
-
+! call ESMF_VMBroadcast(vm, bcstData=layout, count=2, rootPet=pelist(1), rc=rc); ESMF_ERR_ABORT(rc)
+ call ESMF_VMBroadcast(vm, bcstData=layout, count=2, rootPet=0, rc=rc); ESMF_ERR_ABORT(rc)
+ write(6,*) 'HEY, mype, mpp_pe and petlist are',mype, mpp_pe(), pelist
if (n==1) then
! on grid==1 (top level parent) determine if the domain is global or regional
top_parent_is_global = .true.
@@ -875,11 +876,16 @@ if (rc /= ESMF_SUCCESS) write(0,*) 'rc=',rc,__FILE__,__LINE__; if(ESMF_LogFoundE
endif
call mpi_bcast(top_parent_is_global, 1, MPI_LOGICAL, 0, fcst_mpi_comm, rc)
endif
-
+
if (n==1 .and. top_parent_is_global) then
-
- fcstGridComp(n) = ESMF_GridCompCreate(name="global", petList=pelist, rc=rc); ESMF_ERR_ABORT(rc)
-
+ allocate(pelistZero(fcst_ntasks))
+ do ii=1,fcst_ntasks
+ pelistZero(ii) = ii - 1
+ enddo
+ write(6,*) 'HEY, mpp_pe and petlistZero are',mpp_pe(), pelistZero,fcst_ntasks
+ fcstGridComp(n) = ESMF_GridCompCreate(name="global", petList=pelistZero, rc=rc); ESMF_ERR_ABORT(rc)
+! fcstGridComp(n) = ESMF_GridCompCreate(name="global", petList=pelist, rc=rc); ESMF_ERR_ABORT(rc)
+ deallocate(pelistZero)
call ESMF_InfoGetFromHost(fcstGridComp(n), info=info, rc=rc); ESMF_ERR_ABORT(rc)
call ESMF_InfoSet(info, key="layout", values=layout, rc=rc); ESMF_ERR_ABORT(rc)
call ESMF_InfoSet(info, key="tilesize", value=Atmos%mlon, rc=rc); ESMF_ERR_ABORT(rc)
diff --git a/stochastic_physics/mpi_wrapper.F90 b/stochastic_physics/mpi_wrapper.F90
index 7ccd5a7..8961474 100644
--- a/stochastic_physics/mpi_wrapper.F90
+++ b/stochastic_physics/mpi_wrapper.F90
@@ -81,7 +81,8 @@ contains
integer, intent(in) :: mpiroot
type(MPI_Comm), intent(in) :: mpicomm
if (initialized) return
- root = mpiroot
+ root = 0
+! root = mpiroot
comm = mpicomm
call MPI_COMM_RANK(comm, mype, ierror)
call MPI_COMM_SIZE(comm, npes, ierror)