diff --git a/CMakeLists.txt b/CMakeLists.txt index d8643d6..b34c6f0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -42,14 +42,13 @@ find_package(FMS 2022.04 REQUIRED COMPONENTS R4 R8) # Core JEDI repositories # ---------------------- -#ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH feature/ufs-stateset) -ecbuild_bundle( PROJECT oops GIT "https://github.com/jcsda-internal/oops.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda-internal/vader.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda-internal/saber.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT oops GIT "https://github.com/noaa-epic/oops.git" BRANCH feature/Rtranspose UPDATE ) +ecbuild_bundle( PROJECT vader GIT "https://github.com/jcsda/vader.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT saber GIT "https://github.com/jcsda/saber.git" BRANCH develop UPDATE ) if(UFS_APP MATCHES "^(ATMAERO)$" OR UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(NG-GODAS)$") - ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda-internal/GSW-Fortran.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT gsw GIT "https://github.com/jcsda/GSW-Fortran.git" BRANCH develop UPDATE ) endif() # Use last known working tag until issues with new pure cmake build are resolved @@ -57,11 +56,11 @@ ecbuild_bundle( PROJECT crtm GIT "https://github.com/JCSDA/CRTMv3.git" TAG v3.1. option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON) ecbuild_bundle( PROJECT ioda-data GIT "https://github.com/JCSDA-internal/ioda-data.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda-internal/ioda.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT ioda GIT "https://github.com/jcsda/ioda.git" BRANCH develop UPDATE ) option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON) ecbuild_bundle( PROJECT ufo-data GIT "https://github.com/jcsda-internal/ufo-data.git" BRANCH develop UPDATE ) -ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda-internal/ufo.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT ufo GIT "https://github.com/jcsda/ufo.git" BRANCH develop UPDATE ) # Options for building with certain models # ---------------------------------------- @@ -82,7 +81,7 @@ set(FV3_PRECISION DOUBLE CACHE STRING "Precision of FV3 core (SINGLE, DOUBLE)") # fv3-jedi linear model # --------------------- -ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda-internal/fv3-jedi-linearmodel.git" BRANCH develop UPDATE ) +ecbuild_bundle( PROJECT fv3-jedi-lm GIT "https://github.com/jcsda/fv3-jedi-linearmodel.git" BRANCH develop UPDATE ) message(INFO "CMAKE_INSTALL_LIBDIR: ${CMAKE_INSTALL_LIBDIR}") include_directories(${DEPEND_LIB_ROOT}/${CMAKE_INSTALL_INCLUDEDIR}) include_directories(${DEPEND_LIB_ROOT}/include_r8) @@ -118,7 +117,7 @@ set(UFS_TAG "develop" ) if(UFS_APP MATCHES "^(NG-GODAS)$") set(patch_command ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/ng-godas.patch && cd && git apply ng-godas.patch) else() - set(patch_command "") + set(patch_command ${CMAKE_COMMAND} -E copy_if_different ${CMAKE_CURRENT_SOURCE_DIR}/patches/ufs-wm.patch && cd && git apply ufs-wm.patch) endif() ExternalProject_Add(ufs-weather-model @@ -131,7 +130,7 @@ ExternalProject_Add(ufs-weather-model UPDATE_DISCONNECTED ON INSTALL_DIR ${DEPEND_LIB_ROOT} # DH* 20230316 turn off OpenMP for now ... - CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS=-fPIC -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v17_coupled_p8,FV3_GFS_v15p2,FV3_GFS_v17_p8 -DCMAKE_EXE_LINKER_FLAGS=${UFS_CMAKE_EXE_LINKER_FLAGS} -DCMAKE_C_COMPILER=${MPI_C_COMPILER} -DCMAKE_CXX_COMPILER=${MPI_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${MPI_Fortran_COMPILER} -DJEDI_DRIVER=ON -DCMAKE_PREFIX_PATH=${CMAKE_CURRENT_BINARY_DIR}/gsw ${UFS_CMAKE_BUILD_TYPE_FLAG} -DOPENMP=OFF -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} + CMAKE_ARGS -DCMAKE_INSTALL_LIBDIR=${CMAKE_INSTALL_LIBDIR} -DCMAKE_C_FLAGS=-fPIC -DCMAKE_Fortran_FLAGS=-fPIC -DCMAKE_CXX_FLAGS=-fPIC -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v17_coupled_p8,FV3_GFS_v15p2,FV3_GFS_v17_p8 -DCMAKE_EXE_LINKER_FLAGS=${UFS_CMAKE_EXE_LINKER_FLAGS} -DCMAKE_C_COMPILER=${MPI_C_COMPILER} -DCMAKE_CXX_COMPILER=${MPI_CXX_COMPILER} -DCMAKE_Fortran_COMPILER=${MPI_Fortran_COMPILER} -DJEDI_DRIVER=ON -DCMAKE_PREFIX_PATH=${CMAKE_CURRENT_BINARY_DIR}/gsw ${UFS_CMAKE_BUILD_TYPE_FLAG} -DOPENMP=ON -DINLINE_POST=OFF -DMULTI_GASES=OFF -DMPI=ON -DAPP=${UFS_APP} -DCMAKE_INSTALL_PREFIX=${DEPEND_LIB_ROOT} # *DH INSTALL_COMMAND make install BUILD_ALWAYS TRUE @@ -182,13 +181,13 @@ set_target_properties( mom6 PROPERTIES IMPORTED_LOCATION ${DEPEND_LIB_ROOT}/${CM # Additional dependencies that we can't know about because ufs-weather-model is an external project target_link_libraries( fv3atm INTERFACE fms ) -ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda-internal/femps.git" TAG 1.3.0 ) +ecbuild_bundle( PROJECT femps GIT "https://github.com/jcsda/femps.git" TAG 1.3.0 ) if(UFS_APP MATCHES "^(ATMAERO)$" OR UFS_APP MATCHES "^(S2S)$" OR UFS_APP MATCHES "^(ATM)$") option(ENABLE_FV3_JEDI_DATA "Obtain fv3-jedi test data from fv3-jedi-data repository (vs tarball)" ON) - ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH develop UPDATE ) - ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/jcsda-internal/fv3-jedi.git" BRANCH develop UPDATE ) + ecbuild_bundle( PROJECT fv3-jedi-data GIT "https://github.com/JCSDA-internal/fv3-jedi-data.git" BRANCH feature/public UPDATE ) + ecbuild_bundle( PROJECT fv3-jedi GIT "https://github.com/noaa-epic/fv3-jedi.git" BRANCH feature/Rtranspose UPDATE ) elseif(UFS_APP MATCHES "^(NG-GODAS)$") - ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda-internal/soca.git" BRANCH feature/ufs_dom_update UPDATE ) + ecbuild_bundle( PROJECT soca GIT "https://github.com/jcsda/soca.git" BRANCH feature/ufs_dom_update UPDATE ) add_dependencies(soca ufs-weather-model) else() message(FATAL_ERROR "ufs-bundle unknown UFS_APP ${UFS_APP}") diff --git a/patches/ufs-wm.patch b/patches/ufs-wm.patch new file mode 100644 index 0000000..f39bdda --- /dev/null +++ b/patches/ufs-wm.patch @@ -0,0 +1,81 @@ +diff --git a/FV3/fv3/atmos_cubed_sphere/model/fv_control.F90 b/FV3/fv3/atmos_cubed_sphere/model/fv_control.F90 +index bf111af..574e233 100644 +--- a/FV3/fv3/atmos_cubed_sphere/model/fv_control.F90 ++++ b/FV3/fv3/atmos_cubed_sphere/model/fv_control.F90 +@@ -457,6 +457,7 @@ module fv_control_mod + allocate(grids_on_this_pe(ngrids)) + grids_on_this_pe(:) = .false. + ++ grid_pes(1) = npes + do n=1,ngrids + + if (ngrids == 1 .or. grid_pes(n) == 0) then +diff --git a/FV3/fv3/module_fcst_grid_comp.F90 b/FV3/fv3/module_fcst_grid_comp.F90 +index 27cdf95..49825d7 100644 +--- a/FV3/fv3/module_fcst_grid_comp.F90 ++++ b/FV3/fv3/module_fcst_grid_comp.F90 +@@ -575,7 +575,7 @@ if (rc /= ESMF_SUCCESS) write(0,*) 'rc=',rc,__FILE__,__LINE__; if(ESMF_LogFoundE + type(ESMF_DistGrid) :: distgrid + integer :: jsc, jec, isc, iec, nlev + type(domain2D) :: domain +- integer :: n, fcstNpes, tmpvar, k ++ integer :: n, fcstNpes, tmpvar, k, ii + logical :: freq_restart, fexist + integer, allocatable, dimension(:) :: isl, iel, jsl, jel + integer, allocatable, dimension(:,:,:) :: deBlockList +@@ -600,7 +600,7 @@ if (rc /= ESMF_SUCCESS) write(0,*) 'rc=',rc,__FILE__,__LINE__; if(ESMF_LogFoundE + Time_restart, Time_step_restart + type(time_type) :: iautime + integer :: io_unit, calendar_type_res, date_res(6), date_init_res(6) +- ++ integer,allocatable :: pelistZero(:) + integer,allocatable :: grid_number_on_all_pets(:) + logical,allocatable :: is_moving_on_all_pets(:), is_moving(:) + character(len=7) :: nest_suffix +@@ -865,8 +865,9 @@ if (rc /= ESMF_SUCCESS) write(0,*) 'rc=',rc,__FILE__,__LINE__; if(ESMF_LogFoundE + + pelist => null() + call atmos_model_get_nth_domain_info(n, layout, nx, ny, pelist) +- call ESMF_VMBroadcast(vm, bcstData=layout, count=2, rootPet=pelist(1), rc=rc); ESMF_ERR_ABORT(rc) +- ++! call ESMF_VMBroadcast(vm, bcstData=layout, count=2, rootPet=pelist(1), rc=rc); ESMF_ERR_ABORT(rc) ++ call ESMF_VMBroadcast(vm, bcstData=layout, count=2, rootPet=0, rc=rc); ESMF_ERR_ABORT(rc) ++ write(6,*) 'HEY, mype, mpp_pe and petlist are',mype, mpp_pe(), pelist + if (n==1) then + ! on grid==1 (top level parent) determine if the domain is global or regional + top_parent_is_global = .true. +@@ -875,11 +876,16 @@ if (rc /= ESMF_SUCCESS) write(0,*) 'rc=',rc,__FILE__,__LINE__; if(ESMF_LogFoundE + endif + call mpi_bcast(top_parent_is_global, 1, MPI_LOGICAL, 0, fcst_mpi_comm, rc) + endif +- ++ + if (n==1 .and. top_parent_is_global) then +- +- fcstGridComp(n) = ESMF_GridCompCreate(name="global", petList=pelist, rc=rc); ESMF_ERR_ABORT(rc) +- ++ allocate(pelistZero(fcst_ntasks)) ++ do ii=1,fcst_ntasks ++ pelistZero(ii) = ii - 1 ++ enddo ++ write(6,*) 'HEY, mpp_pe and petlistZero are',mpp_pe(), pelistZero,fcst_ntasks ++ fcstGridComp(n) = ESMF_GridCompCreate(name="global", petList=pelistZero, rc=rc); ESMF_ERR_ABORT(rc) ++! fcstGridComp(n) = ESMF_GridCompCreate(name="global", petList=pelist, rc=rc); ESMF_ERR_ABORT(rc) ++ deallocate(pelistZero) + call ESMF_InfoGetFromHost(fcstGridComp(n), info=info, rc=rc); ESMF_ERR_ABORT(rc) + call ESMF_InfoSet(info, key="layout", values=layout, rc=rc); ESMF_ERR_ABORT(rc) + call ESMF_InfoSet(info, key="tilesize", value=Atmos%mlon, rc=rc); ESMF_ERR_ABORT(rc) +diff --git a/stochastic_physics/mpi_wrapper.F90 b/stochastic_physics/mpi_wrapper.F90 +index 7ccd5a7..8961474 100644 +--- a/stochastic_physics/mpi_wrapper.F90 ++++ b/stochastic_physics/mpi_wrapper.F90 +@@ -81,7 +81,8 @@ contains + integer, intent(in) :: mpiroot + type(MPI_Comm), intent(in) :: mpicomm + if (initialized) return +- root = mpiroot ++ root = 0 ++! root = mpiroot + comm = mpicomm + call MPI_COMM_RANK(comm, mype, ierror) + call MPI_COMM_SIZE(comm, npes, ierror)