Skip to content
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
140 changes: 75 additions & 65 deletions cmake/thirdparty/SetupSeracThirdParty.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,29 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
endif()
message(STATUS "ARPACK support is ${ARPACK_FOUND}")

#------------------------------------------------------------------------------
# Enzyme
#------------------------------------------------------------------------------
if (ENZYME_DIR)
serac_assert_is_directory(DIR_VARIABLE ENZYME_DIR)
set(Enzyme_ROOT ${ENZYME_DIR} CACHE PATH "")
find_dependency(Enzyme REQUIRED)

# serac_assert_find_succeeded(PROJECT_NAME Enzyme
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did this not work?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, it does. dFEM doesn't use it, so I temporarily disabled it while I had Tribol manually disabled. Added it back in.

# TARGET LLDEnzymeFlags
# DIR_VARIABLE ENZYME_DIR)

serac_assert_find_succeeded(PROJECT_NAME Enzyme
TARGET ClangEnzymeFlags
DIR_VARIABLE ENZYME_DIR)

message(STATUS "Enzyme support is ON")
set(ENZYME_FOUND TRUE)
else()
message(STATUS "Enzyme support is OFF")
set(ENZYME_FOUND FALSE)
endif()

#------------------------------------------------------------------------------
# MFEM
#------------------------------------------------------------------------------
Expand All @@ -214,7 +237,7 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
message(STATUS "Using MFEM submodule")

#### Store Data that MFEM clears
set(tpls_to_save ADIAK AMGX AXOM CALIPER CAMP CONDUIT HDF5
set(tpls_to_save ADIAK AMGX AXOM CALIPER CAMP CONDUIT ENZYME HDF5
HYPRE LUA METIS MFEM NETCDF PARMETIS PETSC RAJA
SLEPC SUPERLU_DIST STRUMPACK SUNDIALS TRIBOL
UMPIRE)
Expand Down Expand Up @@ -292,6 +315,10 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
endif()
set(MFEM_USE_UMPIRE OFF CACHE BOOL "")
set(MFEM_USE_ZLIB ON CACHE BOOL "")
if(ENZYME_DIR)
serac_assert_is_directory(DIR_VARIABLE ENZYME_DIR)
set(MFEM_USE_ENZYME ON CACHE BOOL "")
endif()

#### MFEM Configuration Options

Expand Down Expand Up @@ -469,78 +496,61 @@ if (NOT SERAC_THIRD_PARTY_LIBRARIES_FOUND)
set(ENABLE_FORTRAN ON CACHE BOOL "" FORCE)
endif()

#------------------------------------------------------------------------------
# Enzyme (used by Tribol)
#------------------------------------------------------------------------------
if (ENZYME_DIR)
serac_assert_is_directory(DIR_VARIABLE ENZYME_DIR)
set(Enzyme_ROOT ${ENZYME_DIR} CACHE PATH "")
find_dependency(Enzyme REQUIRED)

serac_assert_find_succeeded(PROJECT_NAME Enzyme
TARGET LLDEnzymeFlags
DIR_VARIABLE ENZYME_DIR)

message(STATUS "Enzyme support is ON")
set(ENZYME_FOUND TRUE)
else()
message(STATUS "Enzyme support is OFF")
set(ENZYME_FOUND FALSE)
endif()

#------------------------------------------------------------------------------
# Tribol
#------------------------------------------------------------------------------
if (NOT SERAC_ENABLE_CODEVELOP)
if(TRIBOL_DIR)
serac_assert_is_directory(DIR_VARIABLE TRIBOL_DIR)
if (NOT SERAC_DISABLE_TRIBOL)
if (NOT SERAC_ENABLE_CODEVELOP)
if(TRIBOL_DIR)
serac_assert_is_directory(DIR_VARIABLE TRIBOL_DIR)

find_dependency(tribol REQUIRED PATHS "${TRIBOL_DIR}/lib/cmake")
find_dependency(tribol REQUIRED PATHS "${TRIBOL_DIR}/lib/cmake")

serac_assert_find_succeeded(PROJECT_NAME Tribol
TARGET tribol
DIR_VARIABLE TRIBOL_DIR)
blt_convert_to_system_includes(TARGET tribol)
set(TRIBOL_FOUND ON)
serac_assert_find_succeeded(PROJECT_NAME Tribol
TARGET tribol
DIR_VARIABLE TRIBOL_DIR)
blt_convert_to_system_includes(TARGET tribol)
set(TRIBOL_FOUND ON)
else()
set(TRIBOL_FOUND OFF)
endif()

message(STATUS "Tribol support is " ${TRIBOL_FOUND})
else()
set(TRIBOL_FOUND OFF)
endif()

message(STATUS "Tribol support is " ${TRIBOL_FOUND})
else()
set(ENABLE_FORTRAN OFF CACHE BOOL "" FORCE)
# Otherwise we use the submodule
message(STATUS "Using Tribol submodule")
set(BUILD_REDECOMP ${SERAC_ENABLE_MPI} CACHE BOOL "")
set(TRIBOL_USE_MPI ${SERAC_ENABLE_MPI} CACHE BOOL "")
set(TRIBOL_ENABLE_TESTS OFF CACHE BOOL "")
set(TRIBOL_ENABLE_EXAMPLES OFF CACHE BOOL "")
set(TRIBOL_ENABLE_DOCS OFF CACHE BOOL "")
set(ENABLE_FORTRAN OFF CACHE BOOL "" FORCE)
# Otherwise we use the submodule
message(STATUS "Using Tribol submodule")
set(BUILD_REDECOMP ${SERAC_ENABLE_MPI} CACHE BOOL "")
set(TRIBOL_USE_MPI ${SERAC_ENABLE_MPI} CACHE BOOL "")
set(TRIBOL_ENABLE_TESTS OFF CACHE BOOL "")
set(TRIBOL_ENABLE_EXAMPLES OFF CACHE BOOL "")
set(TRIBOL_ENABLE_DOCS OFF CACHE BOOL "")

if(${PROJECT_NAME} STREQUAL "smith")
set(tribol_repo_dir "${PROJECT_SOURCE_DIR}/serac/tribol")
else()
set(tribol_repo_dir "${PROJECT_SOURCE_DIR}/tribol")
endif()

if(${PROJECT_NAME} STREQUAL "smith")
set(tribol_repo_dir "${PROJECT_SOURCE_DIR}/serac/tribol")
else()
set(tribol_repo_dir "${PROJECT_SOURCE_DIR}/tribol")
add_subdirectory(${tribol_repo_dir} ${CMAKE_BINARY_DIR}/tribol)

target_include_directories(redecomp PUBLIC
$<BUILD_INTERFACE:${tribol_repo_dir}/src>
)
target_include_directories(tribol PUBLIC
$<BUILD_INTERFACE:${tribol_repo_dir}/src>
$<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/tribol/include>
$<INSTALL_INTERFACE:include>
)
target_include_directories(tribol_shared PUBLIC
$<BUILD_INTERFACE:${tribol_repo_dir}/src>
$<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/tribol/include>
$<INSTALL_INTERFACE:include>
)

set(TRIBOL_FOUND TRUE CACHE BOOL "" FORCE)
set(ENABLE_FORTRAN ON CACHE BOOL "" FORCE)
endif()

add_subdirectory(${tribol_repo_dir} ${CMAKE_BINARY_DIR}/tribol)

target_include_directories(redecomp PUBLIC
$<BUILD_INTERFACE:${tribol_repo_dir}/src>
)
target_include_directories(tribol PUBLIC
$<BUILD_INTERFACE:${tribol_repo_dir}/src>
$<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/tribol/include>
$<INSTALL_INTERFACE:include>
)
target_include_directories(tribol_shared PUBLIC
$<BUILD_INTERFACE:${tribol_repo_dir}/src>
$<BUILD_INTERFACE:${CMAKE_BINARY_DIR}/tribol/include>
$<INSTALL_INTERFACE:include>
)

set(TRIBOL_FOUND TRUE CACHE BOOL "" FORCE)
set(ENABLE_FORTRAN ON CACHE BOOL "" FORCE)
endif()

#---------------------------------------------------------------------------
Expand Down
149 changes: 149 additions & 0 deletions host-configs/[email protected]_hip_dfem.cmake
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
#------------------------------------------------------------------------------
# !!!! This is a generated file, edit at own risk !!!!
#------------------------------------------------------------------------------
# CMake executable path: /usr/tce/bin/cmake
#------------------------------------------------------------------------------

set(CMAKE_PREFIX_PATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/caliper-2.12.1-h2elmogt2mtorvunf67jrcxqdcrzjjem;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/tribol-0.1.0.18-neize4eyd3wxmeuui44az2x5ecqfe2pd;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/adiak-0.4.1-djvwhzv7uucbw342y2lxn6mjv4onnzeh;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/libunwind-1.8.1-667lulihqw77lixyhoyezlqadqmw3ufq;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/axom-0.10.1.1-4udyvzqnaoic74uv3zcumyguv5mbqqcg;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/conduit-0.9.2-wo23sn3yekayvxjphe2eu4vvmvkqrovj;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/mfem-4.8.0.1-s6vr55m7kq5gylxswlaqy5usltbrslzd;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/raja-2024.07.0-m322mdffprdfso4vg7iox3n5goelhtyp;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/umpire-2024.07.0-h4aizqln4js5uo6o5dnkkjadcq5apw37;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/netcdf-c-4.7.4-m3moxxpcdu3id35a6ib662pmcpe6hlxt;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/strumpack-8.0.0-w5kwnpidy7gzm4q2kplzw7ip2hebkojg;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/sundials-6.7.0-ihjcn5jozjo6hqxvueljdahsvpg4fpyr;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/superlu-dist-8.1.2-ulxns2zfpqyuezhrgxjh7do45ezer2m5;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/camp-2024.07.0-223y52kngyoyilxr7gz3i4pe5btnm5uq;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/fmt-11.0.2-haismsyfch472jneh7tj5hoz5faug2wp;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/hdf5-1.8.23-zhqxrmbx5hqezzazejozk5l7wuz3ijx6;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/netlib-scalapack-2.2.2-zinikcerv4cwfrzt4kbhby6xm76fjmvq;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/hypre-2.26.0-sffmaz74r5zmnzir6l73deslw36yypsh;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/parmetis-4.0.3-7o2cbbpbawncpn24yftcj24gbopz4aev;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/metis-5.1.0-q4s4zr5oj2vsyc4eqjy22wrwndgb4fnh;/usr/tce;/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1/llvm;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1;/opt/rocm-6.2.1" CACHE STRING "")

set(CMAKE_INSTALL_RPATH_USE_LINK_PATH "ON" CACHE STRING "")

set(CMAKE_BUILD_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib64;;/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12" CACHE STRING "")

set(CMAKE_INSTALL_RPATH "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib;/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1/serac-develop-og4fmn5dkubewiybduaji237suiuzssb/lib64;;/opt/rh/gcc-toolset-12/root/usr/lib/gcc/x86_64-redhat-linux/12" CACHE STRING "")

set(CMAKE_BUILD_TYPE "Release" CACHE STRING "")

#------------------------------------------------------------------------------
# Compilers
#------------------------------------------------------------------------------
# Compiler Spec: rocmcc@=6.2.1
#------------------------------------------------------------------------------
if(DEFINED ENV{SPACK_CC})

set(CMAKE_C_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/spack/lib/spack/env/rocmcc/amdclang" CACHE PATH "")

set(CMAKE_CXX_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/spack/lib/spack/env/rocmcc/amdclang++" CACHE PATH "")

set(CMAKE_Fortran_COMPILER "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/spack/lib/spack/env/rocmcc/amdflang" CACHE PATH "")

else()

set(CMAKE_C_COMPILER "/opt/rocm-6.2.1/llvm/bin/amdclang" CACHE PATH "")

set(CMAKE_CXX_COMPILER "/opt/rocm-6.2.1/llvm/bin/amdclang++" CACHE PATH "")

set(CMAKE_Fortran_COMPILER "/opt/rocm-6.2.1/llvm/bin/amdflang" CACHE PATH "")

endif()

#------------------------------------------------------------------------------
# MPI
#------------------------------------------------------------------------------

set(MPI_C_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1/bin/mpicc" CACHE PATH "")

set(MPI_CXX_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1/bin/mpicxx" CACHE PATH "")

set(MPI_Fortran_COMPILER "/usr/tce/packages/cray-mpich-tce/cray-mpich-8.1.29-rocmcc-6.2.1/bin/mpif90" CACHE PATH "")

set(MPIEXEC_NUMPROC_FLAG "-n" CACHE STRING "")

set(ENABLE_MPI ON CACHE BOOL "")

set(MPIEXEC_EXECUTABLE "/usr/global/tools/flux_wrappers/bin/srun" CACHE PATH "")

#------------------------------------------------------------------------------
# Hardware
#------------------------------------------------------------------------------

#------------------------------------------------
# ROCm
#------------------------------------------------

set(HIP_ROOT_DIR "/opt/rocm-6.2.1" CACHE PATH "")

set(CMAKE_HIP_COMPILER "/opt/rocm-6.2.1/llvm/bin/clang++" CACHE FILEPATH "")

set(CMAKE_HIP_ARCHITECTURES "gfx90a" CACHE STRING "")

set(AMDGPU_TARGETS "gfx90a" CACHE STRING "")

set(GPU_TARGETS "gfx90a" CACHE STRING "")

set(ENABLE_OPENMP OFF CACHE BOOL "")

set(ENABLE_HIP ON CACHE BOOL "")

set(ROCM_PATH "/opt/rocm-6.2.1" CACHE PATH "")

set(CMAKE_EXE_LINKER_FLAGS "-L/opt/rocm-6.2.1/lib -Wl,-rpath,/opt/rocm-6.2.1/lib -L/opt/rocm-6.2.1/llvm/lib -Wl,-rpath,/opt/rocm-6.2.1/llvm/lib -lxpmem -L/opt/cray/pe/mpich/8.1.29/gtl/lib -Wl,-rpath,/opt/cray/pe/mpich/8.1.29/gtl/lib -lmpi_gtl_hsa -Wl,--disable-new-dtags -lflang -lflangrti -lamdhip64 -lhsakmt -lhsa-runtime64 -lamd_comgr -lpgmath -lhipblas" CACHE STRING "")

#------------------------------------------------------------------------------
# TPLs
#------------------------------------------------------------------------------

set(TPL_ROOT "/usr/WS2/smithdev/libs/serac/toss_4_x86_64_ib_cray/2025_04_16_13_46_43/rocmcc-6.2.1" CACHE PATH "")

set(AXOM_DIR "${TPL_ROOT}/axom-0.10.1.1-4udyvzqnaoic74uv3zcumyguv5mbqqcg" CACHE PATH "")

set(CAMP_DIR "${TPL_ROOT}/camp-2024.07.0-223y52kngyoyilxr7gz3i4pe5btnm5uq" CACHE PATH "")

set(CONDUIT_DIR "${TPL_ROOT}/conduit-0.9.2-wo23sn3yekayvxjphe2eu4vvmvkqrovj" CACHE PATH "")

set(LUA_DIR "/usr" CACHE PATH "")

set(MFEM_DIR "${TPL_ROOT}/mfem-4.8.0.1-s6vr55m7kq5gylxswlaqy5usltbrslzd" CACHE PATH "")

set(HDF5_DIR "${TPL_ROOT}/hdf5-1.8.23-zhqxrmbx5hqezzazejozk5l7wuz3ijx6" CACHE PATH "")

set(HYPRE_DIR "${TPL_ROOT}/hypre-2.26.0-sffmaz74r5zmnzir6l73deslw36yypsh" CACHE PATH "")

set(METIS_DIR "${TPL_ROOT}/metis-5.1.0-q4s4zr5oj2vsyc4eqjy22wrwndgb4fnh" CACHE PATH "")

set(PARMETIS_DIR "${TPL_ROOT}/parmetis-4.0.3-7o2cbbpbawncpn24yftcj24gbopz4aev" CACHE PATH "")

set(NETCDF_DIR "${TPL_ROOT}/netcdf-c-4.7.4-m3moxxpcdu3id35a6ib662pmcpe6hlxt" CACHE PATH "")

set(SUPERLUDIST_DIR "${TPL_ROOT}/superlu-dist-8.1.2-ulxns2zfpqyuezhrgxjh7do45ezer2m5" CACHE PATH "")

set(ADIAK_DIR "${TPL_ROOT}/adiak-0.4.1-djvwhzv7uucbw342y2lxn6mjv4onnzeh" CACHE PATH "")

# AMGX not built

set(CALIPER_DIR "${TPL_ROOT}/caliper-2.12.1-h2elmogt2mtorvunf67jrcxqdcrzjjem" CACHE PATH "")

# PETSC not built

set(RAJA_DIR "${TPL_ROOT}/raja-2024.07.0-m322mdffprdfso4vg7iox3n5goelhtyp" CACHE PATH "")

# SLEPC not built

set(STRUMPACK_DIR "${TPL_ROOT}/strumpack-8.0.0-w5kwnpidy7gzm4q2kplzw7ip2hebkojg" CACHE PATH "")

set(SUNDIALS_DIR "${TPL_ROOT}/sundials-6.7.0-ihjcn5jozjo6hqxvueljdahsvpg4fpyr" CACHE PATH "")

set(UMPIRE_DIR "${TPL_ROOT}/umpire-2024.07.0-h4aizqln4js5uo6o5dnkkjadcq5apw37" CACHE PATH "")

set(TRIBOL_DIR "${TPL_ROOT}/tribol-0.1.0.18-neize4eyd3wxmeuui44az2x5ecqfe2pd" CACHE PATH "")

set(ENZYME_DIR "/usr/WS2/smithdev/toss_4_x86_64_ib_cray/enzyme/rocm-6.2.1/0.0.180" CACHE PATH "")

#------------------------------------------------------------------------------
# Devtools
#------------------------------------------------------------------------------

# Code checks disabled due to disabled devtools

set(SERAC_ENABLE_CODE_CHECKS OFF CACHE BOOL "")

set(ENABLE_CLANGFORMAT OFF CACHE BOOL "")

set(ENABLE_CLANGTIDY OFF CACHE BOOL "")

set(ENABLE_DOCS OFF CACHE BOOL "")

set(SERAC_ENABLE_CODEVELOP ON CACHE BOOL "")

set(SERAC_DISABLE_TRIBOL ON CACHE BOOL "")

Loading
Loading