Skip to content

Update MFEM and Axom + build enzyme on toss4 cray systems #198

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
chapman39 wants to merge 6 commits into
base: develop
Choose a base branch
from
Open

Update MFEM and Axom + build enzyme on toss4 cray systems #198

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
aee717d
update LC host configs
chapman39 Feb 23, 2026
f491ac9
Merge remote-tracking branch 'origin/develop' into task/chapman39/upd…
chapman39 Feb 23, 2026
0ad5f0d
add release ci check and no redecomp check
chapman39 Feb 27, 2026
9e07e60
ci check without optional dependencies, plus fixes
chapman39 Feb 27, 2026
37db0db
Merge branch 'develop' into task/chapman39/update-tpls-2026-02-05
chapman39 Mar 9, 2026
3fed907
remove adiak dir, fix pipeline ci name
chapman39 Mar 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 11 additions & 2 deletions .gitlab/build_toss4.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
when: never
- when: on_success
before_script:
- module load cmake/3.23.1
- module load cmake/3.30.5


####
Expand All @@ -38,7 +38,7 @@ toss4-llvm_19_1_3-src:
ALLOC_TIME: "30"
extends: .src_build_on_toss4

toss4-llvm_19_1_3-src-codevelop:
Copy link
Collaborator

@srwopschall srwopschall Feb 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Verify that this can be removed.

Copy link
Collaborator Author

@chapman39 chapman39 Feb 24, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i removed this pipeline because the codevelop name was misleading. however i notice now that by default tribol builds as Debug, and this test tests tribol Release builds. if we still want to test Release builds, i can re-add this, and give it a more proper name. e.g. toss4-llvm_19_1_3-src-release

Copy link
Member

@ebchin ebchin Feb 27, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We used to have a no redecomp CI build on toss 4, do you think we could add one back in (in a different PR, potentially) @chapman39? Preferably with no optional TPLs (i.e. no RAJA, no Umpire, no Enzyme)?

Regarding a release build, since we've seen different Enzyme behavior on release vs. debug builds, I think we also need a release build in CI. toss4-llvm_19_1_3-src-release sounds good to me!

toss4-llvm_19_1_3-src-release:
variables:
COMPILER: "llvm@19.1.3"
HOST_CONFIG: "dane-toss_4_x86_64_ib-${COMPILER}.cmake"
Expand All @@ -47,6 +47,15 @@ toss4-llvm_19_1_3-src-codevelop:
ALLOC_TIME: "30"
extends: .src_build_on_toss4

toss4-llvm_19_1_3-src-no-tpls:
variables:
COMPILER: "llvm@19.1.3"
HOST_CONFIG: "dane-toss_4_x86_64_ib-${COMPILER}.cmake"
EXTRA_CMAKE_OPTIONS: "-DBUILD_REDECOMP=OFF -URAJA_DIR -UUMPIRE_DIR -UENZYME_DIR -DTRIBOL_USE_OPENMP=OFF -DENABLE_OPENMP=ON"
ALLOC_NODES: "1"
ALLOC_TIME: "30"
extends: .src_build_on_toss4

toss4-gcc_13_3_1-src:
variables:
COMPILER: "gcc@13.3.1"
Expand Down
2 changes: 1 addition & 1 deletion .gitlab/build_toss4_cray.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
when: never
- when: on_success
before_script:
- module load cmake/3.23.1
- module load cmake/3.29.2

####
# Templates
Expand Down
30 changes: 15 additions & 15 deletions host-configs/dane-toss_4_x86_64_ib-gcc@13.3.1.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,31 +1,31 @@
#------------------------------------------------------------------------------
# !!!! This is a generated file, edit at own risk !!!!
#------------------------------------------------------------------------------
# CMake executable path: /usr/tce/bin/cmake
# CMake executable path: /usr/bin/cmake
#------------------------------------------------------------------------------

set(CMAKE_PREFIX_PATH "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/caliper-2.12.1-cg25q356sb7kiuabwtfka4tywsgbtpiu;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/tribol-0.1.0.23-ysqjlxfxls46fsccfvnaw3teuqj7jp3f;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/adiak-0.4.1-2z3ogwq4rmbuwpc5vl2mgxymme4mfbke;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/axom-0.12.0.0-i37xne6rixnhgwrptvzzfbrgcmdryhoo;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/conduit-0.9.5-bb722yzgl4d3g2diyieu73hrkt732qez;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/mfem-4.9.0.2-rnhik76bnx6j6665kwbnzm355av6lpmp;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/raja-2025.09.0-ume2ifdwlyie3yynwdvhyf2uxjykqpvc;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/umpire-2025.09.0-ir2e37p6sqldodud4cwaezspl2co5v42;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/netcdf-c-4.7.4-zslniorcg7ki7lnsimctkn6ig3zg2qaw;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/slepc-3.21.2-y7425db4zsffjhcc3d33o2s2pz7w6664;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/sundials-6.7.0-7azsxv7nm2sr5pyi6px6abcdb6l475bv;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/camp-2025.09.2-qei7a63hp4u37cas2m7zte3phmv2fyhd;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/fmt-11.0.2-usm3fj6qxr75ga5p2ydlhearxsbtpckd;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/hdf5-1.8.23-frgmbp2plziqwxwypvemeglz6semjqei;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/arpack-ng-3.9.1-foedq43u27rbxrupjqborfxd47smzuch;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/petsc-3.21.6-jfsq7enjou6z5mzggfqndbmtc3sz3jy2;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/hypre-2.33.0-23ph5bzk62r3tntdn277zrtv6dpmon2n;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/strumpack-8.0.0-zn5qsdclfhyvrubcweeqymoptoqm3vja;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/superlu-dist-8.1.2-quwnz5kk2xb42g3j7vejc6hrq2urngof;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/netlib-scalapack-2.2.2-4utyi6inwwno5hssouh7566hmveptxtb;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/parmetis-4.0.3-c7wowvg6qcowqn53dlxtq4msf5m7tfo7;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/metis-5.1.0-nbxqo5m7iy3uyjd32skqzpa4d5cev2dt;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/none-none/gcc-runtime-13.3.1-uv6f6pcmzth37hskwtrhtbmbujh53k6y;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/none-none/compiler-wrapper-1.0-hxzxhboumtniaqs7qivgliluv4h2lbxt;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/python-3.11.7;/usr/tce;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/cppcheck-2.9;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/doxygen-1.9.8;/usr/tce/packages/gcc/gcc-13.3.1;/usr/tce/packages/clang/clang-19.1.3;/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-13.3.1-magic;/usr/tce/packages/python/python-3.9.12" CACHE STRING "")
set(CMAKE_PREFIX_PATH "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/caliper-2.12.1-r4enksghxtrrlgn4bd4z7v4rkzsq7ctx;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/tribol-0.1.0.25-jbvcutayyr2mjoqu7dmkjf6eapjrwh3m;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/adiak-0.4.1-l6ht437dth4h5vj42eybrzayi3tnelgf;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/axom-0.12.0.1-o47bcdfl72xt4qenhbrnhybpbxthhgv6;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/conduit-0.9.5-22qehlaxxv3vutiy5batldrwalqvqdbh;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/mfem-4.9.1.1-ybgs32e3fejsyluja5viharlhbjc2pwr;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/raja-2025.09.0-n2e25hdkwjdeqhcej2loxyb2f4zz6j3c;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/umpire-2025.09.0-rbrxustwpp23jg45xabqcdfv4pbbyxw2;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/netcdf-c-4.7.4-ef2boturomxh7h3zsq2bnmxdqngvpzmz;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/slepc-3.21.2-zag2z75wlhi6vm4nnqatzv2msypckqdj;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/sundials-7.5.0-5mjejit3ofrg24snbuasxapzkcslp4sn;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/superlu-dist-8.1.2-yenosrngdyvgxmai3fr5uqfa3df5ywi3;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/camp-2025.09.2-y42u7xoj3wzkmcm5oppqynjgp6xleyuz;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/fmt-11.0.2-2siwhudpthcvcnsimjhoqzbqj2d35xbu;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/hdf5-1.8.23-rhrfxq6chtmc745bmdyzj4i345gkzuok;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/arpack-ng-3.9.1-mrwsueljnp35iyg2tusqbbz75k5rpace;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/petsc-3.21.6-4t5kick6z26ichvysg55gpxb4scsklwi;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/hypre-2.33.0-jirjzhbsayig3mwuh6xtudax6zsrrmce;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/strumpack-8.0.0-64bol65qlruojtaxcsmet2jddsp27scq;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/netlib-scalapack-2.2.2-75rjlbkhicnfdgu7q5fpgk626x7gkdza;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/parmetis-4.0.3-6sb6x2i5ujyis46u6oqfra544x2inln5;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/metis-5.1.0-prmgy5uekkdvunr4yh4vztw2enjpurfr;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/none-none/gcc-runtime-13.3.1-zsobnkwck2glzcbukupgvgbiwwhdonys;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/none-none/compiler-wrapper-1.0-uesxbqunebynqp5g2i6r3corj4oval5q;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/python-3.11.7;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/cppcheck-2.9;/usr/workspace/smithdev/devtools/toss_4_x86_64_ib/latest/doxygen-1.9.8;/usr/tce/packages/gcc/gcc-13.3.1;/usr/tce/packages/clang/clang-19.1.3;/usr/tce/packages/mvapich2/mvapich2-2.3.7-gcc-13.3.1-magic;/usr/tce/packages/python/python-3.9.12" CACHE STRING "")

set(CMAKE_INSTALL_RPATH_USE_LINK_PATH "ON" CACHE STRING "")

set(CMAKE_BUILD_RPATH "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/smith-develop-iijxlgvwfrtse2ytqlu62iwx7b6dvx3v/lib;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/smith-develop-iijxlgvwfrtse2ytqlu62iwx7b6dvx3v/lib64;;/collab/usr/global/tools/tce4/packages/gcc/gcc-13.3.1/lib/gcc/x86_64-redhat-linux/13" CACHE STRING "")
set(CMAKE_BUILD_RPATH "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/smith-develop-arwwxcnjc6rkjns4dm3gx75kax437xm3/lib;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/smith-develop-arwwxcnjc6rkjns4dm3gx75kax437xm3/lib64;;/collab/usr/global/tools/tce4/packages/gcc/gcc-13.3.1/lib/gcc/x86_64-redhat-linux/13" CACHE STRING "")

set(CMAKE_INSTALL_RPATH "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/smith-develop-iijxlgvwfrtse2ytqlu62iwx7b6dvx3v/lib;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1/smith-develop-iijxlgvwfrtse2ytqlu62iwx7b6dvx3v/lib64;;/collab/usr/global/tools/tce4/packages/gcc/gcc-13.3.1/lib/gcc/x86_64-redhat-linux/13" CACHE STRING "")
set(CMAKE_INSTALL_RPATH "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/smith-develop-arwwxcnjc6rkjns4dm3gx75kax437xm3/lib;/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1/smith-develop-arwwxcnjc6rkjns4dm3gx75kax437xm3/lib64;;/collab/usr/global/tools/tce4/packages/gcc/gcc-13.3.1/lib/gcc/x86_64-redhat-linux/13" CACHE STRING "")

set(CMAKE_BUILD_TYPE "Release" CACHE STRING "")

#------------------------------------------------------------------------------
# Compilers
#------------------------------------------------------------------------------
# Compiler Spec: gcc@13.3.1/hpiynx6mdjb57v77us3h6yyzxymdbr6d
# Compiler Spec: gcc@13.3.1/xrm3n2pex2en2wvm6p4ap62nieoz7kne
#------------------------------------------------------------------------------
if(DEFINED ENV{SPACK_CC})

set(CMAKE_C_COMPILER "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/none-none/compiler-wrapper-1.0-hxzxhboumtniaqs7qivgliluv4h2lbxt/libexec/spack/gcc/gcc" CACHE PATH "")
set(CMAKE_C_COMPILER "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/none-none/compiler-wrapper-1.0-uesxbqunebynqp5g2i6r3corj4oval5q/libexec/spack/gcc/gcc" CACHE PATH "")

set(CMAKE_CXX_COMPILER "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/none-none/compiler-wrapper-1.0-hxzxhboumtniaqs7qivgliluv4h2lbxt/libexec/spack/gcc/g++" CACHE PATH "")
set(CMAKE_CXX_COMPILER "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/none-none/compiler-wrapper-1.0-uesxbqunebynqp5g2i6r3corj4oval5q/libexec/spack/gcc/g++" CACHE PATH "")

set(CMAKE_Fortran_COMPILER "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/none-none/compiler-wrapper-1.0-hxzxhboumtniaqs7qivgliluv4h2lbxt/libexec/spack/gcc/gfortran" CACHE PATH "")
set(CMAKE_Fortran_COMPILER "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/none-none/compiler-wrapper-1.0-uesxbqunebynqp5g2i6r3corj4oval5q/libexec/spack/gcc/gfortran" CACHE PATH "")

else()

Expand Down Expand Up @@ -71,17 +71,19 @@ set(BUILD_REDECOMP ON CACHE BOOL "")
# TPLs
#------------------------------------------------------------------------------

set(TPL_ROOT "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2025_11_20_17_51_08/gcc-13.3.1" CACHE PATH "")
set(TPL_ROOT "/usr/WS2/smithdev/libs/smith/toss_4_x86_64_ib/2026_02_13_17_29_25/gcc-13.3.1" CACHE PATH "")

set(AXOM_DIR "${TPL_ROOT}/axom-0.12.0.0-i37xne6rixnhgwrptvzzfbrgcmdryhoo" CACHE PATH "")
set(AXOM_DIR "${TPL_ROOT}/axom-0.12.0.1-o47bcdfl72xt4qenhbrnhybpbxthhgv6" CACHE PATH "")

set(MFEM_DIR "${TPL_ROOT}/mfem-4.9.0.2-rnhik76bnx6j6665kwbnzm355av6lpmp" CACHE PATH "")
set(MFEM_DIR "${TPL_ROOT}/mfem-4.9.1.1-ybgs32e3fejsyluja5viharlhbjc2pwr" CACHE PATH "")

set(CALIPER_DIR "${TPL_ROOT}/caliper-2.12.1-r4enksghxtrrlgn4bd4z7v4rkzsq7ctx" CACHE PATH "")

# ENZYME not built

set(RAJA_DIR "${TPL_ROOT}/raja-2025.09.0-ume2ifdwlyie3yynwdvhyf2uxjykqpvc" CACHE PATH "")
set(RAJA_DIR "${TPL_ROOT}/raja-2025.09.0-n2e25hdkwjdeqhcej2loxyb2f4zz6j3c" CACHE PATH "")

set(UMPIRE_DIR "${TPL_ROOT}/umpire-2025.09.0-ir2e37p6sqldodud4cwaezspl2co5v42" CACHE PATH "")
set(UMPIRE_DIR "${TPL_ROOT}/umpire-2025.09.0-rbrxustwpp23jg45xabqcdfv4pbbyxw2" CACHE PATH "")

#------------------------------------------------------------------------------
# Devtools
Expand All @@ -104,5 +106,3 @@ set(SPHINX_EXECUTABLE "${DEVTOOLS_ROOT}/python-3.11.7/bin/sphinx-build" CACHE PA
set(CPPCHECK_EXECUTABLE "${DEVTOOLS_ROOT}/cppcheck-2.9/bin/cppcheck" CACHE PATH "")

set(DOXYGEN_EXECUTABLE "${DEVTOOLS_ROOT}/doxygen-1.9.8/bin/doxygen" CACHE PATH "")


Loading