ROCm TPL builds on TOSS4 Cray systems + general TPL update

.github/workflows/ci-tests.yml

@@ -12,9 +12,9 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }}
 
 env:
-  CLANG_DOCKER_IMAGE: seracllnl/tpls:clang-19_10-09-25_23h-54m
+  CLANG_DOCKER_IMAGE: seracllnl/tpls:clang-19_11-24-25_18h-57m
   CUDA_DOCKER_IMAGE: seracllnl/tpls:cuda-12_04-16-25_20h-55m
-  GCC_DOCKER_IMAGE: seracllnl/tpls:gcc-14_10-09-25_23h-54m
+  GCC_DOCKER_IMAGE: seracllnl/tpls:gcc-14_11-24-25_18h-57m
 
 jobs:
   # Hacky solution to reference env variables outside of `run` steps https://stackoverflow.com/a/74217028
@@ -63,8 +63,8 @@ jobs:
     container:
       image: ${{ matrix.config.compiler_image }}
       volumes:
-      - /home/serac/serac
-      # Required - default is set to user "serac"
+      - /home/smith/smith
+      # Required - default is set to user "smith"
       options: --user root
     steps:
       - name: Checkout Smith
@@ -106,8 +106,8 @@ jobs:
     container:
       image: ${{ needs.set_image_vars.outputs.clang_docker_image }}
       volumes:
-      - /home/serac/serac
-      # Required - default is set to user "serac"
+      - /home/smith/smith
+      # Required - default is set to user "smith"
       options: --user root
       env:
         CHECK_TYPE: ${{ matrix.check_type }}

.github/workflows/docker_build_tpls.yml

@@ -14,7 +14,7 @@ jobs:
         dockerfile_suffix: [clang-19, gcc-14] #, cuda-12]
     env:
       REPO: seracllnl/tpls
-      HOSTCONFIG_LOC: /home/serac/export_hostconfig
+      HOSTCONFIG_LOC: /home/smith/export_hostconfig
       DOCKERFILE_PREFIX: ./scripts/docker/dockerfile_
     steps:
     - name: Extract branch name

.gitlab/build_toss4_cray.yml

@@ -3,9 +3,11 @@
 .on_toss4_cray:
   variables:
     SCHEDULER_PARAMETERS: "--queue pci --exclusive --time-limit=${ALLOC_TIME}m --nodes=${ALLOC_NODES}"
+    COMPILER: "[email protected]"
+    HOST_CONFIG: "tuolumne-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake"
   tags:
     - batch
-    - tioga
+    - tuolumne
   rules:
     - if: '$CI_COMMIT_BRANCH =~ /_qnone/ || $ON_TOSS4_CRAY == "OFF"' #run except if ...
       when: never
@@ -32,48 +34,39 @@
 ####
 # Build jobs
 
-toss4_cray-rocmcc_6_2_1-src:
+toss4_cray-llvm_amdgpu_6_4_2-src:
   variables:
-    COMPILER: "[email protected]"
-    HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake"
     EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF -DCMAKE_BUILD_TYPE=Debug"
     ALLOC_NODES: "1"
-    ALLOC_TIME: "30"
+    ALLOC_TIME: "60"
   extends: .src_build_on_toss4_cray
 
-toss4_cray-rocmcc_6_2_1-src-codevelop:
+toss4_cray-llvm_amdgpu_6_4_2-src-codevelop:
   variables:
-    COMPILER: "[email protected]"
-    HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake"
-    # TODO: Add -DSMITH_USE_DFEM=ON when TPLs are updated
+    # TODO: Add -DSMITH_USE_DFEM=ON when enzyme is enabled in toss4_cray host configs
     EXTRA_CMAKE_OPTIONS: "-DENABLE_BENCHMARKS=ON -DENABLE_DOCS=OFF -DCMAKE_BUILD_TYPE=Debug -DSMITH_ENABLE_CODEVELOP=ON"
     EXTRA_BUILD_OPTIONS: "--skip-install"
     ALLOC_NODES: "1"
-    ALLOC_TIME: "30"
+    ALLOC_TIME: "60"
   extends: .src_build_on_toss4_cray
 
 # NOTE: SPEC should matches specs.json, but devtools and profiling variants removed
-toss4_cray-rocmcc_6_2_1-tpl:
+toss4_cray-llvm_amdgpu_6_4_2-tpl:
   variables:
-    COMPILER: "[email protected]"
-    SPEC: "--spec %${COMPILER}~openmp+rocm~petsc amdgpu_target=gfx90a ^[email protected] ^[email protected] ^[email protected] ^[email protected] ^hdf5 cflags=-Wno-int-conversion"
+    SPEC: "--spec %~devtools~profiling~openmp+rocm~petsc~slepc~enzyme~sundials amdgpu_target=gfx942,gfx90a %${COMPILER}"
     ALLOC_NODES: "1"
-    ALLOC_TIME: "45"
+    ALLOC_TIME: "90"
   extends: .tpl_build_on_toss4_cray
 
-toss4_cray-rocmcc_6_2_1-examples:
+toss4_cray-llvm_amdgpu_6_4_2-examples:
   variables:
-    COMPILER: "[email protected]"
-    HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake"
     EXTRA_CMAKE_OPTIONS: "-DCMAKE_BUILD_TYPE=Debug"
     ALLOC_NODES: "1"
-    ALLOC_TIME: "45"
+    ALLOC_TIME: "90"
   extends: .examples_build_on_toss4_cray
 
-toss4_cray-rocmcc_6_2_1-benchmarks:
+toss4_cray-llvm_amdgpu_6_4_2-benchmarks:
   variables:
-    COMPILER: "[email protected]"
-    HOST_CONFIG: "tioga-toss_4_x86_64_ib_cray-${COMPILER}_hip.cmake"
     ALLOC_NODES: "1"
-    ALLOC_TIME: "60"
+    ALLOC_TIME: "120"
   extends: .benchmarks_build_on_toss4_cray

.uberenv_config.json

@@ -4,7 +4,7 @@
 "package_final_phase" : "initconfig",
 "package_source_dir" : "../..",
 "spack_url": "https://github.com/spack/spack.git",
-"spack_commit": "b4b1c5f271c75e918ce922ff1bcc77542eda3ec9",
+"spack_commit": "734c5db2121b01c373eed6538e452f18887e9e44",
 "spack_packages_url": "https://github.com/spack/spack-packages.git",
 "spack_packages_commit": "a75a7f75182ffc7a51c6ca7f0fec4bf9b2705be8",
 "spack_configs_path": "scripts/spack/configs",

cmake/thirdparty/SetupSmithThirdParty.cmake

@@ -283,7 +283,6 @@ if (NOT SMITH_THIRD_PARTY_LIBRARIES_FOUND)
             set(ParMETIS_DIR ${PARMETIS_DIR} CACHE PATH "")
         endif()
         set(MFEM_USE_OPENMP ${SMITH_ENABLE_OPENMP} CACHE BOOL "")
-
         if(PETSC_DIR)
             set(MFEM_USE_PETSC ON CACHE BOOL "")
             set(PETSC_ARCH "" CACHE STRING "")
@@ -297,8 +296,6 @@ if (NOT SMITH_THIRD_PARTY_LIBRARIES_FOUND)
             set(MFEM_USE_PETSC OFF CACHE BOOL "")
             set(MFEM_USE_SLEPC OFF CACHE BOOL "")
         endif()
-
-        set(MFEM_USE_RAJA OFF CACHE BOOL "")
         set(MFEM_USE_SUNDIALS ${SMITH_USE_SUNDIALS} CACHE BOOL "")
         if(SUPERLUDIST_DIR)
             smith_assert_is_directory(DIR_VARIABLE SUPERLUDIST_DIR)
@@ -326,7 +323,6 @@ if (NOT SMITH_THIRD_PARTY_LIBRARIES_FOUND)
                 "Additional packages required by STRUMPACK.")
             set(STRUMPACK_TARGET_NAMES STRUMPACK::strumpack CACHE STRING "")
         endif()
-        set(MFEM_USE_UMPIRE OFF CACHE BOOL "")
         set(MFEM_USE_ZLIB ON CACHE BOOL "")
         if(ENZYME_DIR)
             smith_assert_is_directory(DIR_VARIABLE ENZYME_DIR)
@@ -335,6 +331,15 @@ if (NOT SMITH_THIRD_PARTY_LIBRARIES_FOUND)
             set(MFEM_USE_ENZYME OFF CACHE BOOL "")
         endif()
 
+        # MFEM uses Raja/ Umpire if GPU enabled
+        if (SMITH_ENABLE_HIP OR SMITH_ENABLE_CUDA)
+            set(MFEM_USE_RAJA ON CACHE BOOL "")
+            set(MFEM_USE_UMPIRE ON CACHE BOOL "")
+        else()
+            set(MFEM_USE_RAJA OFF CACHE BOOL "")
+            set(MFEM_USE_UMPIRE OFF CACHE BOOL "")
+        endif()
+
         #### MFEM Configuration Options
 
         # Prefix the "check" targets

config-build.py

@@ -23,10 +23,10 @@
 _host_configs_map = {"rzgenie"   : "[email protected]",
                      "rzwhippet" : "[email protected]",
                      "rzvernal"  : "[email protected]_hip.cmake",
-                     "rzansel"   : "[email protected]_cuda.cmake",
+                     "rzadams"   : "[email protected]_hip.cmake",
                      "dane"      : "[email protected]",
-                     "tioga"     : "tioga-toss_4_x86_64_ib_cray-rocmcc@6.2.1_hip.cmake",
-                     "lassen"    : "[email protected]_cuda.cmake"}
+                     "tioga"     : "tuolumne-toss_4_x86_64_ib_cray-llvm-amdgpu@6.4.2_hip.cmake",
+                     "tuolumne"  : "[email protected]_hip.cmake"}
 
 def get_machine_name():
     return socket.gethostname().rstrip('1234567890')

examples/using-with-cmake/CMakeLists.txt

@@ -4,6 +4,14 @@ project(smith_example LANGUAGES C CXX)
 
 message(STATUS "CMake Version: ${CMAKE_VERSION}")
 
+if (ENABLE_CUDA)
+    enable_language(CUDA)
+endif()
+
+if (ENABLE_HIP)
+    enable_language(HIP)
+endif()
+
 include(CMakeFindDependencyMacro)
 find_dependency(smith REQUIRED NO_DEFAULT_PATH PATHS "${SMITH_DIR}/lib/cmake")
 
@@ -46,6 +54,11 @@ endforeach()
 add_executable(smith_example smith_example.cpp)
 target_link_libraries(smith_example smith::smith)
 
+# Compile as HIP source if enabled
+if (ENABLE_HIP)
+    set_source_files_properties(smith_example.cpp PROPERTIES LANGUAGE HIP)
+endif()
+
 enable_testing()
 add_test(NAME    smith_example 
          COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 1 ./smith_example)

examples/using-with-cmake/host-config.cmake.in

@@ -30,6 +30,10 @@ set(MPIEXEC_EXECUTABLE "@MPIEXEC_EXECUTABLE@" CACHE PATH "")
 set(MPIEXEC_NUMPROC_FLAG "@MPIEXEC_NUMPROC_FLAG@" CACHE STRING "")
 
 
+# OpenMP
+set(ENABLE_OPENMP @ENABLE_OPENMP@ CACHE BOOL "")
+
+
 # CUDA
 set(ENABLE_CUDA                @ENABLE_CUDA@                  CACHE BOOL "")
 
@@ -44,5 +48,19 @@ if(ENABLE_CUDA)
 endif()
 
 
+# HIP/ ROCm
+set(ENABLE_HIP @ENABLE_HIP@ CACHE BOOL "")
+
+if(ENABLE_HIP)
+  set(ROCM_PATH               "@ROCM_PATH@"               CACHE PATH "")
+  set(CMAKE_HIP_ARCHITECTURES "@CMAKE_HIP_ARCHITECTURES@" CACHE STRING "")
+  set(CMAKE_HIP_COMPILER      "@CMAKE_HIP_COMPILER@"      CACHE FILEPATH "")
+  set(HIP_ROOT_DIR            "@HIP_ROOT_DIR@"            CACHE PATH "")
+  set(AMDGPU_TARGETS          "@AMDGPU_TARGETS@"          CACHE STRING "")
+  set(GPU_TARGETS             "@GPU_TARGETS@"             CACHE STRING "")
+  set(ROCM_ROOT_DIR           "@ROCM_ROOT_DIR@"           CACHE PATH "")
+endif()
+
+
 # Path to Smith
 set(SMITH_DIR        "@CMAKE_INSTALL_PREFIX@" CACHE PATH "")