GPU aware transfers in Redecomp

.clangd

@@ -0,0 +1,17 @@
+CompileFlags:
+  # 1. Remove NVCC-specific flags that Clang doesn't understand
+  Remove: 
+    - -forward-unknown-to-host-compiler
+    - -gencode*
+    - --generate-code*
+    - --options-file*
+    - -Xcompiler*
+    - --compiler-options*
+    - -ccbin*
+    - -restrict
+    - --expt-extended-lambda
+    - -rdc*
+  Add:
+    - -xcuda
+    - --cuda-path=/usr/local/cuda
+    - -std=c++17  # Ensure this matches your project's standard

cmake/Options.cmake

@@ -7,11 +7,16 @@
 cmake_dependent_option(TRIBOL_USE_CUDA "Enables Tribol with CUDA support" ON "ENABLE_CUDA" OFF)
 cmake_dependent_option(TRIBOL_USE_HIP "Enables Tribol with HIP support" ON "ENABLE_HIP" OFF)
 cmake_dependent_option(TRIBOL_USE_MPI "Enables MPI in Tribol" ON "ENABLE_MPI" OFF)
+cmake_dependent_option(TRIBOL_USE_GPU_MPI "Enables GPU-aware MPI in Tribol" ON "ENABLE_GPU_MPI" OFF)
 cmake_dependent_option(TRIBOL_USE_OPENMP "Enables Tribol with OpenMP support" ON "ENABLE_OPENMP" OFF)
 cmake_dependent_option(TRIBOL_ENABLE_TESTS "Enables Tribol Tests" ON "ENABLE_TESTS" OFF)
 cmake_dependent_option(TRIBOL_ENABLE_EXAMPLES "Enables Tribol Examples" ON "ENABLE_EXAMPLES" OFF)
 cmake_dependent_option(TRIBOL_ENABLE_DOCS "Enables Tribol Docs" ON "ENABLE_DOCS" OFF)
 
+if(TRIBOL_USE_GPU_MPI AND NOT (TRIBOL_USE_CUDA OR TRIBOL_USE_HIP))
+    message(FATAL_ERROR "TRIBOL_USE_GPU_MPI requires either TRIBOL_USE_CUDA or TRIBOL_USE_HIP")
+endif()
+
 option(TRIBOL_USE_SINGLE_PRECISION "Use single-precision floating point" OFF)
 option(TRIBOL_USE_64BIT_INDEXTYPE "Use 64-bit index type" OFF)
 

cmake/SetupThirdParty.cmake

@@ -18,6 +18,9 @@ include(CMakeFindDependencyMacro)
 #------------------------------------------------------------------------------
 if(TRIBOL_USE_CUDA)
   set(tribol_device_depends blt::cuda CACHE STRING "" FORCE)
+
+  # This lets clangd see includes with CUDA builds
+  set(CMAKE_CUDA_USE_RESPONSE_FILE_FOR_INCLUDES OFF)
 endif()
 if(TRIBOL_USE_HIP)
   set(tribol_device_depends blt::hip CACHE STRING "" FORCE)

cmake/tribol-config.cmake.in

@@ -35,6 +35,7 @@ if(NOT TRIBOL_FOUND)
   # Config options
   set(TRIBOL_USE_SINGLE_PRECISION "@TRIBOL_USE_SINGLE_PRECISION@")
   set(TRIBOL_USE_64BIT_INDEXTYPE  "@TRIBOL_USE_64BIT_INDEXTYPE@")
+  set(TRIBOL_USE_GPU_MPI          "@TRIBOL_USE_GPU_MPI@")
 
   # TPLs
   set(TRIBOL_USE_UMPIRE  "@TRIBOL_USE_UMPIRE@")

scripts/spack/configs/linux_ubuntu_24/spack.yaml

@@ -32,17 +32,13 @@ spack:
       when: '%cxx'
     - spec: '%fortran=gcc'
       when: '%fortran'
-    - spec: '%openmpi@4.1.6'
-      when: '%mpi'
     gcc_13:
     - spec: '%c=gcc'
       when: '%c'
     - spec: '%cxx=gcc'
       when: '%cxx'
     - spec: '%fortran=gcc'
       when: '%fortran'
-    - spec: '%openmpi@4.1.6'
-      when: '%mpi'
 
   packages:
     #Compiler packages
@@ -74,8 +70,6 @@ spack:
       buildable: false
     libllvm:
       buildable: false
-    mpi:
-      buildable: false
     D:
       buildable: false
     golang:
@@ -84,163 +78,198 @@ spack:
       buildable: false
     c:
       buildable: false
-    grep:
+    autoconf:
       externals:
-      - spec: grep@3.11
+      - spec: autoconf@2.71
         prefix: /usr
       buildable: false
-    openssh:
+    automake:
       externals:
-      - spec: openssh@9.6p1
+      - spec: automake@1.16.5
         prefix: /usr
       buildable: false
-    hwloc:
+    bash:
       externals:
-      - spec: hwloc@2.10.0
+      - spec: bash@5.2.21
         prefix: /usr
       buildable: false
-    groff:
+    bzip2:
       externals:
-      - spec: groff@1.23.0
+      - spec: bzip2@1.0.8
         prefix: /usr
       buildable: false
-    file:
+    binutils:
       externals:
-      - spec: file@5.45
+      - spec: binutils@2.42+gold~headers
         prefix: /usr
       buildable: false
-    lua:
+    cmake:
       externals:
-      - spec: lua@5.2
+      - spec: cmake@3.28.3
         prefix: /usr
       buildable: false
-    unzip:
+    coreutils:
       externals:
-      - spec: unzip@6.0
+      - spec: coreutils@9.4
         prefix: /usr
       buildable: false
-    ncurses:
+    cuda:
       externals:
-      - spec: ncurses@6.4.20240113+termlib abi=6
+      - spec: cuda@12.8.93
+        prefix: /usr/local/cuda
+      buildable: false
+    curl:
+      externals:
+      - spec: curl@8.5.0+gssapi+ldap+nghttp2
         prefix: /usr
       buildable: false
-    openssl:
+    diffutils:
       externals:
-      - spec: openssl@3.0.13
+      - spec: diffutils@3.10
         prefix: /usr
       buildable: false
-    binutils:
+    file:
       externals:
-      - spec: binutils@2.42+gold~headers
+      - spec: file@5.45
         prefix: /usr
       buildable: false
-    git:
+    findutils:
       externals:
-      - spec: git@2.43.0~tcltk
+      - spec: findutils@4.9.0
         prefix: /usr
       buildable: false
-    bash:
+    flex:
       externals:
-      - spec: bash@5.2.21
+      - spec: flex@2.6.4
         prefix: /usr
       buildable: false
-    coreutils:
+    gawk:
       externals:
-      - spec: coreutils@9.4
+      - spec: gawk@5.2.1
         prefix: /usr
       buildable: false
-    cuda:
+    gettext:
       externals:
-      - spec: cuda@12.8.93
-        prefix: /usr/local/cuda
+      - spec: gettext@0.21
+        prefix: /usr
+      buildable: false
+    git:
+      externals:
+      - spec: git@2.43.0~tcltk
+        prefix: /usr
+      buildable: false
+    gmake:
+      externals:
+      - spec: gmake@4.3
+        prefix: /usr
       buildable: false
     gnupg:
       externals:
       - spec: gnupg@2.4.4
         prefix: /usr
       buildable: false
-    diffutils:
+    grep:
       externals:
-      - spec: diffutils@3.10
+      - spec: grep@3.11
         prefix: /usr
       buildable: false
-    rsync:
+    groff:
       externals:
-      - spec: rsync@3.2.7
+      - spec: groff@1.23.0
         prefix: /usr
       buildable: false
-    gmake:
+    hwloc:
       externals:
-      - spec: gmake@4.3
+      - spec: hwloc@2.10.0
         prefix: /usr
       buildable: false
-    cmake:
+    libevent:
       externals:
-      - spec: cmake@3.28.3
+      - spec: libevent@2.1.12
         prefix: /usr
       buildable: false
     libfuse:
       externals:
       - spec: libfuse@3.14.0
         prefix: /usr
       buildable: false
-    python:
+    libtool:
       externals:
-      - spec: python@3.12.3+bz2+crypt+ctypes+dbm+lzma+pyexpat~pythoncmd+readline+sqlite3+ssl~tkinter+uuid+zlib
+      - spec: libtool@2.4.7
         prefix: /usr
       buildable: false
-    bzip2:
+    lua:
       externals:
-      - spec: bzip2@1.0.8
+      - spec: lua@5.2
         prefix: /usr
       buildable: false
-    xz:
+    m4:
       externals:
-      - spec: xz@5.4.5
+      - spec: m4@1.4.19
         prefix: /usr
       buildable: false
-    tar:
+    ncurses:
       externals:
-      - spec: tar@1.35
+      - spec: ncurses@6.4.20240113+termlib abi=6
         prefix: /usr
       buildable: false
-    gettext:
+    numactl:
       externals:
-      - spec: gettext@0.21
+      - spec: numactl@2.0.18
         prefix: /usr
       buildable: false
-    openmpi:
+    openssh:
       externals:
-      - spec: openmpi@4.1.6~cuda+cxx~cxx_exceptions+java~memchecker+pmi~static~wrapper-rpath fabrics=ofi,psm,psm2,ucx schedulers=slurm
+      - spec: openssh@9.6p1
         prefix: /usr
       buildable: false
-    curl:
+    openssl:
       externals:
-      - spec: curl@8.5.0+gssapi+ldap+nghttp2
+      - spec: openssl@3.0.13
+        prefix: /usr
+      buildable: false
+    pkgconf:
+      externals:
+      - spec: pkgconf@1.8.1
         prefix: /usr
       buildable: false
     perl:
       externals:
       - spec: perl@5.38.2~cpanm+opcode+open+shared+threads
         prefix: /usr
       buildable: false
+    pmix:
+      externals:
+      - spec: pmix@5.0.1
+        prefix: /usr
+      buildable: false
+    python:
+      externals:
+      - spec: python@3.12.3+bz2+crypt+ctypes+dbm+lzma+pyexpat~pythoncmd+readline+sqlite3+ssl~tkinter+uuid+zlib
+        prefix: /usr
+      buildable: false
+    rsync:
+      externals:
+      - spec: rsync@3.2.7
+        prefix: /usr
+      buildable: false
     sed:
       externals:
       - spec: sed@4.9
         prefix: /usr
       buildable: false
-    findutils:
+    tar:
       externals:
-      - spec: findutils@4.9.0
+      - spec: tar@1.35
         prefix: /usr
       buildable: false
-    gawk:
+    unzip:
       externals:
-      - spec: gawk@5.2.1
+      - spec: unzip@6.0
         prefix: /usr
       buildable: false
-    elfutils:
+    xz:
       externals:
-      - spec: elfutils@0.190
+      - spec: xz@5.4.5
         prefix: /usr
       buildable: false

scripts/spack/configs/versions.yaml

@@ -20,7 +20,7 @@ packages:
     - spec: "@2.26.0"
   mfem:
     require:
-    - spec: "@4.8.0.1"
+    - spec: "@4.9.0.2"
   petsc:
     require:
     - spec: "@3.21.6"

scripts/spack/packages/mfem/package.py

@@ -11,7 +11,7 @@ class Mfem(BuiltinMfem):
     # Note: Make sure this sha coincides with the git submodule
     # Note: We add a number to the end of the real version number to indicate that we have
     # moved forward past the release. Increment the last number when updating the commit sha.
-    version("4.9.0", commit="d9d6526cc1749980a2ba1da16e2c1ca1e07d82ec")
+    version("4.9.0.2", commit="72805e340996a73179082bbe1c7bfb785c2afb47")
 
     variant('asan', default=False, description='Add Address Sanitizer flags')
 

scripts/spack/specs.json

@@ -28,6 +28,7 @@
 
     "linux_ubuntu_24":
     [ "+raja+umpire+enzyme+profiling ^hypre+int64 %clang_19",
-      "+cuda+raja+umpire+enzyme+profiling cuda_arch=86 ^hypre+int64 %clang_19" ]
+      "+cuda+raja+umpire+enzyme cuda_arch=89 ^openmpi+cuda cuda_arch=89 fabrics=ucx ^ucx+cuda cuda_arch=89 ^hypre+int64 %clang_19"
+    ]
 
 }

src/redecomp/CMakeLists.txt

@@ -10,6 +10,7 @@ set( redecomp_headers
      MultiRedecomp.hpp
      RedecompMesh.hpp
      RedecompTransfer.hpp
+     common/CoordList.hpp
      common/TypeDefs.hpp
      partition/PartitionElements.hpp
      partition/PartitionEntity.hpp