Clean up basic kernels and CMakeLists.txt

LLNL · Jan 13, 2022 · 4888c3d · 4888c3d
1 parent f3f4a1b
commit 4888c3d
Show file tree

Hide file tree

Showing 22 changed files with 58 additions and 83 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -27,13 +27,24 @@ set(ENABLE_EXAMPLES Off CACHE BOOL "Enable RAJA examples")
 set(RAJA_ENABLE_EXERCISES Off CACHE BOOL "Enable RAJA exercises")
 
 if(ENABLE_KOKKOS)
-  set(CMAKE_CXX_STANDARD 14)
-  set(BLT_CXX_STANDARD 14)
+  set(CMAKE_CXX_STANDARD 17)
+  set(BLT_CXX_STANDARD 17)
+else()
+  set(CMAKE_CXX_STANDARD 11)
+  set(BLT_CXX_STANDARD 11)
 endif()
+
 include(blt/SetupBLT.cmake)
 
+#
+# Define RAJA PERFSUITE settings...
+#
+
+cmake_dependent_option(RAJA_PERFSUITE_ENABLE_MPI "Build with MPI" On "ENABLE_MPI" Off)
+
 #
 # Define RAJA settings...
+#
 
 set(ENABLE_TESTS Off CACHE BOOL "")
 set(ENABLE_EXAMPLES Off CACHE BOOL "")
@@ -54,8 +65,11 @@ add_subdirectory(tpl/RAJA)
 get_property(RAJA_INCLUDE_DIRS DIRECTORY tpl/RAJA PROPERTY INCLUDE_DIRECTORIES)
 include_directories(${RAJA_INCLUDE_DIRS})
 
+
 #
 # Setup variables to pass to Perf suite
+#
+
 #
 # These (hopefully temporary) macro constants are needed to work-around
 # performance issues in the xl compiler.
@@ -73,6 +87,9 @@ set(RAJA_PERFSUITE_VERSION_PATCHLEVEL 0)
 
 set(RAJA_PERFSUITE_DEPENDS RAJA)
 
+if (RAJA_PERFSUITE_ENABLE_MPI)
+  list(APPEND RAJA_PERFSUITE_DEPENDS mpi)
+endif()
 if (ENABLE_OPENMP)
   list(APPEND RAJA_PERFSUITE_DEPENDS openmp)
 endif()
@@ -81,7 +98,7 @@ if (ENABLE_CUDA)
 endif()
 
 # Kokkos requires hipcc as the CMAKE_CXX_COMPILER for HIP AMD/VEGA GPU
-# platforms, whereas RAJAPerf Suite uses blt/CMake FindHIP to set HIP compiler 
+# platforms, whereas RAJAPerf Suite uses blt/CMake FindHIP to set HIP compiler. 
 # Separate RAJAPerf Suite and Kokkos handling of HIP compilers
 
 if ((ENABLE_HIP) AND (NOT ENABLE_KOKKOS))
@@ -92,10 +109,16 @@ set(RAJAPERF_BUILD_SYSTYPE $ENV{SYS_TYPE})
 set(RAJAPERF_BUILD_HOST $ENV{HOSTNAME})
 
 if (ENABLE_CUDA)
-  if (ENABLE_CUDA AND ENABLE_KOKKOS)
-  	set(CMAKE_CUDA_STANDARD 14)
+  set(CMAKE_CUDA_STANDARD 11)
+
+# Kokkos Cuda handling
+  if (ENABLE_KOKKOS)
+    set(CMAKE_CUDA_STANDARD 17)
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -restrict --extended-lambda --expt-relaxed-constexpr")
+  else()
+    set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -restrict -arch ${CUDA_ARCH} --expt-extended-lambda --expt-relaxed-constexpr")
   endif()
-  set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -restrict -arch ${CUDA_ARCH} --expt-extended-lambda --expt-relaxed-constexpr")
+
   set(RAJAPERF_COMPILER "${CUDA_NVCC_EXECUTABLE}")
   list(APPEND RAJAPERF_COMPILER ${CMAKE_CXX_COMPILER})
   set(RAJAPERF_COMPILER_OPTIONS "${CUDA_NVCC_FLAGS}")
@@ -115,44 +138,41 @@ configure_file(${CMAKE_SOURCE_DIR}/src/rajaperf_config.hpp.in
 
 include_directories($<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/bin>)
 
-# Make sure RAJA flags propagate (we need to do some tidying to 
+# Make sure RAJA flags propagate (we need to do some tidying to
 # remove project-specific CMake variables that are no longer needed)
 set (CUDA_NVCC_FLAGS ${RAJA_NVCC_FLAGS})
 
+#
+# Each directory in the perf suite has its own CMakeLists.txt file.
+#
+
 # The statement below is required for Kokkos compilation.
 if(ENABLE_KOKKOS)
   include_directories(SYSTEM ${CMAKE_CURRENT_SOURCE_DIR}/tpl/RAJA/include/)
 endif()
 
-
 # ENABLE_KOKKOS is A RAJAPerf Suite Option
 if(ENABLE_KOKKOS)
   add_definitions(-DRUN_KOKKOS)
   if(ENABLE_HIP)
     set(Kokkos_ENABLE_HIP ON CACHE BOOL "Kokkos builds for AMD HIP set the
-Kokkos_ENABLE_HIP variable to ON")	
-    #set(Kokkos_ARCH_VEGA900 ON CACHE BOOL "Docstring") #TODO: better
+Kokkos_ENABLE_HIP variable to ON")
   endif()
+
   if(ENABLE_TARGET_OPENMP)
-    set(Kokkos_ENABLE_OPENMPTARGET ON CACHE BOOL "Docstring")	
-    set(Kokkos_ARCH_VOLTA70 ON CACHE BOOL "Docstring") #TODO: better
-    set(CMAKE_CXX_STANDARD 17)
-    set(BLT_CXX_STANDARD 17)
+    set(Kokkos_ENABLE_OPENMPTARGET ON CACHE BOOL "Docstring")
     set(RAJA_ENABLE_TARGET_OPENMP ON CACHE BOOL "Docstring")
     if(NOT CMAKE_BUILD_TYPE MATCHES Debug)
       if(NOT EXPERIMENTAL_BUILD)
-	      message(FATAL_ERROR "Kokkos builds with OpenMPTarget require a Debug build to succeed at the moment. Rebuild with CMAKE_BUILD_TYPE=Debug. If you're a compiler developer, rebuild with -DEXPERIMENTAL_BUILD=ON")
+      message(FATAL_ERROR "Kokkos builds with OpenMPTarget require a Debug build to succeed at the moment. Rebuild with CMAKE_BUILD_TYPE=Debug. If you're a compiler developer, rebuild with -DEXPERIMENTAL_BUILD=ON")
       endif()
     endif()
-
-    #add_definitions(-DRAJA_ENABLE_TARGET_OPENMP)
   endif()
 
 # ENABLE_CUDA IS A RAJA PERFSUITE OPTION
   if(ENABLE_CUDA)
-	  set(Kokkos_ENABLE_CUDA ON CACHE BOOL "Docstring")
-	  set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "Docstring")
-	  set(Kokkos_ARCH_VOLTA70 ON CACHE BOOL "Docstring") #TODO: better
+    set(Kokkos_ENABLE_CUDA ON CACHE BOOL "Docstring")
+    set(Kokkos_ENABLE_CUDA_LAMBDA ON CACHE BOOL "Docstring")
     enable_language(CUDA)
   endif()
   if(ENABLE_OPENMP)
@@ -162,7 +182,6 @@ Kokkos_ENABLE_HIP variable to ON")
   add_subdirectory(tpl/kokkos)
   get_property(KOKKOS_INCLUDE_DIRS DIRECTORY tpl/kokkos PROPERTY INCLUDE_DIRECTORIES)
   include_directories(${KOKKOS_INCLUDE_DIRS})
-
   list(APPEND RAJA_PERFSUITE_DEPENDS kokkos)
 endif()
 

diff --git a/src/basic/DAXPY.hpp b/src/basic/DAXPY.hpp
@@ -52,7 +52,6 @@ class DAXPY : public KernelBase
   void runCudaVariant(VariantID vid);
   void runHipVariant(VariantID vid);
   void runOpenMPTargetVariant(VariantID vid);
-
   void runKokkosVariant(VariantID vid);
 
 private:

diff --git a/src/basic/IF_QUAD.cpp b/src/basic/IF_QUAD.cpp
@@ -37,10 +37,6 @@ IF_QUAD::IF_QUAD(const RunParams& params)
 
   setUsesFeature(Forall);
 
-
-  setVariantDefined( Kokkos_Lambda );
-
-
   setVariantDefined( Base_Seq );
   setVariantDefined( Lambda_Seq );
   setVariantDefined( RAJA_Seq );
@@ -59,6 +55,8 @@ IF_QUAD::IF_QUAD(const RunParams& params)
   setVariantDefined( Base_HIP );
   setVariantDefined( Lambda_HIP );
   setVariantDefined( RAJA_HIP );
+
+  setVariantDefined( Kokkos_Lambda );
 }
 
 IF_QUAD::~IF_QUAD()

diff --git a/src/basic/IF_QUAD.hpp b/src/basic/IF_QUAD.hpp
@@ -69,11 +69,7 @@ class IF_QUAD : public KernelBase
   void runCudaVariant(VariantID vid);
   void runHipVariant(VariantID vid);
   void runOpenMPTargetVariant(VariantID vid);
-
   void runKokkosVariant(VariantID vid);
-
-
-
 
 private:
   Real_ptr m_a;

diff --git a/src/basic/INIT3.cpp b/src/basic/INIT3.cpp
@@ -36,24 +36,23 @@ INIT3::INIT3(const RunParams& params)
   setVariantDefined( Base_Seq );
   setVariantDefined( Lambda_Seq );
   setVariantDefined( RAJA_Seq );
-  setVariantDefined( Kokkos_Lambda );
 
   setVariantDefined( Base_OpenMP );
   setVariantDefined( Lambda_OpenMP );
   setVariantDefined( RAJA_OpenMP );
 
-
   setVariantDefined( Base_OpenMPTarget );
   setVariantDefined( RAJA_OpenMPTarget );
 
   setVariantDefined( Base_CUDA );
   setVariantDefined( Lambda_CUDA );
   setVariantDefined( RAJA_CUDA );
 
-
   setVariantDefined( Base_HIP );
   setVariantDefined( Lambda_HIP );
   setVariantDefined( RAJA_HIP );
+
+  setVariantDefined( Kokkos_Lambda );
 }
 
 INIT3::~INIT3()

diff --git a/src/basic/INIT3.hpp b/src/basic/INIT3.hpp
@@ -55,11 +55,7 @@ class INIT3 : public KernelBase
   void runCudaVariant(VariantID vid);
   void runHipVariant(VariantID vid);
   void runOpenMPTargetVariant(VariantID vid);
-
   void runKokkosVariant(VariantID vid);
-
-
-
 
 private:
   Real_ptr m_out1;

diff --git a/src/basic/INIT_VIEW1D.cpp b/src/basic/INIT_VIEW1D.cpp
@@ -54,9 +54,6 @@ INIT_VIEW1D::INIT_VIEW1D(const RunParams& params)
   setVariantDefined( RAJA_HIP );
 
   setVariantDefined( Kokkos_Lambda );
-
-
-
 }
 
 INIT_VIEW1D::~INIT_VIEW1D()

diff --git a/src/basic/INIT_VIEW1D.hpp b/src/basic/INIT_VIEW1D.hpp
@@ -66,10 +66,8 @@ class INIT_VIEW1D : public KernelBase
   void runCudaVariant(VariantID vid);
   void runHipVariant(VariantID vid);
   void runOpenMPTargetVariant(VariantID vid);
-    void runKokkosVariant(VariantID vid);
-
-
-
+  void runKokkosVariant(VariantID vid);
+
 private:
   Real_ptr m_a;
   Real_type m_val;

diff --git a/src/basic/INIT_VIEW1D_OFFSET.cpp b/src/basic/INIT_VIEW1D_OFFSET.cpp
@@ -54,9 +54,6 @@ INIT_VIEW1D_OFFSET::INIT_VIEW1D_OFFSET(const RunParams& params)
   setVariantDefined( RAJA_HIP );
 
   setVariantDefined( Kokkos_Lambda );
-
-
-
 }
 
 INIT_VIEW1D_OFFSET::~INIT_VIEW1D_OFFSET()

diff --git a/src/basic/INIT_VIEW1D_OFFSET.hpp b/src/basic/INIT_VIEW1D_OFFSET.hpp
@@ -65,10 +65,8 @@ class INIT_VIEW1D_OFFSET : public KernelBase
   void runCudaVariant(VariantID vid);
   void runHipVariant(VariantID vid);
   void runOpenMPTargetVariant(VariantID vid);
-    void runKokkosVariant(VariantID vid);
-
-
-
+  void runKokkosVariant(VariantID vid);
+
 private:
   Real_ptr m_a;
   Real_type m_val;

diff --git a/src/basic/MAT_MAT_SHARED.hpp b/src/basic/MAT_MAT_SHARED.hpp
@@ -154,7 +154,6 @@ class MAT_MAT_SHARED : public KernelBase {
   void runCudaVariant(VariantID vid);
   void runHipVariant(VariantID vid);
   void runOpenMPTargetVariant(VariantID vid);
-
   void runKokkosVariant(VariantID vid)
   {
      getCout() << "\n MAT_MAT_SHARED : Unknown Kokkos variant id = " << vid << std::endl;

diff --git a/src/basic/MULADDSUB.cpp b/src/basic/MULADDSUB.cpp
@@ -53,10 +53,6 @@ MULADDSUB::MULADDSUB(const RunParams& params)
   setVariantDefined( RAJA_HIP );
 
   setVariantDefined( Kokkos_Lambda );
-
-
-
-
 }
 
 MULADDSUB::~MULADDSUB()

diff --git a/src/basic/MULADDSUB.hpp b/src/basic/MULADDSUB.hpp
@@ -58,10 +58,8 @@ class MULADDSUB : public KernelBase
   void runCudaVariant(VariantID vid);
   void runHipVariant(VariantID vid);
   void runOpenMPTargetVariant(VariantID vid);
-    void runKokkosVariant(VariantID vid);
-
-
-
+  void runKokkosVariant(VariantID vid);
+
 private:
   Real_ptr m_out1;
   Real_ptr m_out2;

diff --git a/src/basic/NESTED_INIT.cpp b/src/basic/NESTED_INIT.cpp
@@ -64,8 +64,6 @@ NESTED_INIT::NESTED_INIT(const RunParams& params)
   setVariantDefined( RAJA_HIP );
 
   setVariantDefined( Kokkos_Lambda );
-
-
 }
 
 NESTED_INIT::~NESTED_INIT()

diff --git a/src/basic/NESTED_INIT.hpp b/src/basic/NESTED_INIT.hpp
@@ -58,10 +58,8 @@ class NESTED_INIT : public KernelBase
   void runCudaVariant(VariantID vid);
   void runHipVariant(VariantID vid);
   void runOpenMPTargetVariant(VariantID vid);
-    void runKokkosVariant(VariantID vid);
-
-
-
+  void runKokkosVariant(VariantID vid);
+
 private:
   Index_type m_array_length;
 

diff --git a/src/basic/PI_ATOMIC.cpp b/src/basic/PI_ATOMIC.cpp
@@ -57,6 +57,8 @@ PI_ATOMIC::PI_ATOMIC(const RunParams& params)
   setVariantDefined( Base_HIP );
   setVariantDefined( Lambda_HIP );
   setVariantDefined( RAJA_HIP );
+
+  setVariantDefined( Kokkos_Lambda );
 }
 
 PI_ATOMIC::~PI_ATOMIC()

diff --git a/src/basic/PI_ATOMIC.hpp b/src/basic/PI_ATOMIC.hpp
@@ -53,8 +53,6 @@ class PI_ATOMIC : public KernelBase
   void runCudaVariant(VariantID vid);
   void runHipVariant(VariantID vid);
   void runOpenMPTargetVariant(VariantID vid);
-
-
   void runKokkosVariant(VariantID vid);
 
 private:

diff --git a/src/basic/PI_REDUCE.hpp b/src/basic/PI_REDUCE.hpp
@@ -56,7 +56,6 @@ class PI_REDUCE : public KernelBase
   void runCudaVariant(VariantID vid);
   void runHipVariant(VariantID vid);
   void runOpenMPTargetVariant(VariantID vid);
-
   void runKokkosVariant(VariantID vid)
   {
      getCout() << "\n PI_REDUCE : Unknown Kokkos variant id = " << vid << std::endl;

diff --git a/src/basic/REDUCE3_INT.cpp b/src/basic/REDUCE3_INT.cpp
@@ -58,9 +58,6 @@ REDUCE3_INT::REDUCE3_INT(const RunParams& params)
   setVariantDefined( RAJA_HIP );
 
   setVariantDefined( Kokkos_Lambda );
-
-
-
 }
 
 REDUCE3_INT::~REDUCE3_INT()

diff --git a/src/basic/REDUCE3_INT.hpp b/src/basic/REDUCE3_INT.hpp
@@ -70,10 +70,8 @@ class REDUCE3_INT : public KernelBase
   void runCudaVariant(VariantID vid);
   void runHipVariant(VariantID vid);
   void runOpenMPTargetVariant(VariantID vid);
-    void runKokkosVariant(VariantID vid);
-
-
-
+  void runKokkosVariant(VariantID vid);
+
 private:
   Int_ptr m_vec;
   Int_type m_vsum;

diff --git a/src/basic/TRAP_INT.cpp b/src/basic/TRAP_INT.cpp
@@ -53,9 +53,6 @@ TRAP_INT::TRAP_INT(const RunParams& params)
   setVariantDefined( RAJA_HIP );
 
   setVariantDefined( Kokkos_Lambda );
-
-
-
 }
 
 TRAP_INT::~TRAP_INT()

diff --git a/src/basic/TRAP_INT.hpp b/src/basic/TRAP_INT.hpp
@@ -67,10 +67,8 @@ class TRAP_INT : public KernelBase
   void runCudaVariant(VariantID vid);
   void runHipVariant(VariantID vid);
   void runOpenMPTargetVariant(VariantID vid);
-    void runKokkosVariant(VariantID vid);
-
-
-
+  void runKokkosVariant(VariantID vid);
+
 private:
   Real_type m_x0;
   Real_type m_xp;