Use caliper

.gitignore

@@ -1,5 +1,7 @@
 *.pyc
 *.o
 *.exe
+*.swp
+*.swo
 build*/
 install*/

.gitmodules

@@ -1,6 +1,7 @@
 [submodule "blt"]
 	path = blt
 	url = https://github.com/LLNL/blt.git
+	ignore = dirty
 [submodule "tpl/RAJA"]
 	path = tpl/RAJA
 	url = https://github.com/LLNL/RAJA.git

CMakeLists.txt

@@ -120,6 +120,27 @@ if ((ENABLE_HIP) AND (NOT ENABLE_KOKKOS))
   list(APPEND RAJA_PERFSUITE_DEPENDS blt::hip_runtime)
 endif()
 
+#
+# Are we using Caliper
+#
+set(RAJA_PERFSUITE_USE_CALIPER off CACHE BOOL "")
+if (RAJA_PERFSUITE_USE_CALIPER)
+   find_package(caliper REQUIRED)
+   list(APPEND RAJA_PERFSUITE_DEPENDS caliper)
+   add_definitions(-DRAJA_PERFSUITE_USE_CALIPER)
+   message(STATUS "Using Caliper")
+   find_package(adiak REQUIRED)
+   # use ${adiak_LIBRARIES} since version could have adiak vs adiak::adiak export
+   list(APPEND RAJA_PERFSUITE_DEPENDS ${adiak_LIBRARIES})
+   message(STATUS "Using Adiak INCLUDES: ${adiak_INCLUDE_DIRS}")
+   include_directories(${adiak_INCLUDE_DIRS})
+   if (ENABLE_CUDA)
+      # Adiak will propagate -pthread from spectrum mpi from a spack install of Caliper with +mpi; and needs to be handled even if RAJAPerf is non MPI program
+      # We should delegate to BLT to handle unguarded -pthread from any dependencies, but currently BLT doesn't
+      set_target_properties(${adiak_LIBRARIES} PROPERTIES INTERFACE_COMPILE_OPTIONS "$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:-pthread>;$<$<COMPILE_LANGUAGE:CUDA>:-Xcompiler=-pthread>")
+   endif ()
+endif ()
+
 set(RAJAPERF_BUILD_SYSTYPE $ENV{SYS_TYPE})
 set(RAJAPERF_BUILD_HOST $ENV{HOSTNAME})
 
@@ -147,7 +168,7 @@ else()
 endif()
 
 configure_file(${CMAKE_SOURCE_DIR}/src/rajaperf_config.hpp.in
-  ${CMAKE_CURRENT_BINARY_DIR}/include/rajaperf_config.hpp)
+               ${CMAKE_CURRENT_BINARY_DIR}/include/rajaperf_config.hpp)
 
 include_directories($<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/include>)
 

scripts/lc-builds/blueos_nvcc10_caliper25_gcc8.3.1.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+###############################################################################
+# Copyright (c) 2017-20, Lawrence Livermore National Security, LLC
+# and RAJA Performance Suite project contributors.
+# See the RAJAPerf/COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (BSD-3-Clause)
+#################################################################################
+
+BUILD_SUFFIX=lc_blueos-nvcc10-caliper-gcc8.3.1
+RAJA_HOSTCONFIG=../tpl/RAJA/host-configs/lc-builds/blueos/nvcc_gcc_X.cmake
+
+rm -rf build_${BUILD_SUFFIX} >/dev/null
+mkdir build_${BUILD_SUFFIX} && cd build_${BUILD_SUFFIX}
+
+module load cmake/3.14.5
+module load caliper-2.5.0-gcc-8.3.1-cu3vy3k
+
+CALIPER_PREFIX=/usr/WS2/holger/spack/opt/spack/linux-rhel7-power9le/gcc-8.3.1/caliper-2.5.0-cu3vy3kjwjerpdm6xis2kauhz4s6wto2/
+
+ADIAK_PREFIX=/usr/WS2/holger/spack/opt/spack/linux-rhel7-power9le/gcc-8.3.1/adiak-0.2.1-hsv444o7ofb6s2znkvvnh6hcmr774g73/
+
+cmake \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DCMAKE_CXX_COMPILER=/usr/tce/packages/gcc/gcc-8.3.1/bin/g++ \
+  -C ${RAJA_HOSTCONFIG} \
+  -DCMAKE_PREFIX_PATH="${CALIPER_PREFIX}/share/cmake/caliper;${ADIAK_PREFIX}/lib/cmake/adiak" \
+  -DENABLE_OPENMP=On \
+  -DENABLE_CUDA=On \
+  -DCMAKE_CUDA_FLAGS="-Xcompiler -mno-float128" \
+  -DCUDA_TOOLKIT_ROOT_DIR=/usr/tce/packages/cuda/cuda-10.2.89 \
+  -DCMAKE_CUDA_COMPILER=/usr/tce/packages/cuda/cuda-10.1.243/bin/nvcc \
+  -DCUDA_ARCH=sm_70 \
+  -DENABLE_CALIPER=On \
+  -DCMAKE_INSTALL_PREFIX=../install_${BUILD_SUFFIX} \
+  -DCMAKE_EXPORT_COMPILE_COMMANDS=On \
+  -DCMAKE_VERBOSE_MAKEFILE=On \
+  "$@" \
+  ..

scripts/spack_packages/raja_perf/package.py

@@ -76,10 +76,14 @@ class RajaPerf(CMakePackage, CudaPackage, ROCmPackage):
     variant('libcpp', default=False, description='Uses libc++ instead of libstdc++')
     variant('tests', default='basic', values=('none', 'basic', 'benchmarks'),
             multi=False, description='Tests to run')
+    variant('caliper',default=False, description='Build with support for Caliper based profiling')
 
     depends_on('cmake@3.9:', type='build')
     depends_on('blt@0.4.1', type='build', when='@main')
     depends_on('blt@0.4.1:', type='build')
+    depends_on('caliper@2.8.0',when='+caliper')
+    depends_on('caliper@2.8.0 +cuda',when='+caliper +cuda')
+    depends_on('caliper@2.8.0 +rocm',when='+caliper +rocm')
 
     conflicts('+openmp', when='+rocm')
     conflicts('~openmp', when='+openmp_target', msg='OpenMP target requires OpenMP')
@@ -343,6 +347,7 @@ def hostconfig(self, spec, prefix, py_site_pkgs_dir=None):
 
         cfg.write(cmake_cache_option("ENABLE_BENCHMARKS", 'tests=benchmarks' in spec))
         cfg.write(cmake_cache_option("ENABLE_TESTS", not 'tests=none' in spec or self.run_tests))
+        cfg.write(cmake_cache_option("RAJA_PERFSUITE_USE_CALIPER","+caliper" in spec))
 
         #######################
         # Close and save

src/CMakeLists.txt

@@ -8,6 +8,13 @@
 
 include_directories(.)
 
+if(RAJAPERF_USE_CALIPER)
+   message(STATUS "Caliper includes : ${caliper_INCLUDE_DIR}")
+   message(STATUS "Adiak includes : ${adiak_INCLUDE_DIRS}")
+   include_directories(${caliper_INCLUDE_DIR})
+   include_directories(${adiak_INCLUDE_DIRS})
+endif ()
+
 add_subdirectory(common)
 add_subdirectory(basic)
 add_subdirectory(basic-kokkos)
@@ -240,6 +247,10 @@ else()
 blt_add_executable(
   NAME raja-perf.exe
   SOURCES RAJAPerfSuiteDriver.cpp
+  INCLUDES ${PROJECT_BINARY_DIR}/include
   DEPENDS_ON ${RAJA_PERFSUITE_EXECUTABLE_DEPENDS}
   )
 endif()
+
+#blt_print_target_properties(TARGET raja-perf.exe)
+

src/common/CMakeLists.txt

@@ -14,5 +14,6 @@ blt_add_library(
           OutputUtils.cpp 
           RAJAPerfSuite.cpp 
           RunParams.cpp
+  INCLUDES ${PROJECT_BINARY_DIR}/include/
   DEPENDS_ON ${RAJA_PERFSUITE_DEPENDS}
   )

src/common/Executor.cpp

@@ -26,6 +26,7 @@
 #include <list>
 #include <vector>
 #include <string>
+#include <regex>
 #include <unordered_map>
 
 #include <iostream>
@@ -37,11 +38,20 @@
 
 #include <unistd.h>
 
-
 namespace rajaperf {
 
 using namespace std;
 
+#ifdef RAJA_PERFSUITE_USE_CALIPER
+vector<string> split(const string str, const string regex_str)
+{
+    regex regexz(regex_str);
+    vector<string> list(sregex_token_iterator(str.begin(), str.end(), regexz, -1),
+                                  sregex_token_iterator());
+    return list;
+}
+#endif
+
 namespace {
 
 #ifdef RAJA_PERFSUITE_ENABLE_MPI
@@ -98,13 +108,96 @@ void Allreduce(const Checksum_type* send, Checksum_type* recv, int count,
 
 #endif
 
-}
+} // close unnamed namespace 
+
 
 Executor::Executor(int argc, char** argv)
   : run_params(argc, argv),
     reference_vid(NumVariants),
     reference_tune_idx(KernelBase::getUnknownTuningIdx())
 {
+#ifdef RAJA_PERFSUITE_USE_CALIPER
+  struct configuration cc;
+  adiak::init(NULL);
+  adiak::user();
+  adiak::launchdate();
+  adiak::libraries();
+  adiak::cmdline();
+  adiak::clustername();
+  adiak::value("perfsuite_version", cc.adiak_perfsuite_version);
+  adiak::value("raja_version", cc.adiak_raja_version);
+  adiak::value("cmake_build_type", cc.adiak_cmake_build_type);
+  adiak::value("cmake_cxx_flags", cc.adiak_cmake_cxx_flags);
+  adiak::value("cmake_exe_linker_flags", cc.adiak_cmake_exe_linker_flags);
+  adiak::value("rajaperf_compiler", cc.adiak_rajaperf_compiler);
+  adiak::value("rajaperf_compiler_options", cc.adiak_rajaperf_compiler_options);
+  adiak::value("compiler_version", cc.adiak_compiler_version);
+
+  auto tokens = split(cc.adiak_rajaperf_compiler, "/");
+  string compiler_exec = tokens.back();
+  string compiler = compiler_exec + "-" + cc.adiak_compiler_version;
+  cout << "Compiler: " << compiler << "\n";
+  adiak::value("compiler", compiler.c_str());
+  auto tsize = tokens.size();
+  if (tsize >= 3) {
+    // pickup path version <compiler-version-hash|date>/bin/exec
+    string path_version = tokens[tsize-3];
+    //cout << "Compiler path version: " << path_version << "\n";
+    auto s = split(path_version,"-");
+    if (s.size() >= 2) {
+      string path_version_short = s[0] + "-" + s[1];
+      //cout << "Compiler path version short: " << path_version_short << "\n";
+      adiak::value("Compiler_path_version",path_version_short.c_str());
+    } 
+  }
+
+  if (strlen(cc.adiak_cuda_compiler_version) > 0) {
+    adiak::value("cuda_compiler_version", cc.adiak_cuda_compiler_version);
+  }
+  if (strlen(cc.adiak_gpu_targets) > 0) {
+    adiak::value("gpu_targets", cc.adiak_gpu_targets);
+  }
+  if (strlen(cc.adiak_cmake_hip_architectures) > 0) {
+    adiak::value("cmake_hip_architectures", cc.adiak_cmake_hip_architectures);
+  }
+  if (strlen(cc.adiak_gpu_targets_block_sizes) > 0) {
+    adiak::value("gpu_targets_block_sizes", cc.adiak_gpu_targets_block_sizes);
+  }
+  if (strlen(cc.adiak_raja_hipcc_flags) > 0) {
+    adiak::value("raja_hipcc_flags", cc.adiak_raja_hipcc_flags);
+  }
+  if (strlen(cc.adiak_mpi_cxx_compiler) > 0) {
+    adiak::value("mpi_cxx_compiler", cc.adiak_mpi_cxx_compiler);
+  }
+  if (strlen(cc.adiak_systype_build) > 0) {
+    adiak::value("systype_build", cc.adiak_systype_build);
+  }
+  if (strlen(cc.adiak_machine_build) > 0) {
+    adiak::value("machine_build", cc.adiak_machine_build);
+  }
+
+  adiak::value("Tuning","default");
+  adiak::value("ProblemSize",1.0);
+  adiak::value("SizeMeaning",run_params.SizeMeaningToStr(run_params.getSizeMeaning()).c_str());
+  if (run_params.getSizeMeaning() == RunParams::SizeMeaning::Factor) {
+    adiak::value("ProblemSize",run_params.getSizeFactor());
+  } else if (run_params.getSizeMeaning() == RunParams::SizeMeaning::Direct) {
+    adiak::value("ProblemSize",run_params.getSize());
+  }
+
+  // Openmp section
+  // todo get lib : e.g libomp,libiomp5,libgomp etc  : parse adiak::libraries via tool callback
+  // note version map only goes to 5.1; revise as needed
+#if defined(_OPENMP)
+  std::unordered_map<unsigned,std::string> map{
+    {200505,"2.5"},{200805,"3.0"},{201107,"3.1"},{201307,"4.0"},{201511,"4.5"},{201811,"5.0"},{202011,"5.1"}};
+  string strval = map.at(_OPENMP);
+  adiak::value("omp_version",strval.c_str());
+  strval = std::to_string(omp_get_max_threads());
+  adiak::value("omp_max_threads",strval.c_str());
+#endif
+
+#endif
 }
 
 
@@ -113,6 +206,9 @@ Executor::~Executor()
   for (size_t ik = 0; ik < kernels.size(); ++ik) {
     delete kernels[ik];
   }
+#ifdef RAJA_PERFSUITE_USE_CALIPER 
+  adiak::fini();
+#endif
 }
 
 
@@ -611,6 +707,9 @@ void Executor::setupSuite()
       for (VIDset::iterator vid = run_var.begin();
            vid != run_var.end(); ++vid) {
         variant_ids.push_back( *vid );
+#ifdef RAJA_PERFSUITE_USE_CALIPER
+        KernelBase::setCaliperMgrVariant(*vid);
+#endif
       }
 
       //
@@ -861,12 +960,21 @@ void Executor::runSuite()
 
   for (size_t ik = 0; ik < warmup_kernels.size(); ++ik) {
     KernelBase* warmup_kernel = warmup_kernels[ik];
+#ifdef RAJA_PERFSUITE_USE_CALIPER
+    warmup_kernel->caliperOff();
+#endif
     runKernel(warmup_kernel, true);
+#ifdef RAJA_PERFSUITE_USE_CALIPER
+    warmup_kernel->caliperOn();
+#endif
     delete warmup_kernel;
     warmup_kernels[ik] = nullptr;
   }
 
 
+//
+// Now, run kernels for real....
+//
   getCout() << "\n\nRunning specified kernels and variants...\n";
 
   const int npasses = run_params.getNumPasses();
@@ -882,6 +990,17 @@ void Executor::runSuite()
 
   } // loop over passes through suite
 
+#ifdef RAJA_PERFSUITE_USE_CALIPER
+  // setup per kernel Adiak meta data
+  for (size_t ik = 0; ik < kernels.size(); ++ik) {
+      KernelBase* kernel = kernels[ik];
+      kernel->setKernelAdiakMeta();
+  } // loop over kernels
+
+  // Flush Caliper data
+  KernelBase::setCaliperMgrFlush();
+#endif
+
 }
 
 template < typename Kernel >

src/common/Executor.hpp

@@ -12,6 +12,10 @@
 #include "common/RAJAPerfSuite.hpp"
 #include "common/RunParams.hpp"
 
+#ifdef RAJA_PERFSUITE_USE_CALIPER
+#include "rajaperf_config.hpp"
+#endif
+
 #include <iosfwd>
 #include <streambuf>
 #include <memory>