Add compute-sanitizer testing to CI.

[skip-vdc][skip-docs][skip-rapids]
NVIDIA · Jun 20, 2024 · db7c151 · db7c151
1 parent bd5dd0a
commit db7c151
Show file tree

Hide file tree

Showing 4 changed files with 177 additions and 19 deletions.
diff --git a/ci/matrix.yaml b/ci/matrix.yaml
@@ -8,6 +8,7 @@ workflows:
   #   - {jobs: ['test'], project: 'thrust', std: 17, ctk: 'curr', cxx: ['gcc12', 'llvm16']}
   #
   override:
+    - {jobs: ['compute_sanitizer'], project: 'cub', std: 17, cxx: 'gcc13', cmake_options: '-DCMAKE_CUDA_FLAGS=-lineinfo'}
 
   pull_request:
     # Old CTK
@@ -178,6 +179,11 @@ jobs:
   # General:
   build: { gpu: false }
   test:  { gpu: true, needs: 'build' }
+  compute_sanitizer:
+    gpu: true
+    name: 'ComputeSanitizer'
+    needs: 'build'
+    invoke: { prefix: 'test', args: '-compute-sanitizer' }
 
   # CCCL:
   infra: { gpu: true } # example project launches a kernel
@@ -196,12 +202,29 @@ jobs:
   test_lid1:  { name: 'DeviceLaunch', gpu: true, needs: 'build', invoke: { prefix: 'test', args: '-lid1'} }
   # - captured in a CUDA graph for deferred launch (lid2):
   test_lid2:  { name: 'GraphCapture', gpu: true, needs: 'build', invoke: { prefix: 'test', args: '-lid2'} }
+  # Compute sanitizer jobs:
+  compute_mem_nolid:  { name: 'TestGPU-CS-MemCheck',       gpu: true, needs: 'build', invoke: { prefix: 'test', args: '-compute-sanitizer-memcheck  -no-lid'} }
+  compute_mem_lid0:   { name: 'HostLaunch-CS-MemCheck',    gpu: true, needs: 'build', invoke: { prefix: 'test', args: '-compute-sanitizer-memcheck  -lid0'} }
+  compute_mem_lid1:   { name: 'DeviceLaunch-CS-MemCheck',  gpu: true, needs: 'build', invoke: { prefix: 'test', args: '-compute-sanitizer-memcheck  -lid1'} }
+  compute_mem_lid2:   { name: 'GraphCapture-CS-MemCheck',  gpu: true, needs: 'build', invoke: { prefix: 'test', args: '-compute-sanitizer-memcheck  -lid2'} }
+  compute_race_nolid: { name: 'TestGPU-CS-RaceCheck',      gpu: true, needs: 'build', invoke: { prefix: 'test', args: '-compute-sanitizer-racecheck -no-lid'} }
+  compute_race_lid0:  { name: 'HostLaunch-CS-RaceCheck',   gpu: true, needs: 'build', invoke: { prefix: 'test', args: '-compute-sanitizer-racecheck -lid0'} }
+  compute_race_lid1:  { name: 'DeviceLaunch-CS-RaceCheck', gpu: true, needs: 'build', invoke: { prefix: 'test', args: '-compute-sanitizer-racecheck -lid1'} }
+  compute_race_lid2:  { name: 'GraphCapture-CS-RaceCheck', gpu: true, needs: 'build', invoke: { prefix: 'test', args: '-compute-sanitizer-racecheck -lid2'} }
+  compute_init_nolid: { name: 'TestGPU-CS-InitCheck',      gpu: true, needs: 'build', invoke: { prefix: 'test', args: '-compute-sanitizer-initcheck -no-lid'} }
+  compute_init_lid0:  { name: 'HostLaunch-CS-InitCheck',   gpu: true, needs: 'build', invoke: { prefix: 'test', args: '-compute-sanitizer-initcheck -lid0'} }
+  compute_init_lid1:  { name: 'DeviceLaunch-CS-InitCheck', gpu: true, needs: 'build', invoke: { prefix: 'test', args: '-compute-sanitizer-initcheck -lid1'} }
+  compute_init_lid2:  { name: 'GraphCapture-CS-InitCheck', gpu: true, needs: 'build', invoke: { prefix: 'test', args: '-compute-sanitizer-initcheck -lid2'} }
+  compute_sync_nolid: { name: 'TestGPU-CS-SyncCheck',      gpu: true, needs: 'build', invoke: { prefix: 'test', args: '-compute-sanitizer-synccheck -no-lid'} }
+  compute_sync_lid0:  { name: 'HostLaunch-CS-SyncCheck',   gpu: true, needs: 'build', invoke: { prefix: 'test', args: '-compute-sanitizer-synccheck -lid0'} }
+  # No compute_sync_lid1 job -- compute-sanitizer's synccheck tool doesn't support CDP
+  compute_sync_lid2:  { name: 'GraphCapture-CS-SyncCheck', gpu: true, needs: 'build', invoke: { prefix: 'test', args: '-compute-sanitizer-synccheck -lid2'} }
 
   # Thrust:
   test_cpu: { name: 'TestCPU', gpu: false, needs: 'build', invoke: { prefix: 'test', args: '-cpu-only'} }
   test_gpu: { name: 'TestGPU', gpu: true,  needs: 'build', invoke: { prefix: 'test', args: '-gpu-only'} }
 
-# Project have the following properties:
+# Projects have the following properties:
 #
 # Keys are project subdirectories names. These will also be used in script names.
 #
@@ -221,7 +244,25 @@ projects:
   cub:
     name: 'CUB'
     stds: [11, 14, 17, 20]
-    job_map: { test: ['test_nolid', 'test_lid0', 'test_lid1', 'test_lid2'] }
+    job_map:
+      test: ['test_nolid', 'test_lid0', 'test_lid1', 'test_lid2']
+      compute_sanitizer:
+        # - compute_mem_nolid
+        # - compute_mem_lid0
+        # - compute_mem_lid1
+        # - compute_mem_lid2
+        # - compute_race_nolid
+        # - compute_race_lid0
+        # - compute_race_lid1
+        # - compute_race_lid2
+        # - compute_init_nolid
+        # - compute_init_lid0
+        # - compute_init_lid1
+        # - compute_init_lid2
+        - compute_sync_nolid
+        - compute_sync_lid0
+        # synccheck doesn't support lid1 (CDP launch)
+        - compute_sync_lid2
   thrust:
     name: 'Thrust'
     stds: [11, 14, 17, 20]

diff --git a/ci/test_cub.sh b/ci/test_cub.sh
@@ -6,33 +6,49 @@ NO_LID=false
 LID0=false
 LID1=false
 LID2=false
+COMPUTE_SANITIZER=false
 
 ci_dir=$(dirname "$0")
 
-new_args=$("${ci_dir}/util/extract_switches.sh" -no-lid -lid0 -lid1 -lid2 -- "$@")
+new_args=$("${ci_dir}/util/extract_switches.sh" \
+  -no-lid \
+  -lid0 \
+  -lid1 \
+  -lid2 \
+  -compute-sanitizer-memcheck \
+  -compute-sanitizer-racecheck \
+  -compute-sanitizer-initcheck \
+  -compute-sanitizer-synccheck \
+  -- "$@")
+
 eval set -- ${new_args}
 while true; do
   case "$1" in
-  -no-lid)
-    NO_LID=true
-    shift
-    ;;
-  -lid0)
-    LID0=true
+  -no-lid) NO_LID=true; shift;;
+  -lid0)   LID0=true;   shift;;
+  -lid1)   LID1=true;   shift;;
+  -lid2)   LID2=true;   shift;;
+  -compute-sanitizer-memcheck)
+    COMPUTE_SANITIZER=true
+    TOOL=memcheck
     shift
     ;;
-  -lid1)
-    LID1=true
+  -compute-sanitizer-racecheck)
+    COMPUTE_SANITIZER=true
+    TOOL=racecheck
     shift
     ;;
-  -lid2)
-    LID2=true
+  -compute-sanitizer-initcheck)
+    COMPUTE_SANITIZER=true
+    TOOL=initcheck
     shift
     ;;
-  --)
+  -compute-sanitizer-synccheck)
+    COMPUTE_SANITIZER=true
+    TOOL=synccheck
     shift
-    break
     ;;
+  --) shift; break;;
   *)
     echo "Unknown argument: $1"
     exit 1
@@ -58,6 +74,11 @@ else
   PRESETS=("cub-cpp$CXX_STANDARD")
 fi
 
+if $COMPUTE_SANITIZER; then
+  echo "Setting CCCL_TEST_MODE=compute-sanitizer-${TOOL}"
+  export CCCL_TEST_MODE=compute-sanitizer-${TOOL}
+fi
+
 for PRESET in ${PRESETS[@]}; do
   test_preset "CUB (${PRESET})" ${PRESET}
 done

diff --git a/cub/test/CMakeLists.txt b/cub/test/CMakeLists.txt
@@ -219,7 +219,11 @@ function(cub_add_test target_name_var test_name test_src cub_target launcher_id)
       target_link_libraries(${test_target} PRIVATE ${catch2_main_objects})
       add_dependencies(${config_meta_target} ${test_target})
 
-      add_test(NAME ${test_target} COMMAND "$<TARGET_FILE:${test_target}>")
+      add_test(NAME ${test_target} COMMAND
+        "${CMAKE_COMMAND}"
+        "-DTEST=$<TARGET_FILE:${test_target}>"
+        -P "${CUB_SOURCE_DIR}/test/run_test.cmake"
+      )
     else() # Not CUB_SEPARATE_CATCH2
       # Per config catch2 runner
       set(config_c2run_target ${config_prefix}.catch2_test.lid_${launcher_id})
@@ -244,8 +248,10 @@ function(cub_add_test target_name_var test_name test_src cub_target launcher_id)
           thrust_fix_clang_nvcc_build_for(${config_c2run_target})
         endif()
 
-        add_test(NAME ${config_c2run_target}
-          COMMAND "$<TARGET_FILE:${config_c2run_target}>"
+        add_test(NAME ${config_c2run_target} COMMAND
+          "${CMAKE_COMMAND}"
+          "-DTEST=$<TARGET_FILE:${config_c2run_target}>"
+          -P "${CUB_SOURCE_DIR}/test/run_test.cmake"
         )
       endif() # per config catch2 runner
 
@@ -335,7 +341,11 @@ function(cub_add_test target_name_var test_name test_src cub_target launcher_id)
       endif()
       add_dependencies(${test_meta_target} ${test_target})
 
-      add_test(NAME ${test_target} COMMAND "$<TARGET_FILE:${test_target}>")
+      add_test(NAME ${test_target} COMMAND
+        "${CMAKE_COMMAND}"
+        "-DTEST=$<TARGET_FILE:${test_target}>"
+        -P "${CUB_SOURCE_DIR}/test/run_test.cmake"
+    )
     endif()
   endif() # Not catch2 test
 endfunction()

diff --git a/cub/test/run_test.cmake b/cub/test/run_test.cmake
@@ -0,0 +1,86 @@
+#
+# Launch a test, optionally enabling runtime sanitizers, etc.
+#
+
+function(usage)
+  message("Usage:")
+  message("  cmake -D TEST=bin/test.exe \\")
+  message("        -D ARGS=\"arg1 arg2\" \\")
+  message("        -D MODE=compute-sanitizer-memcheck \\")
+  message("        -P cccl/cub/test/run_test.cmake")
+  message("")
+  message("  - TEST: Required.  Path to the test executable.")
+  message("  - ARGS: Optional.  Arguments to pass to the test executable.")
+  message("  - MODE: Optional.")
+  message("    - May be set through CCCL_TEST_MODE env var.")
+  message("    - Must be one of the following:")
+  message("    -  \"none\" (default)")
+  message("    -  \"compute-sanitizer-memcheck\"")
+  message("    -  \"compute-sanitizer-racecheck\"")
+  message("    -  \"compute-sanitizer-initcheck\"")
+  message("    -  \"compute-sanitizer-synccheck\"")
+endfunction()
+
+# Usage:
+#   run_command(COMMAND [ARGS...])
+#
+# The command is printed before it is executed.
+# The new process's stdout, sterr are redirected to the current process.
+# The current process will exit with an error if the new process exits with a non-zero status.
+function(run_command command)
+  list(APPEND command ${ARGN})
+  list(JOIN command " " command_str)
+  message(STATUS ">> Running:\n\t${command_str}")
+  execute_process(COMMAND ${command} RESULT_VARIABLE result)
+  if (NOT result EQUAL 0)
+    message(FATAL_ERROR ">> Exit Status: ${result}")
+  else()
+    message(STATUS ">>Exit Status: ${result}")
+  endif()
+endfunction()
+
+######################################################################
+
+# Parse arguments
+if(NOT DEFINED TEST)
+  usage()
+  message(FATAL_ERROR "TEST must be defined")
+endif()
+
+if(NOT DEFINED ARGS)
+  set(ARGS "")
+endif()
+
+if(NOT DEFINED MODE)
+  if(DEFINED ENV{CCCL_TEST_MODE})
+    message(STATUS "Using CCCL_TEST_MODE from env: $ENV{CCCL_TEST_MODE}")
+    set(MODE $ENV{CCCL_TEST_MODE})
+  else()
+    set(MODE "none")
+  endif()
+elseif(NOT MODE)
+  set(MODE "none")
+endif()
+
+if (MODE STREQUAL "none")
+  run_command(${TEST} ${ARGS})
+elseif (MODE MATCHES "^compute-sanitizer-(.*)$")
+  set(tool ${CMAKE_MATCH_1})
+  run_command(compute-sanitizer
+    --tool ${tool}
+    # TODO Figure out what the min version needed is for this:
+    # --check-bulk-copy yes
+    --check-device-heap yes
+    --leak-check full
+    --padding 512
+    --track-stream-ordered-races all
+    --check-warpgroup-mma yes
+    --check-exit-code yes
+    --error-exitcode 1
+    --nvtx true
+    ${TEST} ${ARGS}
+  )
+else()
+  usage()
+  message(FATAL_ERROR "Invalid MODE: ${MODE}")
+endif()