Enable tests (#167)

* Add DEVCONTAINER_NAME env variable to devcontainers.

* Remove enabling install rules.

These were needed for the cmake install testing which is disabled.

We will add a separate job for testing the install.

* Factor out common build/configure logic.

* Add compile commands to common cmake options.

* Update sccache_stats to selectively add "notfice" prefix when running on action runner.

* Remove manual sccache stats calculation from workflow.

This is no longer needed because the build scripts do it automatically.

* Remove devcontainer name env variable.

* Revert "Remove devcontainer name env variable."

This reverts commit 2cbc501.

* Use devcontainer specific build directory.

* Get rid of stale Windows runs-on handling.

* Make test job conditional on new input.

* Add simple matrix for test.

* Enable Thrust test script.

* Update thrust test script to use build_common.

* Remove redundant build job.

* Run nvidia-smi step in test job.

* Update CUB test script.

* Update libcu++ test script.

* Enable CUB and libcu++ test jobs.

* Hack to disable verbose CUB test output.

* Exclude device_radix_sort CUB tests for now.

* Update matrix to full matrix with tests on latest host compiler.

* Run tests for all C++ dialects on latest host compiler.

* Limit libcu++ tests parallel execution.

* Add back set +x to build_common

* Disable Thrust and CUB jobs for now.

* Hardcode 8 workers for libcu++ tests.

* Re-enable Thrust/CUB builds and tests.

* Add build/latest.

Make it so builds outside of devcontainer build to build/local.

Also, always make it so the most recent build is symlinked to build/latest.

.github/workflows/build-and-test.yml

@@ -16,15 +16,16 @@ on:
       cpu: {type: string, required: true}
       os: {type: string, required: true}
       build_script: {type: string, required: false}
-      test_script: {type: string, required: false }
+      test_script: {type: string, required: false}
+      run_tests: {type: boolean, required: false, default: true}
       build_image: {type: string, required: false}
       test_image: {type: string, required: false}
 
 jobs:
   build:
     if: inputs.build_script != '' && inputs.build_image != ''
     name: Build ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
-    runs-on: ${{ (inputs.os == 'windows' && 'windows-latest') || format('linux-{0}-cpu16', inputs.cpu) }}
+    runs-on: linux-${{inputs.cpu}}-cpu16
     container: 
       options: -u root
       image: ${{ inputs.build_image }}
@@ -62,7 +63,7 @@ jobs:
             fi
   test:
     needs: build
-    if:  ${{ !cancelled() && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != '' && inputs.test_image != '' }}
+    if:  ${{ !cancelled() && ( needs.build.result == 'success' || needs.build.result == 'skipped' ) && inputs.test_script != '' && inputs.test_image != '' && inputs.run_tests}}
     name: Test ${{inputs.compiler}}${{inputs.compiler_version}}/C++${{inputs.std}}
     runs-on: linux-${{inputs.cpu}}-gpu-v100-latest-1
     container: 
@@ -73,6 +74,8 @@ jobs:
     permissions:
       id-token: write
     steps:
+      - name: nvidia-smi
+        run: nvidia-smi
       - name: Checkout repo
         uses: actions/checkout@v3
         with:

.github/workflows/dispatch-build-and-test.yml

@@ -30,5 +30,6 @@ jobs:
       build_script: ${{ inputs.build_script }}
       build_image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-${{matrix.os}}
       test_script: ${{ inputs.test_script }}
+      run_tests: ${{ contains(matrix.jobs, 'test') }}
       test_image: rapidsai/devcontainers:23.06-cpp-${{matrix.compiler.name}}${{matrix.compiler.version}}-cuda${{matrix.cuda}}-${{matrix.os}}
 

.github/workflows/pr.yml

@@ -36,7 +36,7 @@ jobs:
     with:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
       build_script: "./ci/build_thrust.sh"
-      test_script: #"./ci/test_thrust.sh"
+      test_script: "./ci/test_thrust.sh"
 
   cub:
     name: CUB CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
@@ -50,7 +50,7 @@ jobs:
     with:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
       build_script: "./ci/build_cub.sh"
-      test_script: #"./ci/test_cub.sh"
+      test_script: "./ci/test_cub.sh"
 
   libcudacxx:
     name: libcudacxx CUDA${{ matrix.cuda_version }} ${{ matrix.compiler }}
@@ -64,7 +64,7 @@ jobs:
     with:
       per_cuda_compiler_matrix: ${{ toJSON(fromJSON(needs.compute-matrix.outputs.PER_CUDA_COMPILER_MATRIX)[ format('{0}-{1}', matrix.cuda_version, matrix.compiler) ]) }}
       build_script: "./ci/build_libcudacxx.sh"
-      test_script: #"./ci/test_libcudacxx.sh" 
+      test_script: "./ci/test_libcudacxx.sh" 
 
   # This job is the final job that runs after all other jobs and is used for branch protection status checks.
   # See: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/about-status-checks

ci/matrix.yaml

@@ -22,21 +22,23 @@ gpus:
 
 # Configurations that will run for every PR
 pull-request:
-- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '6', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14]}
-- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
-- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
-- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
-- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
-- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
-- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
-- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
-- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20]}
+# gcc
+- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '6', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14], jobs: ['build']}
+- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '7', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '8', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '9', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'gcc', version: '10', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
+- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '11', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
+- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'gcc', version: '12', exe: 'g++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build', 'test']}
+# clang
+- {cuda: '11.1', os: 'ubuntu18.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '9', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '10', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17], jobs: ['build']}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '11', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '12', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '13', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
+- {cuda: '12.1', os: 'ubuntu20.04', cpu: 'amd64', compiler: {name: 'llvm', version: '14', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build']}
+- {cuda: '12.1', os: 'ubuntu22.04', cpu: 'amd64', compiler: {name: 'llvm', version: '15', exe: 'clang++'}, gpu_build_archs: '70', std: [11, 14, 17, 20], jobs: ['build', 'test']}

ci/test_cub.sh

@@ -1,12 +1,9 @@
 #!/bin/bash
 
-set -xeuo pipefail
-
-# Ensure the script is being executed in its containing directory
-cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
+source "$(dirname "$0")/build_common.sh"
 
 ./build_cub.sh "$@"
 
-ctest --test-dir ../build --output-on-failure --timeout 15
+ctest --test-dir ${BUILD_DIR} --output-on-failure -E device_radix_sort
 
 echo "CUB test complete"

ci/test_libcudacxx.sh

@@ -2,20 +2,17 @@
 
 source "$(dirname "$0")/build_common.sh"
 
-cmake -S .. -B ../build \
+CMAKE_OPTIONS="
     -DCCCL_ENABLE_THRUST=OFF \
     -DCCCL_ENABLE_LIBCUDACXX=ON \
     -DCCCL_ENABLE_CUB=OFF \
     -DCCCL_ENABLE_TESTING=OFF \
-    -DCMAKE_BUILD_TYPE=Release \
-    -DCMAKE_CXX_COMPILER=${HOST_COMPILER} \
-    -DCMAKE_CUDA_ARCHITECTURES=${GPU_ARCHS} \
-    -DCMAKE_CUDA_HOST_COMPILER=${HOST_COMPILER} \
     -DLIBCUDACXX_ENABLE_LIBCUDACXX_TESTS=ON \
-    -Dlibcudacxx_ENABLE_INSTALL_RULES=ON \
-    -DCUB_ENABLE_INSTALL_RULES=ON \
-    -DTHRUST_ENABLE_INSTALL_RULES=ON \
-    -G Ninja
+"
+configure "$CMAKE_OPTIONS"
 
-LIBCUDACXX_SITE_CONFIG="../build/libcudacxx/test/lit.site.cfg" lit -v --no-progress-bar -Dcompute_archs=${GPU_ARCHS} -Dstd="c++${CXX_STANDARD}" ../libcudacxx/.upstream-tests/test
+readonly TEST_PARALLEL_LEVEL=8
 
+source "./sccache_stats.sh" "start"
+LIBCUDACXX_SITE_CONFIG="${BUILD_DIR}/libcudacxx/test/lit.site.cfg" lit -v -j ${TEST_PARALLEL_LEVEL} --no-progress-bar -Dcompute_archs=${GPU_ARCHS} -Dstd="c++${CXX_STANDARD}" ../libcudacxx/.upstream-tests/test
+source "./sccache_stats.sh" "end"

ci/test_thrust.sh

@@ -1,13 +1,10 @@
 #!/bin/bash
 
-set -xeuo pipefail
-
-# Ensure the script is being executed in its containing directory
-cd "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )";
+source "$(dirname "$0")/build_common.sh"
 
 ./build_thrust.sh "$@"
 
-ctest --test-dir ../build --output-on-failure --timeout 15
+ctest --test-dir ${BUILD_DIR} --output-on-failure 
 
 echo "Thrust test complete"
 

cub/test/CMakeLists.txt

@@ -230,7 +230,7 @@ function(cub_add_test target_name_var test_name test_src cub_target)
     target_link_libraries(${test_target} ${cub_target})
     cub_clone_target_properties(${test_target} ${cub_target})
     target_include_directories(${test_target} PRIVATE "${CUB_SOURCE_DIR}/test")
-    target_compile_definitions(${test_target} PRIVATE CUB_DEBUG_HOST_ASSERTIONS)
+    target_compile_definitions(${test_target} PRIVATE CUB_DETAIL_DEBUG_ENABLE_SYNC)
 
     if (CUB_IN_THRUST)
       thrust_fix_clang_nvcc_build_for(${test_target})