diff --git a/projects/hipcub/.clang-format b/projects/hipcub/.clang-format index 21f7c2598084..ddb455408800 100644 --- a/projects/hipcub/.clang-format +++ b/projects/hipcub/.clang-format @@ -167,4 +167,10 @@ Macros: - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS=[[DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS___]] BreakAfterAttributes: Always +WhitespaceSensitiveMacros: [ + 'HIPCUB_HAS_INCLUDE', + '_HIPCUB_LIBCXX_INCLUDE', + '_HIPCUB_STD_INCLUDE' +] + --- diff --git a/projects/hipcub/.gitlab/run_benchmarks.py b/projects/hipcub/.gitlab/run_benchmarks.py new file mode 100644 index 000000000000..f07b7e11f406 --- /dev/null +++ b/projects/hipcub/.gitlab/run_benchmarks.py @@ -0,0 +1,369 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2022-2026 Advanced Micro Devices, Inc. All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. + +import argparse +import json +import os +import re +import stat +import subprocess +import sys + + +def run_benchmarks( + benchmark_executables_dir, + gpu_architecture, + json_out_dir, + csv_out_dir, + benchmark_executable_filter, + skip_gathered, + size, + hot, + seed, + filter, + dry, + min_gpu_ms_per_batch, + min_secs, + noise_timeout_secs, + batch_window_size, + noise_tolerance_percent, + min_gpu_temp, + max_gpu_temp, + max_warming_secs, + max_cooling_secs, + output_hip_device_properties_context, + output_amdsmi_context, + output_batches, + spaces_per_indent, + stream_blocking_timeout_secs, +): + def is_benchmark_executable(filename): + if not re.search(benchmark_executable_filter, filename): + return False + + path = os.path.join(benchmark_executables_dir, filename) + st_mode = os.stat(path).st_mode + + # Return True when any execution flag is set, AND it is a regular file. + # Doesn't check permissions. + return (st_mode & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)) and ( + st_mode & stat.S_IFREG + ) + + def should_skip(json_out_path): + if not skip_gathered: + return False + + try: + with open(json_out_path) as f: + json.load(f) + except (FileNotFoundError, json.JSONDecodeError): + return False + + return True + + # Create output directories if they don't exist + os.makedirs(json_out_dir, exist_ok=True) + if csv_out_dir: + os.makedirs(csv_out_dir, exist_ok=True) + + benchmark_names = [ + name + for name in os.listdir(benchmark_executables_dir) + if is_benchmark_executable(name) + ] + print( + "The following benchmarks will be run:\n{}".format("\n".join(benchmark_names)), + file=sys.stderr, + flush=True, + ) + + success = True + + for benchmark_name in benchmark_names: + results_json_name = f"{benchmark_name}_{gpu_architecture}.json" + json_out_path = os.path.join(json_out_dir, results_json_name) + + if should_skip(json_out_path): + print( + f"Skipping {benchmark_name}, because its results have already been gathered at {json_out_path}", + file=sys.stderr, + flush=True, + ) + continue + + benchmark_path = os.path.join(benchmark_executables_dir, benchmark_name) + + args = [benchmark_path] + args += ["--json-out", json_out_path] + + if size: + args += ["--size", size] + if hot: + args += ["--hot"] + if seed: + args += ["--seed", seed] + if csv_out_dir: + results_csv_name = f"{benchmark_name}_{gpu_architecture}.csv" + csv_out_path = os.path.join(csv_out_dir, results_csv_name) + args += ["--csv-out", csv_out_path] + if filter: + args += ["--filter", filter] + if dry: + args += ["--dry"] + if min_gpu_ms_per_batch: + args += ["--min-gpu-ms-per-batch", min_gpu_ms_per_batch] + if min_secs: + args += ["--min-secs", min_secs] + if noise_timeout_secs: + args += ["--noise-timeout-secs", noise_timeout_secs] + if batch_window_size: + args += ["--batch-window-size", batch_window_size] + if noise_tolerance_percent: + args += ["--noise-tolerance-percent", noise_tolerance_percent] + if min_gpu_temp: + args += ["--min-gpu-temp", min_gpu_temp] + if max_gpu_temp: + args += ["--max-gpu-temp", max_gpu_temp] + if max_warming_secs: + args += ["--max-warming-secs", max_warming_secs] + if max_cooling_secs: + args += ["--max-cooling-secs", max_cooling_secs] + if output_hip_device_properties_context: + args += ["--output-hip-device-properties-context"] + if output_amdsmi_context: + args += ["--output-amdsmi-context"] + if output_batches: + args += ["--output-batches"] + if spaces_per_indent: + args += ["--spaces-per-indent", spaces_per_indent] + if stream_blocking_timeout_secs: + args += ["--stream-blocking-timeout-secs", stream_blocking_timeout_secs] + + try: + subprocess.check_call(args) + except subprocess.CalledProcessError as error: + print( + f'Could not run benchmark at {benchmark_path}. Error: "{error}"', + file=sys.stderr, + flush=True, + ) + success = False + + return success + + +def main(): + parser = argparse.ArgumentParser() + + # Python arguments + parser.add_argument( + "--benchmark-executables-dir", + help="Benchmark executables directory", + required=True, + ) + parser.add_argument( + "--gpu-architecture", + help="Architecture of the currently enabled GPU", + required=True, + ) + parser.add_argument( + "--json-out-dir", + help="Directory to write JSON result files to, creating parent directories if necessary", + required=True, + ) + parser.add_argument( + "--csv-out-dir", + help="Directory to write CSV result files to, creating parent directories if necessary", + required=False, + ) + parser.add_argument( + "--benchmark-executable-filter", + help="Regular expression that controls the list of benchmark executables to run", + default=r"^benchmark", + required=False, + ) + parser.add_argument( + "--skip-gathered", + help="Skip running benchmarks whose JSON file has already been gathered", + default=False, + action="store_true", + required=False, + ) + + # primbench arguments + parser.add_argument( + "--size", + help="Input size. Benchmarks decide what this represents, but it is commonly the number of bytes or items", + default="", + required=False, + ) + parser.add_argument( + "--hot", + help="Skip clearing the GPU cache between batch iterations", + default=False, + action="store_true", + required=False, + ) + parser.add_argument( + "--seed", help="Seed used for input generation", default="", required=False + ) + parser.add_argument( + "--filter", + help="Regex filter of specialization names to benchmark", + default="", + required=False, + ) + parser.add_argument( + "--dry", + help="Perform a dry run. The benchmark setup is still run, and JSON and CSV files are still output, but state.run() immediately returns", + default=False, + action="store_true", + required=False, + ) + parser.add_argument( + "--min-gpu-ms-per-batch", + help="Minimum duration of a batch in milliseconds (GPU time)", + default="", + required=False, + ) + parser.add_argument( + "--min-secs", + help="Minimum total benchmark duration in seconds (wall time)", + default="", + required=False, + ) + parser.add_argument( + "--noise-timeout-secs", + help="Maximum total benchmark duration in seconds before timing out a noisy run (wall time)", + default="", + required=False, + ) + parser.add_argument( + "--batch-window-size", + help="Number of batch times used in the noise (coefficient of variation) window to decide early stopping", + default="", + required=False, + ) + parser.add_argument( + "--noise-tolerance-percent", + help="Noise tolerance of batch times in percent, used to determine early benchmark stopping", + default="", + required=False, + ) + parser.add_argument( + "--min-gpu-temp", + help="Minimum GPU temperature in °C. Too low slows benchmarks; too high increases noise", + default="", + required=False, + ) + parser.add_argument( + "--max-gpu-temp", + help="Maximum GPU temperature in °C. Too low slows benchmarks; too high increases noise", + default="", + required=False, + ) + parser.add_argument( + "--max-warming-secs", + help="Maximum seconds allowed for GPU warming before an error is thrown", + default="", + required=False, + ) + parser.add_argument( + "--max-cooling-secs", + help="Maximum seconds allowed for GPU cooling before an error is thrown", + default="", + required=False, + ) + parser.add_argument( + "--output-hip-device-properties-context", + help="Output a `hip_device_properties` object in the context, containing GPU details", + default=False, + action="store_true", + required=False, + ) + parser.add_argument( + "--output-amdsmi-context", + help="Output an `amdsmi` object in the context, containing GPU details", + default=False, + action="store_true", + required=False, + ) + parser.add_argument( + "--output-batches", + help="Output a `batches` array for each specialization, containing per-batch details", + default=False, + action="store_true", + required=False, + ) + parser.add_argument( + "--spaces-per-indent", + help="Number of spaces per indentation level in JSON output. Set to 0 for no indentation", + default="", + required=False, + ) + parser.add_argument( + "--stream-blocking-timeout-secs", + help="Maximum stream blocking duration in seconds before timing out. Stream is blocked while queueing kernel calls. Use `primbench::flags::sync` if kernel is synchronous", + default="", + required=False, + ) + + args = parser.parse_args() + + benchmark_run_successful = run_benchmarks( + args.benchmark_executables_dir, + args.gpu_architecture, + args.json_out_dir, + args.csv_out_dir, + args.benchmark_executable_filter, + args.skip_gathered, + args.size, + args.hot, + args.seed, + args.filter, + args.dry, + args.min_gpu_ms_per_batch, + args.min_secs, + args.noise_timeout_secs, + args.batch_window_size, + args.noise_tolerance_percent, + args.min_gpu_temp, + args.max_gpu_temp, + args.max_warming_secs, + args.max_cooling_secs, + args.output_hip_device_properties_context, + args.output_amdsmi_context, + args.output_batches, + args.spaces_per_indent, + args.stream_blocking_timeout_secs, + ) + + return benchmark_run_successful + + +if __name__ == "__main__": + success = main() + if success: + exit(0) + else: + exit(1) diff --git a/projects/hipcub/CHANGELOG.md b/projects/hipcub/CHANGELOG.md index 30b45edaf426..782a8a47422a 100644 --- a/projects/hipcub/CHANGELOG.md +++ b/projects/hipcub/CHANGELOG.md @@ -2,6 +2,35 @@ Full documentation for hipCUB is available at [https://rocm.docs.amd.com/projects/hipCUB/en/latest/](https://rocm.docs.amd.com/projects/hipCUB/en/latest/). +## hipCUB-5.0.0 for ROCm 10.0 + +### Added + +* Added `::hip::std` support. + +### Changed + +* Changed `CCCL_MINIMUM_VERSION` to `3.0.0` to align with CUB. +* Add support for large num_items `DeviceMerge` and `DeviceSegmentedSort`. +* Replace `#pragma unroll` by `_CCCL_PRAGMA_UNROLL_FULL()` and `_CCCL_PRAGMA_NOUNROLL()` by `_CCCL_PRAGMA_NOUNROLL()`. +* Add `_CCCL_SORT_MAYBE_UNROLL()` in block merge sort and thread sort. +* Update `WarpExchange` template parameters for CUB compatibility. +* Benchmarking now uses primbench for its benchmarks instead of Google Benchmark. + * See `shared/primbench/README.md` for its documentation. + +### Removed + +* Removed `hipcub::BaseTraits::CATEGORY`, `hipcub::BaseTraits::nullptr_TYPE` and `hipcub::BaseTraits::PRIMITIVE`. +* Removed `ConstantInputIterator`, `CountingInputIterator`, `DiscardOutputIterator` and `TransformInputIterator` which were deprecated in hipCUB-4.1.0. +* Removed `DeviceSpmv`, which was removed from CUB after CCCL's 2.8.0 release. Use `hipSPARSE` or `rocSPARSE` libraries instead. +* Removed `GridBarrier`. +* Removed `HIPCUB_MIN`, `HIPCUB_MAX`, `HIPCUB_QUOTIENT_FLOOR`, `HIPCUB_QUOTIENT_CEILING`, `HIPCUB_ROUND_UP_NEAREST` and `HIPCUB_ROUND_DOWN_NEAREST` which were deprecated in hipCUB-4.1.0. +* Removed `LEGACY_PTX_ARCH`. +* Removed `hipcub:max` and `hipcub:min`, which were deprecated. Use `hip::std::max` and `hip::std::min` instead. +* Deprecated `hipcub::Swap`, use `rocprim::swap` instead. +* Deprecated `HIPCUB_IS_INT128_ENABLED`, use `_CCCL_HAS_INT128()` instead. +* Deprecated `hipcub::Equality`, `hipcub::Inequality`, `hipcub::InequalityWrapper`, `hipcub::Sum`, `hipcub::Difference`, `hipcub::Division`, `hipcub::Max` and `hipcub::Min` operators. Use `hip::std::equal_to`, `hip::std::not_equal_to`, `hip::std::plus`, `hip::std::minus`, `hip::std::divides`, `hip::maximum` and `hip:minimum` operators instead. + ## hipCUB 4.5.0 for ROCm 7.14 ### Added diff --git a/projects/hipcub/CMakeLists.txt b/projects/hipcub/CMakeLists.txt index b152b21e8cd3..424aa547c8b8 100644 --- a/projects/hipcub/CMakeLists.txt +++ b/projects/hipcub/CMakeLists.txt @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -35,11 +35,11 @@ endif() # Update these variables at release time # # Set the library version -set(VERSION_STRING "4.5.0") +set(VERSION_STRING "5.0.0") # Set the CCCL-compatible version. -set(HIPCUB_CCCL_VERSION_MAJOR 2) -set(HIPCUB_CCCL_VERSION_MINOR 8) -set(HIPCUB_CCCL_VERSION_PATCH 2) +set(HIPCUB_CCCL_VERSION_MAJOR 3) +set(HIPCUB_CCCL_VERSION_MINOR 0) +set(HIPCUB_CCCL_VERSION_PATCH 3) # Set the minimum required rocPRIM version set(MIN_ROCPRIM_PACKAGE_VERSION "4.1.0" CACHE STRING "Minimum version of rocPRIM to search for when ROCPRIM_FETCH_METHOD is set to PACKAGE.") # Set download branch for dependency rocPRIM @@ -59,10 +59,8 @@ endif() set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CXX_EXTENSIONS OFF) -if (CMAKE_CXX_STANDARD EQUAL 14) - message(WARNING "C++14 will be deprecated in the next major release") -elseif(NOT CMAKE_CXX_STANDARD EQUAL 17) - message(FATAL_ERROR "Only C++14 and C++17 are supported") +if(NOT CMAKE_CXX_STANDARD EQUAL 17) + message(FATAL_ERROR "Only C++17 are supported") endif() # Set HIP flags diff --git a/projects/hipcub/README.md b/projects/hipcub/README.md index 0b7863afff7a..9f55c6dfcc8f 100644 --- a/projects/hipcub/README.md +++ b/projects/hipcub/README.md @@ -30,9 +30,9 @@ environment, hipCUB uses the rocPRIM library as the backend. Optional: * [GoogleTest](https://github.com/google/googletest) -* [Google Benchmark](https://github.com/google/benchmark) - -GoogleTest and Google Benchmark are automatically downloaded and built by the CMake script. + * This is automatically downloaded and built by the CMake script +* [AMD SMI](https://github.com/ROCm/amdsmi) + * Required only for benchmarks. Building benchmarks is off by default. ## Build and install @@ -69,7 +69,7 @@ mkdir build; cd build # BUILD_TEST - OFF by default, # BUILD_BENCHMARK - OFF by default. # ROCPRIM_FETCH_METHOD - One of PACKAGE (default), DOWNLOAD, and MONOREPO. See below for a description of each. -# EXTERNAL_DEPS_FORCE_DOWNLOAD - OFF by default, forces download for non-ROCm dependencies (eg. Google Test / Benchmark). +# EXTERNAL_DEPS_FORCE_DOWNLOAD - OFF by default, forces download for non-ROCm dependencies (e.g., Google Test). # DOWNLOAD_CUB - OFF by default, (Nvidia CUB backend only) forces download of CUB instead of searching for an installed package. # BUILD_OFFLOAD_COMPRESS - ON by default, compresses device code to reduce the size of the generated binary. # BUILD_EXAMPLE - OFF by default, builds examples. @@ -253,24 +253,20 @@ ctest --resource-spec-file --parallel [--size ] [--trials ] +./benchmark/benchmark_warp_ # To run benchmark for block functions: -# Further option can be found using --help -# [] Fields are optional -./benchmark/benchmark_block_ [--size ] [--trials ] +./benchmark/benchmark_block_ # To run benchmark for device functions: -# Further option can be found using --help -# [] Fields are optional -./benchmark/benchmark_device_ [--size ] [--trials ] +./benchmark/benchmark_device_ ``` ## Building the documentation locally diff --git a/projects/hipcub/benchmark/CMakeLists.txt b/projects/hipcub/benchmark/CMakeLists.txt index 3e8f663d1a1b..f77496b36d58 100644 --- a/projects/hipcub/benchmark/CMakeLists.txt +++ b/projects/hipcub/benchmark/CMakeLists.txt @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2020-2025 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -20,52 +20,79 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +# Force CMake to find Git immediately so ${GIT_EXECUTABLE} is populated +find_package(Git REQUIRED) + +# Get the current branch, or fall back to DETACHED +execute_process( + COMMAND ${GIT_EXECUTABLE} symbolic-ref -q --short HEAD + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + OUTPUT_VARIABLE BRANCH_NAME + OUTPUT_STRIP_TRAILING_WHITESPACE +) +if(BRANCH_NAME STREQUAL "") + set(BRANCH_NAME "DETACHED") +endif() + +# Get the short commit hash +execute_process( + COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + OUTPUT_VARIABLE COMMIT_HASH + OUTPUT_STRIP_TRAILING_WHITESPACE +) + function(add_hipcub_benchmark BENCHMARK_SOURCE) get_filename_component(BENCHMARK_TARGET ${BENCHMARK_SOURCE} NAME_WE) + if(USE_HIPCXX) set_source_files_properties(${BENCHMARK_SOURCE} PROPERTIES LANGUAGE HIP) endif() + add_executable(${BENCHMARK_TARGET} ${BENCHMARK_SOURCE}) - target_include_directories(${BENCHMARK_TARGET} SYSTEM BEFORE - PUBLIC - "$" - ) - target_link_libraries(${BENCHMARK_TARGET} - PRIVATE - benchmark::benchmark - hipcub - ) - if((HIP_COMPILER STREQUAL "nvcc")) + target_link_libraries(${BENCHMARK_TARGET} PRIVATE hipcub) + + target_compile_definitions(${BENCHMARK_TARGET} PUBLIC BRANCH_NAME="${BRANCH_NAME}") + target_compile_definitions(${BENCHMARK_TARGET} PUBLIC COMMIT_HASH="${COMMIT_HASH}") + + target_include_directories(${BENCHMARK_TARGET} PRIVATE ../../../shared/primbench) + + if(WIN32) + target_compile_definitions(${BENCHMARK_TARGET} PRIVATE PRIMBENCH_NO_MONITORING) + elseif((HIP_COMPILER STREQUAL "nvcc")) set_property(TARGET ${BENCHMARK_TARGET} PROPERTY CUDA_STANDARD 17) set_source_files_properties(${BENCHMARK_SOURCE} PROPERTIES LANGUAGE CUDA) target_compile_options(${BENCHMARK_TARGET} PRIVATE - $<$:--expt-extended-lambda> - ) - target_link_libraries(${BENCHMARK_TARGET} - PRIVATE - hipcub_cub + $<$:--expt-extended-lambda> ) + target_link_libraries(${BENCHMARK_TARGET} PRIVATE hipcub_cub nvidia-ml) + else() + target_link_libraries(${BENCHMARK_TARGET} PRIVATE amd_smi) endif() + set_target_properties(${BENCHMARK_TARGET} PROPERTIES - RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/benchmark" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/benchmark" ) rocm_install(TARGETS ${BENCHMARK_TARGET} COMPONENT benchmarks) - if (WIN32 AND NOT DEFINED DLLS_COPIED) + + if(WIN32 AND NOT DEFINED DLLS_COPIED) set(DLLS_COPIED "YES") set(DLLS_COPIED ${DLLS_COPIED} PARENT_SCOPE) + # for now adding in all .dll as dependency chain is not cmake based on win32 - file( GLOB third_party_dlls - LIST_DIRECTORIES ON - CONFIGURE_DEPENDS - ${HIP_DIR}/bin/*.dll - ${CMAKE_SOURCE_DIR}/rtest.* + file(GLOB third_party_dlls + LIST_DIRECTORIES ON + CONFIGURE_DEPENDS + ${HIP_DIR}/bin/*.dll + ${CMAKE_SOURCE_DIR}/rtest.* ) - foreach( file_i ${third_party_dlls}) - add_custom_command( TARGET ${BENCHMARK_TARGET} POST_BUILD COMMAND ${CMAKE_COMMAND} ARGS -E copy_if_different ${file_i} ${PROJECT_BINARY_DIR}/benchmark ) - endforeach( file_i ) - endif() + + foreach(file_i ${third_party_dlls}) + add_custom_command(TARGET ${BENCHMARK_TARGET} POST_BUILD COMMAND ${CMAKE_COMMAND} ARGS -E copy_if_different ${file_i} ${PROJECT_BINARY_DIR}/benchmark) + endforeach(file_i) + endif() endfunction() # **************************************************************************** @@ -76,8 +103,8 @@ add_hipcub_benchmark(benchmark_block_discontinuity.cpp) add_hipcub_benchmark(benchmark_block_exchange.cpp) add_hipcub_benchmark(benchmark_block_histogram.cpp) add_hipcub_benchmark(benchmark_block_merge_sort.cpp) -add_hipcub_benchmark(benchmark_block_radix_sort.cpp) add_hipcub_benchmark(benchmark_block_radix_rank.cpp) +add_hipcub_benchmark(benchmark_block_radix_sort.cpp) add_hipcub_benchmark(benchmark_block_reduce.cpp) add_hipcub_benchmark(benchmark_block_run_length_decode.cpp) add_hipcub_benchmark(benchmark_block_scan.cpp) @@ -85,25 +112,24 @@ add_hipcub_benchmark(benchmark_block_shuffle.cpp) add_hipcub_benchmark(benchmark_device_adjacent_difference.cpp) add_hipcub_benchmark(benchmark_device_batch_copy.cpp) add_hipcub_benchmark(benchmark_device_batch_memcpy.cpp) -add_hipcub_benchmark(benchmark_device_for.cpp) +add_hipcub_benchmark(benchmark_device_for_each.cpp) add_hipcub_benchmark(benchmark_device_histogram.cpp) add_hipcub_benchmark(benchmark_device_memory.cpp) -add_hipcub_benchmark(benchmark_device_merge.cpp) add_hipcub_benchmark(benchmark_device_merge_sort.cpp) +add_hipcub_benchmark(benchmark_device_merge.cpp) add_hipcub_benchmark(benchmark_device_partition.cpp) add_hipcub_benchmark(benchmark_device_radix_sort.cpp) add_hipcub_benchmark(benchmark_device_reduce_by_key.cpp) add_hipcub_benchmark(benchmark_device_reduce.cpp) add_hipcub_benchmark(benchmark_device_run_length_encode.cpp) add_hipcub_benchmark(benchmark_device_scan.cpp) -add_hipcub_benchmark(benchmark_device_segmented_sort.cpp) add_hipcub_benchmark(benchmark_device_segmented_radix_sort.cpp) add_hipcub_benchmark(benchmark_device_segmented_reduce.cpp) +add_hipcub_benchmark(benchmark_device_segmented_sort.cpp) add_hipcub_benchmark(benchmark_device_select.cpp) -add_hipcub_benchmark(benchmark_device_spmv.cpp) add_hipcub_benchmark(benchmark_warp_exchange.cpp) add_hipcub_benchmark(benchmark_warp_load.cpp) +add_hipcub_benchmark(benchmark_warp_merge_sort.cpp) add_hipcub_benchmark(benchmark_warp_reduce.cpp) add_hipcub_benchmark(benchmark_warp_scan.cpp) add_hipcub_benchmark(benchmark_warp_store.cpp) -add_hipcub_benchmark(benchmark_warp_merge_sort.cpp) diff --git a/projects/hipcub/benchmark/benchmark_block_adjacent_difference.cpp b/projects/hipcub/benchmark/benchmark_block_adjacent_difference.cpp index 7c7ac6bc1938..11d31468a3e6 100644 --- a/projects/hipcub/benchmark/benchmark_block_adjacent_difference.cpp +++ b/projects/hipcub/benchmark/benchmark_block_adjacent_difference.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2022 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,23 +20,21 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include #include #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 128; -#endif +constexpr unsigned int Trials = 100; template -__global__ __launch_bounds__(BlockSize) void kernel(Args... args) +__global__ __launch_bounds__(BlockSize) +void kernel(Args... args) { Benchmark::template run(args...); } @@ -44,7 +42,8 @@ __global__ __launch_bounds__(BlockSize) void kernel(Args... args) template struct minus { - HIPCUB_HOST_DEVICE inline constexpr T operator()(const T& a, const T& b) const + HIPCUB_HOST_DEVICE + inline constexpr T operator()(const T& a, const T& b) const { return a - b; } @@ -52,8 +51,11 @@ struct minus struct subtract_left { + static constexpr const char* name = "subtract_left"; + template - __device__ static void run(const T* d_input, T* d_output, unsigned int trials) + __device__ + static void run(const T* d_input, T* d_output, unsigned int trials) { const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; @@ -63,14 +65,15 @@ struct subtract_left hipcub::BlockAdjacentDifference adjacent_difference; -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < trials; trial++) { T output[ItemsPerThread]; if(WithTile) { adjacent_difference.SubtractLeft(input, output, minus{}, T(123)); - } else + } + else { adjacent_difference.SubtractLeft(input, output, minus{}); } @@ -89,9 +92,11 @@ struct subtract_left struct subtract_left_partial_tile { + static constexpr const char* name = "subtract_left_partial_tile"; + template - __device__ static void - run(const T* d_input, const int* tile_sizes, T* d_output, unsigned int trials) + __device__ + static void run(const T* d_input, const int* tile_sizes, T* d_output, unsigned int trials) { const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; @@ -106,7 +111,7 @@ struct subtract_left_partial_tile // Try to evenly distribute the length of tile_sizes between all the trials const auto tile_size_diff = (BlockSize * ItemsPerThread) / trials + 1; -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < trials; trial++) { T output[ItemsPerThread]; @@ -118,7 +123,8 @@ struct subtract_left_partial_tile minus{}, tile_size, T(123)); - } else + } + else { adjacent_difference.SubtractLeftPartialTile(input, output, minus{}, tile_size); } @@ -139,8 +145,11 @@ struct subtract_left_partial_tile struct subtract_right { + static constexpr const char* name = "subtract_right"; + template - __device__ static void run(const T* d_input, T* d_output, unsigned int trials) + __device__ + static void run(const T* d_input, T* d_output, unsigned int trials) { const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; @@ -150,14 +159,15 @@ struct subtract_right hipcub::BlockAdjacentDifference adjacent_difference; -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < trials; trial++) { T output[ItemsPerThread]; if(WithTile) { adjacent_difference.SubtractRight(input, output, minus{}, T(123)); - } else + } + else { adjacent_difference.SubtractRight(input, output, minus{}); } @@ -176,9 +186,11 @@ struct subtract_right struct subtract_right_partial_tile { + static constexpr const char* name = "subtract_right_partial_tile"; + template - __device__ static void - run(const T* d_input, const int* tile_sizes, T* d_output, unsigned int trials) + __device__ + static void run(const T* d_input, const int* tile_sizes, T* d_output, unsigned int trials) { const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; @@ -193,7 +205,7 @@ struct subtract_right_partial_tile // Try to evenly distribute the length of tile_sizes between all the trials const auto tile_size_diff = (BlockSize * ItemsPerThread) / trials + 1; -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < trials; trial++) { T output[ItemsPerThread]; @@ -218,122 +230,143 @@ template -auto run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) - -> std::enable_if_t::value - && !std::is_same::value> + bool WithTile> +class block_adjacent_difference_benchmark : public primbench::benchmark_interface { - constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto num_blocks = (N + items_per_block - 1) / items_per_block; - // Round up size to the next multiple of items_per_block - const auto size = num_blocks * items_per_block; - - const std::vector input = benchmark_utils::get_random_data(size, T(0), T(10)); - T* d_input; - T* d_output; - HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(input[0]))); - HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(T))); - HIP_CHECK( - hipMemcpy(d_input, input.data(), input.size() * sizeof(input[0]), hipMemcpyHostToDevice)); - - for(auto _ : state) + primbench::json meta() const override { - auto start = std::chrono::high_resolution_clock::now(); - - hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), - dim3(num_blocks), - dim3(BlockSize), - 0, - stream, - d_input, - d_output, - Trials); - HIP_CHECK(hipGetLastError()); - HIP_CHECK(hipDeviceSynchronize()); - - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + return primbench::json{} + .add("algo", "block_adjacent_difference") + .add("subalgo", Benchmark::name) + .add("lvl", "block") + .add("data_type", primbench::name()) + .add("block_size", BlockSize) + .add("items_per_thread", ItemsPerThread) + .add("with_tile", WithTile); } - state.SetBytesProcessed(state.iterations() * Trials * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * Trials * size); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); -} + void run(primbench::state& state) override + { + const size_t input_items = state.size; + const auto& stream = state.stream; + + constexpr auto items_per_block = BlockSize * ItemsPerThread; + const auto num_blocks = (input_items + items_per_block - 1) / items_per_block; + + // Round up items to the next multiple of items_per_block + const auto items = num_blocks * items_per_block; + + const std::vector input = benchmark_utils::get_random_data(items, T(0), T(10)); + T* d_input; + T* d_output; + HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(input[0]))); + HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(T))); + HIP_CHECK(hipMemcpy(d_input, + input.data(), + input.size() * sizeof(input[0]), + hipMemcpyHostToDevice)); + + state.set_items(Trials * items); + state.add_writes(Trials * items); + + state.run( + [&] + { + hipLaunchKernelGGL( + HIP_KERNEL_NAME(kernel), + dim3(num_blocks), + dim3(BlockSize), + 0, + stream, + d_input, + d_output, + Trials); + }); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); + } +}; template -auto run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) - -> std::enable_if_t::value - || std::is_same::value> + bool WithTile> +class block_adjacent_difference_partial_tile_benchmark : public primbench::benchmark_interface { - constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto num_blocks = (N + items_per_block - 1) / items_per_block; - // Round up size to the next multiple of items_per_block - const auto size = num_blocks * items_per_block; - - const std::vector input = benchmark_utils::get_random_data(size, T(0), T(10)); - const std::vector tile_sizes - = benchmark_utils::get_random_data(num_blocks, 0, items_per_block); - - T* d_input; - int* d_tile_sizes; - T* d_output; - HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(input[0]))); - HIP_CHECK(hipMalloc(&d_tile_sizes, tile_sizes.size() * sizeof(tile_sizes[0]))); - HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(T))); - HIP_CHECK( - hipMemcpy(d_input, input.data(), input.size() * sizeof(input[0]), hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_tile_sizes, - tile_sizes.data(), - tile_sizes.size() * sizeof(tile_sizes[0]), - hipMemcpyHostToDevice)); - - for(auto _ : state) + primbench::json meta() const override { - auto start = std::chrono::high_resolution_clock::now(); - - hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), - dim3(num_blocks), - dim3(BlockSize), - 0, - stream, - d_input, - d_tile_sizes, - d_output, - Trials); - HIP_CHECK(hipGetLastError()); - HIP_CHECK(hipDeviceSynchronize()); - - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + return primbench::json{} + .add("algo", "block_adjacent_difference") + .add("subalgo", Benchmark::name) + .add("lvl", "block") + .add("data_type", primbench::name()) + .add("block_size", BlockSize) + .add("items_per_thread", ItemsPerThread) + .add("with_tile", WithTile); } - state.SetBytesProcessed(state.iterations() * Trials * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * Trials * size); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_tile_sizes)); - HIP_CHECK(hipFree(d_output)); -} + void run(primbench::state& state) override + { + const size_t input_items = state.size; + const auto& stream = state.stream; + + constexpr auto items_per_block = BlockSize * ItemsPerThread; + const auto num_blocks = (input_items + items_per_block - 1) / items_per_block; + + // Round up items to the next multiple of items_per_block + const auto items = num_blocks * items_per_block; + + const std::vector input = benchmark_utils::get_random_data(items, T(0), T(10)); + const std::vector tile_sizes + = benchmark_utils::get_random_data(num_blocks, 0, items_per_block); + + T* d_input; + int* d_tile_sizes; + T* d_output; + HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(input[0]))); + HIP_CHECK(hipMalloc(&d_tile_sizes, tile_sizes.size() * sizeof(tile_sizes[0]))); + HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(T))); + HIP_CHECK(hipMemcpy(d_input, + input.data(), + input.size() * sizeof(input[0]), + hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(d_tile_sizes, + tile_sizes.data(), + tile_sizes.size() * sizeof(tile_sizes[0]), + hipMemcpyHostToDevice)); + + state.set_items(Trials * items); + state.add_writes(Trials * items); + + state.run( + [&] + { + hipLaunchKernelGGL( + HIP_KERNEL_NAME(kernel), + dim3(num_blocks), + dim3(BlockSize), + 0, + stream, + d_input, + d_tile_sizes, + d_output, + Trials); + }); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_tile_sizes)); + HIP_CHECK(hipFree(d_output)); + } +}; -#define CREATE_BENCHMARK(T, BS, IPT, WITH_TILE) \ - benchmark::RegisterBenchmark( \ - std::string("block_adjacent_difference.sub_algorithm_name:" \ - + name + "") \ - .c_str(), \ - &run_benchmark, \ - stream, \ - size) +// or use block_adjacent_difference_partial_tile_benchmark +#define CREATE_BENCHMARK(T, BS, IPT, WITH_TILE) \ + executor.queue, \ + block_adjacent_difference_benchmark>>() #define BENCHMARK_TYPE(type, block, with_tile) \ CREATE_BENCHMARK(type, block, 1, with_tile), CREATE_BENCHMARK(type, block, 3, with_tile), \ @@ -341,79 +374,39 @@ auto run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) CREATE_BENCHMARK(type, block, 16, with_tile), CREATE_BENCHMARK(type, block, 32, with_tile) template -void add_benchmarks(const std::string& name, - std::vector& benchmarks, - hipStream_t stream, - size_t size) +void add_benchmarks(primbench::executor& executor) { - std::vector bs = {BENCHMARK_TYPE(int, 256, false), - BENCHMARK_TYPE(float, 256, false), - BENCHMARK_TYPE(int8_t, 256, false), - BENCHMARK_TYPE(long long, 256, false), - BENCHMARK_TYPE(double, 256, false)}; + constexpr bool is_partial = std::is_same_v + || std::is_same_v; - if(!std::is_same::value) + BENCHMARK_TYPE(int, 256, false); + BENCHMARK_TYPE(float, 256, false); + BENCHMARK_TYPE(int8_t, 256, false); + BENCHMARK_TYPE(int64_t, 256, false); + BENCHMARK_TYPE(double, 256, false); + + if(!std::is_same_v) { - bs.insert(bs.end(), - {BENCHMARK_TYPE(int, 256, true), - BENCHMARK_TYPE(float, 256, true), - BENCHMARK_TYPE(int8_t, 256, true), - BENCHMARK_TYPE(long long, 256, true), - BENCHMARK_TYPE(double, 256, true)}); + BENCHMARK_TYPE(int, 256, true); + BENCHMARK_TYPE(float, 256, true); + BENCHMARK_TYPE(int8_t, 256, true); + BENCHMARK_TYPE(int64_t, 256, true); + BENCHMARK_TYPE(double, 256, true); } - - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - - std::cout << "benchmark_block_adjacent_difference" << std::endl; - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - add_benchmarks("subtract_left", benchmarks, stream, size); - add_benchmarks("subtract_right", benchmarks, stream, size); - add_benchmarks("subtract_left_partial_tile", benchmarks, stream, size); - add_benchmarks("subtract_right_partial_tile", - benchmarks, - stream, - size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; + settings.size = 128 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + primbench::executor executor(argc, argv, settings); + + add_benchmarks(executor); + add_benchmarks(executor); + add_benchmarks(executor); + add_benchmarks(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; -} \ No newline at end of file + executor.run(); +} diff --git a/projects/hipcub/benchmark/benchmark_block_discontinuity.cpp b/projects/hipcub/benchmark/benchmark_block_discontinuity.cpp index 5e36160c1405..b818560fd4af 100644 --- a/projects/hipcub/benchmark/benchmark_block_discontinuity.cpp +++ b/projects/hipcub/benchmark/benchmark_block_discontinuity.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,47 +20,38 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -// HIP API +#include "benchmark_utils.hpp" + #include #include #include -#include //to use hipcub::Equality - -#include "common_benchmark_header.hpp" -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 128; -#endif +constexpr unsigned int Trials = 100; -template -struct custom_flag_op1 +struct equal { + template HIPCUB_HOST_DEVICE - bool operator()(const T& a, const T& b) const + bool operator()(const A& a, const B& b) const { return (a == b); } }; -template -__global__ __launch_bounds__(BlockSize) void kernel(const T* d_input, T* d_output) +template +__global__ __launch_bounds__(BlockSize) +void kernel(const T* d_input, T* d_output) { - Runner::template run(d_input, d_output); + Runner::template run(d_input, d_output); } struct flag_heads { - template - __device__ static void run(const T* d_input, T* d_output) + static constexpr const char* name = "flag_heads"; + + template + __device__ + static void run(const T* d_input, T* d_output) { const unsigned int lid = hipThreadIdx_x; const unsigned int block_offset = hipBlockIdx_x * ItemsPerThread * BlockSize; @@ -68,17 +59,18 @@ struct flag_heads T input[ItemsPerThread]; hipcub::LoadDirectStriped(lid, d_input + block_offset, input); -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockDiscontinuity bdiscontinuity; bool head_flags[ItemsPerThread]; if(WithTile) { - bdiscontinuity.FlagHeads(head_flags, input, hipcub::Equality(), T(123)); - } else + bdiscontinuity.FlagHeads(head_flags, input, equal(), T(123)); + } + else { - bdiscontinuity.FlagHeads(head_flags, input, hipcub::Equality()); + bdiscontinuity.FlagHeads(head_flags, input, equal()); } for(unsigned int i = 0; i < ItemsPerThread; i++) @@ -93,12 +85,11 @@ struct flag_heads struct flag_tails { - template - __device__ static void run(const T* d_input, T* d_output) + static constexpr const char* name = "flag_tails"; + + template + __device__ + static void run(const T* d_input, T* d_output) { const unsigned int lid = hipThreadIdx_x; const unsigned int block_offset = hipBlockIdx_x * ItemsPerThread * BlockSize; @@ -106,17 +97,18 @@ struct flag_tails T input[ItemsPerThread]; hipcub::LoadDirectStriped(lid, d_input + block_offset, input); -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockDiscontinuity bdiscontinuity; bool tail_flags[ItemsPerThread]; if(WithTile) { - bdiscontinuity.FlagTails(tail_flags, input, hipcub::Equality(), T(123)); - } else + bdiscontinuity.FlagTails(tail_flags, input, equal(), T(123)); + } + else { - bdiscontinuity.FlagTails(tail_flags, input, hipcub::Equality()); + bdiscontinuity.FlagTails(tail_flags, input, equal()); } for(unsigned int i = 0; i < ItemsPerThread; i++) @@ -131,12 +123,11 @@ struct flag_tails struct flag_heads_and_tails { - template - __device__ static void run(const T* d_input, T* d_output) + static constexpr const char* name = "flag_heads_and_tails"; + + template + __device__ + static void run(const T* d_input, T* d_output) { const unsigned int lid = hipThreadIdx_x; const unsigned int block_offset = hipBlockIdx_x * ItemsPerThread * BlockSize; @@ -144,7 +135,7 @@ struct flag_heads_and_tails T input[ItemsPerThread]; hipcub::LoadDirectStriped(lid, d_input + block_offset, input); -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockDiscontinuity bdiscontinuity; @@ -152,15 +143,12 @@ struct flag_heads_and_tails bool tail_flags[ItemsPerThread]; if(WithTile) { - bdiscontinuity.FlagHeadsAndTails(head_flags, - T(123), - tail_flags, - T(234), - input, - hipcub::Equality()); - } else + bdiscontinuity + .FlagHeadsAndTails(head_flags, T(123), tail_flags, T(234), input, equal()); + } + else { - bdiscontinuity.FlagHeadsAndTails(head_flags, tail_flags, input, hipcub::Equality()); + bdiscontinuity.FlagHeadsAndTails(head_flags, tail_flags, input, equal()); } for(unsigned int i = 0; i < ItemsPerThread; i++) @@ -178,56 +166,61 @@ template -void run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) + bool WithTile> +class block_discontinuity_benchmark : public primbench::benchmark_interface { - constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto size = items_per_block * ((N + items_per_block - 1) / items_per_block); - - std::vector input = benchmark_utils::get_random_data(size, T(0), T(10)); - T* d_input; - T* d_output; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMalloc(&d_output, size * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - for(auto _ : state) + primbench::json meta() const override + { + return primbench::json{} + .add("algo", "block_discontinuity") + .add("subalgo", Benchmark::name) + .add("lvl", "block") + .add("data_type", primbench::name()) + .add("block_size", BlockSize) + .add("items_per_thread", ItemsPerThread) + .add("with_tile", WithTile); + } + + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); - - hipLaunchKernelGGL( - HIP_KERNEL_NAME(kernel), - dim3(size / items_per_block), - dim3(BlockSize), - 0, - stream, - d_input, - d_output); - HIP_CHECK(hipPeekAtLastError()); + const size_t input_items = state.size; + const auto& stream = state.stream; + + constexpr auto items_per_block = BlockSize * ItemsPerThread; + const auto items + = items_per_block * ((input_items + items_per_block - 1) / items_per_block); + + std::vector input = benchmark_utils::get_random_data(items, T(0), T(10)); + T* d_input; + T* d_output; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK(hipMalloc(&d_output, items * sizeof(T))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * Trials * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * Trials * size); + state.set_items(Trials * items); + state.add_writes(Trials * items); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); -} + state.run( + [&] + { + hipLaunchKernelGGL( + HIP_KERNEL_NAME(kernel), + dim3(items / items_per_block), + dim3(BlockSize), + 0, + stream, + d_input, + d_output); + }); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); + } +}; -#define CREATE_BENCHMARK(T, BS, IPT, WITH_TILE) \ - benchmark::RegisterBenchmark( \ - std::string("block_discontinuity.sub_algorithm_name:" \ - + name + ".") \ - .c_str(), \ - &run_benchmark, \ - stream, \ - size) +#define CREATE_BENCHMARK(T, BS, IPT, WITH_TILE) \ + executor.queue>() #define BENCHMARK_TYPE(type, block, bool) \ CREATE_BENCHMARK(type, block, 1, bool), CREATE_BENCHMARK(type, block, 2, bool), \ @@ -235,70 +228,30 @@ void run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) CREATE_BENCHMARK(type, block, 8, bool) template -void add_benchmarks(const std::string& name, - std::vector& benchmarks, - hipStream_t stream, - size_t size) +void add_benchmarks(primbench::executor& executor) { - std::vector bs = { - BENCHMARK_TYPE(int, 256, false), - BENCHMARK_TYPE(int, 256, true), - BENCHMARK_TYPE(int8_t, 256, false), - BENCHMARK_TYPE(int8_t, 256, true), - BENCHMARK_TYPE(uint8_t, 256, false), - BENCHMARK_TYPE(uint8_t, 256, true), - BENCHMARK_TYPE(long long, 256, false), - BENCHMARK_TYPE(long long, 256, true), - }; - - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + BENCHMARK_TYPE(int, 256, false); + BENCHMARK_TYPE(int, 256, true); + BENCHMARK_TYPE(int8_t, 256, false); + BENCHMARK_TYPE(int8_t, 256, true); + BENCHMARK_TYPE(uint8_t, 256, false); + BENCHMARK_TYPE(uint8_t, 256, true); + BENCHMARK_TYPE(int64_t, 256, false); + BENCHMARK_TYPE(int64_t, 256, true); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_block_discontinuity" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - add_benchmarks("flag_heads", benchmarks, stream, size); - add_benchmarks("flag_tails", benchmarks, stream, size); - add_benchmarks("flag_heads_and_tails", benchmarks, stream, size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; + settings.size = 128 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 1000; + settings.batch_window_size = 3; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + primbench::executor executor(argc, argv, settings); + + add_benchmarks(executor); + add_benchmarks(executor); + add_benchmarks(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_block_exchange.cpp b/projects/hipcub/benchmark/benchmark_block_exchange.cpp index 000cd41be6a0..0bb272907643 100644 --- a/projects/hipcub/benchmark/benchmark_block_exchange.cpp +++ b/projects/hipcub/benchmark/benchmark_block_exchange.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,33 +20,28 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include #include #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif - -template -__global__ __launch_bounds__(BlockSize) void kernel(const T* d_input, - const unsigned int* d_ranks, - T* d_output) +constexpr unsigned int Trials = 100; + +template +__global__ __launch_bounds__(BlockSize) +void kernel(const T* d_input, const unsigned int* d_ranks, T* d_output) { - Runner::template run(d_input, d_ranks, d_output); + Runner::template run(d_input, d_ranks, d_output); } struct blocked_to_striped { - template - __device__ static void run(const T* d_input, const unsigned int*, T* d_output) + static constexpr const char* name = "blocked_to_striped"; + + template + __device__ + static void run(const T* d_input, const unsigned int*, T* d_output) { const unsigned int lid = hipThreadIdx_x; const unsigned int block_offset = hipBlockIdx_x * ItemsPerThread * BlockSize; @@ -54,7 +49,7 @@ struct blocked_to_striped T input[ItemsPerThread]; hipcub::LoadDirectBlocked(lid, d_input + block_offset, input); -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockExchange exchange; @@ -69,8 +64,11 @@ struct blocked_to_striped struct striped_to_blocked { - template - __device__ static void run(const T* d_input, const unsigned int*, T* d_output) + static constexpr const char* name = "striped_to_blocked"; + + template + __device__ + static void run(const T* d_input, const unsigned int*, T* d_output) { const unsigned int lid = hipThreadIdx_x; const unsigned int block_offset = hipBlockIdx_x * ItemsPerThread * BlockSize; @@ -78,7 +76,7 @@ struct striped_to_blocked T input[ItemsPerThread]; hipcub::LoadDirectStriped(lid, d_input + block_offset, input); -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockExchange exchange; @@ -93,8 +91,11 @@ struct striped_to_blocked struct blocked_to_warp_striped { - template - __device__ static void run(const T* d_input, const unsigned int*, T* d_output) + static constexpr const char* name = "blocked_to_warp_striped"; + + template + __device__ + static void run(const T* d_input, const unsigned int*, T* d_output) { const unsigned int lid = hipThreadIdx_x; const unsigned int block_offset = hipBlockIdx_x * ItemsPerThread * BlockSize; @@ -102,7 +103,7 @@ struct blocked_to_warp_striped T input[ItemsPerThread]; hipcub::LoadDirectBlocked(lid, d_input + block_offset, input); -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockExchange exchange; @@ -117,8 +118,11 @@ struct blocked_to_warp_striped struct warp_striped_to_blocked { - template - __device__ static void run(const T* d_input, const unsigned int*, T* d_output) + static constexpr const char* name = "warp_striped_to_blocked"; + + template + __device__ + static void run(const T* d_input, const unsigned int*, T* d_output) { const unsigned int lid = hipThreadIdx_x; const unsigned int block_offset = hipBlockIdx_x * ItemsPerThread * BlockSize; @@ -126,7 +130,7 @@ struct warp_striped_to_blocked T input[ItemsPerThread]; hipcub::LoadDirectWarpStriped(lid, d_input + block_offset, input); -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockExchange exchange; @@ -141,8 +145,11 @@ struct warp_striped_to_blocked struct scatter_to_blocked { - template - __device__ static void run(const T* d_input, const unsigned int* d_ranks, T* d_output) + static constexpr const char* name = "scatter_to_blocked"; + + template + __device__ + static void run(const T* d_input, const unsigned int* d_ranks, T* d_output) { const unsigned int lid = hipThreadIdx_x; const unsigned int block_offset = hipBlockIdx_x * ItemsPerThread * BlockSize; @@ -152,7 +159,7 @@ struct scatter_to_blocked hipcub::LoadDirectStriped(lid, d_input + block_offset, input); hipcub::LoadDirectStriped(lid, d_ranks + block_offset, ranks); -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockExchange exchange; @@ -167,8 +174,11 @@ struct scatter_to_blocked struct scatter_to_striped { - template - __device__ static void run(const T* d_input, const unsigned int* d_ranks, T* d_output) + static constexpr const char* name = "scatter_to_striped"; + + template + __device__ + static void run(const T* d_input, const unsigned int* d_ranks, T* d_output) { const unsigned int lid = hipThreadIdx_x; const unsigned int block_offset = hipBlockIdx_x * ItemsPerThread * BlockSize; @@ -178,7 +188,7 @@ struct scatter_to_striped hipcub::LoadDirectStriped(lid, d_input + block_offset, input); hipcub::LoadDirectStriped(lid, d_ranks + block_offset, ranks); -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockExchange exchange; @@ -191,77 +201,81 @@ struct scatter_to_striped } }; -template -void run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) +template +class block_exchange_benchmark : public primbench::benchmark_interface { - constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto size = items_per_block * ((N + items_per_block - 1) / items_per_block); - - std::vector input(size); - // Fill input - for(size_t i = 0; i < size; i++) + primbench::json meta() const override { - input[i] = T(i); + return primbench::json{} + .add("algo", "block_exchange") + .add("subalgo", Benchmark::name) + .add("lvl", "block") + .add("data_type", primbench::name()) + .add("block_size", BlockSize) + .add("items_per_thread", ItemsPerThread); } - std::vector ranks(size); - // Fill ranks (for scatter operations) - std::mt19937 gen; - for(size_t bi = 0; bi < size / items_per_block; bi++) - { - auto block_ranks = ranks.begin() + bi * items_per_block; - std::iota(block_ranks, block_ranks + items_per_block, 0); - std::shuffle(block_ranks, block_ranks + items_per_block, gen); - } - T* d_input; - unsigned int* d_ranks; - T* d_output; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMalloc(&d_ranks, size * sizeof(unsigned int))); - HIP_CHECK(hipMalloc(&d_output, size * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_ranks, ranks.data(), size * sizeof(unsigned int), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - for(auto _ : state) + + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); - - hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), - dim3(size / items_per_block), - dim3(BlockSize), - 0, - stream, - d_input, - d_ranks, - d_output); - HIP_CHECK(hipPeekAtLastError()); + const size_t input_items = state.size; + const auto& stream = state.stream; + + constexpr auto items_per_block = BlockSize * ItemsPerThread; + const auto items + = items_per_block * ((input_items + items_per_block - 1) / items_per_block); + + std::vector input(items); + + // Fill input + for(size_t i = 0; i < items; i++) + { + input[i] = T(i); + } + + // Fill ranks (for scatter operations) + std::vector ranks(items); + std::mt19937 gen; + for(size_t bi = 0; bi < items / items_per_block; bi++) + { + auto block_ranks = ranks.begin() + bi * items_per_block; + std::iota(block_ranks, block_ranks + items_per_block, 0); + std::shuffle(block_ranks, block_ranks + items_per_block, gen); + } + T* d_input; + unsigned int* d_ranks; + T* d_output; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK(hipMalloc(&d_ranks, items * sizeof(unsigned int))); + HIP_CHECK(hipMalloc(&d_output, items * sizeof(T))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK( + hipMemcpy(d_ranks, ranks.data(), items * sizeof(unsigned int), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + state.set_items(Trials * items); + state.add_writes(Trials * items); + + state.run( + [&] + { + hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), + dim3(items / items_per_block), + dim3(BlockSize), + 0, + stream, + d_input, + d_ranks, + d_output); + }); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_ranks)); + HIP_CHECK(hipFree(d_output)); } - state.SetBytesProcessed(state.iterations() * Trials * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * Trials * size); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_ranks)); - HIP_CHECK(hipFree(d_output)); -} +}; -#define CREATE_BENCHMARK(T, BS, IPT) \ - benchmark::RegisterBenchmark(std::string("block_exchange.sub_algorithm_name:" \ - + name) \ - .c_str(), \ - &run_benchmark, \ - stream, \ - size) +#define CREATE_BENCHMARK(T, BS, IPT) \ + executor.queue>() #define BENCHMARK_TYPE(type, block) \ CREATE_BENCHMARK(type, block, 1), CREATE_BENCHMARK(type, block, 2), \ @@ -269,73 +283,30 @@ void run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) CREATE_BENCHMARK(type, block, 7), CREATE_BENCHMARK(type, block, 8) template -void add_benchmarks(const std::string& name, - std::vector& benchmarks, - hipStream_t stream, - size_t size) +void add_benchmarks(primbench::executor& executor) { - using custom_float2 = benchmark_utils::custom_type; - using custom_double2 = benchmark_utils::custom_type; - - std::vector bs = { - BENCHMARK_TYPE(int, 256), - BENCHMARK_TYPE(int8_t, 256), - BENCHMARK_TYPE(long long, 256), - BENCHMARK_TYPE(custom_float2, 256), - BENCHMARK_TYPE(custom_double2, 256), - }; - - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + + BENCHMARK_TYPE(int, 256); + BENCHMARK_TYPE(int8_t, 256); + BENCHMARK_TYPE(int64_t, 256); + BENCHMARK_TYPE(custom_float2, 256); + BENCHMARK_TYPE(custom_double2, 256); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_block_exchange" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - add_benchmarks("blocked_to_striped", benchmarks, stream, size); - add_benchmarks("striped_to_blocked", benchmarks, stream, size); - add_benchmarks("blocked_to_warp_striped", benchmarks, stream, size); - add_benchmarks("warp_striped_to_blocked", benchmarks, stream, size); - add_benchmarks("scatter_to_blocked", benchmarks, stream, size); - add_benchmarks("scatter_to_striped", benchmarks, stream, size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + primbench::executor executor(argc, argv, settings); + + add_benchmarks(executor); + add_benchmarks(executor); + add_benchmarks(executor); + add_benchmarks(executor); + add_benchmarks(executor); + add_benchmarks(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_block_histogram.cpp b/projects/hipcub/benchmark/benchmark_block_histogram.cpp index 1206e0423537..f9c8263c5c73 100644 --- a/projects/hipcub/benchmark/benchmark_block_histogram.cpp +++ b/projects/hipcub/benchmark/benchmark_block_histogram.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,35 +20,29 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 128; -#endif +constexpr unsigned int Trials = 100; template -__global__ __launch_bounds__(BlockSize) void kernel(const T* input, T* output) + unsigned int BinSize> +__global__ __launch_bounds__(BlockSize) +void kernel(const T* input, T* output) { - Runner::template run(input, output); + Runner::template run(input, output); } template struct histogram { - template - __device__ static void run(const T* input, T* output) + template + __device__ + static void run(const T* input, T* output) { const unsigned int index = ((hipBlockIdx_x * BlockSize) + hipThreadIdx_x) * ItemsPerThread; unsigned int global_offset = hipBlockIdx_x * BinSize; @@ -61,16 +55,18 @@ struct histogram using bhistogram_t = hipcub::BlockHistogram; - __shared__ T histogram[BinSize]; - __shared__ typename bhistogram_t::TempStorage storage; + __shared__ + T histogram[BinSize]; + __shared__ + typename bhistogram_t::TempStorage storage; -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { bhistogram_t(storage).Histogram(values, histogram); } -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(unsigned int offset = 0; offset < BinSize; offset += BlockSize) { if(offset + hipThreadIdx_x < BinSize) @@ -82,63 +78,88 @@ struct histogram } }; +using histogram_a_t = histogram; +using histogram_s_t = histogram; + +template +struct histogram_algorithm_name; + +template<> +struct histogram_algorithm_name +{ + static constexpr const char* value = "using_atomic"; +}; + +template<> +struct histogram_algorithm_name +{ + static constexpr const char* value = "using_sort"; +}; + template -void run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) + unsigned int BinSize = BlockSize> +class block_histogram_benchmark : public primbench::benchmark_interface { - // Make sure size is a multiple of BlockSize - constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto size = items_per_block * ((N + items_per_block - 1) / items_per_block); - const auto bin_size = BinSize * ((N + items_per_block - 1) / items_per_block); - // Allocate and fill memory - std::vector input(size, 0.0f); - T* d_input; - T* d_output; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMalloc(&d_output, bin_size * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - for(auto _ : state) + primbench::json meta() const override + { + return primbench::json{} + .add("algo", "block_histogram") + .add("subalgo", histogram_algorithm_name::value) + .add("lvl", "block") + .add("data_type", primbench::name()) + .add("block_size", BlockSize) + .add("items_per_thread", ItemsPerThread); + + // BinSize is always equal to BlockSize + // .add("bin_size", BinSize); + } + + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); - hipLaunchKernelGGL( - HIP_KERNEL_NAME(kernel), - dim3(size / items_per_block), - dim3(BlockSize), - 0, - stream, - d_input, - d_output); - HIP_CHECK(hipPeekAtLastError()); + const size_t input_items = state.size; + const auto& stream = state.stream; + + // Make sure size is a multiple of BlockSize + constexpr auto items_per_block = BlockSize * ItemsPerThread; + const auto items + = items_per_block * ((input_items + items_per_block - 1) / items_per_block); + const auto bin_size = BinSize * ((input_items + items_per_block - 1) / items_per_block); + + // Allocate and fill memory + std::vector input(items, 0.0f); + T* d_input; + T* d_output; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK(hipMalloc(&d_output, bin_size * sizeof(T))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); + state.set_items(Trials * items); + state.add_writes(Trials * items); - state.SetIterationTime(elapsed_seconds.count()); + state.run( + [&] + { + hipLaunchKernelGGL( + HIP_KERNEL_NAME(kernel), + dim3(items / items_per_block), + dim3(BlockSize), + 0, + stream, + d_input, + d_output); + }); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); } - state.SetBytesProcessed(state.iterations() * size * sizeof(T) * Trials); - state.SetItemsProcessed(state.iterations() * size * Trials); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); -} +}; -// IPT - items per thread -#define CREATE_BENCHMARK(T, BS, IPT) \ - benchmark::RegisterBenchmark(std::string("block_histogram.method_name:" + method_name) \ - .c_str(), \ - &run_benchmark, \ - stream, \ - size) +#define CREATE_BENCHMARK(T, BS, IPT) \ + executor.queue>() #define BENCHMARK_TYPE(type, block) \ CREATE_BENCHMARK(type, block, 1), CREATE_BENCHMARK(type, block, 2), \ @@ -146,70 +167,26 @@ void run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) CREATE_BENCHMARK(type, block, 8), CREATE_BENCHMARK(type, block, 16) template -void add_benchmarks(std::vector& benchmarks, - const std::string& method_name, - const std::string& algorithm_name, - hipStream_t stream, - size_t size) +void add_benchmarks(primbench::executor& executor) { - std::vector new_benchmarks - = {BENCHMARK_TYPE(int, 256), - BENCHMARK_TYPE(int, 320), - BENCHMARK_TYPE(int, 512), - - BENCHMARK_TYPE(unsigned long long, 256), - BENCHMARK_TYPE(unsigned long long, 320)}; - benchmarks.insert(benchmarks.end(), new_benchmarks.begin(), new_benchmarks.end()); + BENCHMARK_TYPE(int, 256); + BENCHMARK_TYPE(int, 320); + BENCHMARK_TYPE(int, 512); + + BENCHMARK_TYPE(unsigned long long, 256); + BENCHMARK_TYPE(unsigned long long, 320); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_block_histogram" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - // using_atomic - using histogram_a_t = histogram; - add_benchmarks(benchmarks, "histogram", "using_atomic", stream, size); - // using_sort - using histogram_s_t = histogram; - add_benchmarks(benchmarks, "histogram", "using_sort", stream, size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; + settings.size = 128 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + primbench::executor executor(argc, argv, settings); + + add_benchmarks(executor); + add_benchmarks(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_block_merge_sort.cpp b/projects/hipcub/benchmark/benchmark_block_merge_sort.cpp index c8c7402b1366..c2939e540d29 100644 --- a/projects/hipcub/benchmark/benchmark_block_merge_sort.cpp +++ b/projects/hipcub/benchmark/benchmark_block_merge_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,17 +20,15 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" #include "../test/hipcub/test_utils_sort_comparator.hpp" -// HIP API + #include #include #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 128; -#endif +constexpr unsigned int Trials = 10; enum class benchmark_kinds { @@ -38,11 +36,7 @@ enum class benchmark_kinds sort_pairs }; -template +template __global__ __launch_bounds__(BlockSize) void sort_keys_kernel(const T* input, T* output, CompareOp compare_op) { @@ -52,7 +46,7 @@ void sort_keys_kernel(const T* input, T* output, CompareOp compare_op) T keys[ItemsPerThread]; hipcub::LoadDirectStriped(lid, input + block_offset, keys); -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockMergeSort sort; @@ -62,11 +56,7 @@ void sort_keys_kernel(const T* input, T* output, CompareOp compare_op) hipcub::StoreDirectStriped(lid, output + block_offset, keys); } -template +template __global__ __launch_bounds__(BlockSize) void sort_pairs_kernel(const T* input, T* output, CompareOp compare_op) { @@ -82,7 +72,7 @@ void sort_pairs_kernel(const T* input, T* output, CompareOp compare_op) values[i] = keys[i] + T(1); } -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { hipcub::BlockMergeSort sort; @@ -96,162 +86,134 @@ void sort_pairs_kernel(const T* input, T* output, CompareOp compare_op) hipcub::StoreDirectStriped(lid, output + block_offset, keys); } -template -void run_benchmark(benchmark::State& state, - benchmark_kinds benchmark_kind, - hipStream_t stream, - size_t N) + class CompareOp = test_utils::less> +class merge_sort_benchmark : public primbench::benchmark_interface { - constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto size = items_per_block * ((N + items_per_block - 1) / items_per_block); - - std::vector input - = benchmark_utils::get_random_data(size, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); - - T* d_input; - T* d_output; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMalloc(&d_output, size * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - for(auto _ : state) + primbench::json meta() const override { - auto start = std::chrono::high_resolution_clock::now(); - - if(benchmark_kind == benchmark_kinds::sort_keys) - { - hipLaunchKernelGGL( - HIP_KERNEL_NAME(sort_keys_kernel), - dim3(size / items_per_block), - dim3(BlockSize), - 0, - stream, - d_input, - d_output, - CompareOp()); - } - else if(benchmark_kind == benchmark_kinds::sort_pairs) - { - hipLaunchKernelGGL( - HIP_KERNEL_NAME(sort_pairs_kernel), - dim3(size / items_per_block), - dim3(BlockSize), - 0, - stream, - d_input, - d_output, - CompareOp()); - } - HIP_CHECK(hipPeekAtLastError()); + return primbench::json{} + .add("algo", "block_merge_sort") + .add("subalgo", get_algorithm_name(BenchmarkKind)) + .add("data_type", primbench::name()) + .add("block_size", BlockSize) + .add("items_per_thread", ItemsPerThread); + } + + void run(primbench::state& state) override + { + const size_t input_items = state.size; + const auto& stream = state.stream; + + constexpr auto items_per_block = BlockSize * ItemsPerThread; + const auto items + = items_per_block * ((input_items + items_per_block - 1) / items_per_block); + + std::vector input + = benchmark_utils::get_random_data(items, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); + + T* d_input; + T* d_output; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK(hipMalloc(&d_output, items * sizeof(T))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + state.set_items(Trials * items); + state.add_writes(Trials * items); + + state.run( + [&] + { + if constexpr(BenchmarkKind == benchmark_kinds::sort_keys) + { + hipLaunchKernelGGL( + HIP_KERNEL_NAME(sort_keys_kernel), + dim3(items / items_per_block), + dim3(BlockSize), + 0, + stream, + d_input, + d_output, + CompareOp()); + } + else if constexpr(BenchmarkKind == benchmark_kinds::sort_pairs) + { + hipLaunchKernelGGL( + HIP_KERNEL_NAME(sort_pairs_kernel), + dim3(items / items_per_block), + dim3(BlockSize), + 0, + stream, + d_input, + d_output, + CompareOp()); + } + }); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); } - state.SetBytesProcessed(state.iterations() * Trials * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * Trials * size); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); -} +}; -#define CREATE_BENCHMARK(T, BS, IPT) \ - benchmark::RegisterBenchmark(std::string("block_merge_sort.sub_algorithm_name:" \ - + name) \ - .c_str(), \ - &run_benchmark, \ - benchmark_kind, \ - stream, \ - size) +#define CREATE_BENCHMARK(T, BS, IPT) \ + executor.queue>() #define BENCHMARK_TYPE(type, block) \ CREATE_BENCHMARK(type, block, 1), CREATE_BENCHMARK(type, block, 2), \ CREATE_BENCHMARK(type, block, 3), CREATE_BENCHMARK(type, block, 4), \ CREATE_BENCHMARK(type, block, 8) -void add_benchmarks(benchmark_kinds benchmark_kind, - const std::string& name, - std::vector& benchmarks, - hipStream_t stream, - size_t size) +template +void add_benchmarks(primbench::executor& executor) { - std::vector bs = {BENCHMARK_TYPE(int, 64), - BENCHMARK_TYPE(int, 128), - BENCHMARK_TYPE(int, 256), - BENCHMARK_TYPE(int, 512), - - BENCHMARK_TYPE(int8_t, 64), - BENCHMARK_TYPE(int8_t, 128), - BENCHMARK_TYPE(int8_t, 256), - BENCHMARK_TYPE(int8_t, 512), - - BENCHMARK_TYPE(uint8_t, 64), - BENCHMARK_TYPE(uint8_t, 128), - BENCHMARK_TYPE(uint8_t, 256), - BENCHMARK_TYPE(uint8_t, 512), - - BENCHMARK_TYPE(long long, 64), - BENCHMARK_TYPE(long long, 128), - BENCHMARK_TYPE(long long, 256), - BENCHMARK_TYPE(long long, 512)}; - - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + BENCHMARK_TYPE(int, 64); + BENCHMARK_TYPE(int, 128); + BENCHMARK_TYPE(int, 256); + BENCHMARK_TYPE(int, 512); + + BENCHMARK_TYPE(int8_t, 64); + BENCHMARK_TYPE(int8_t, 128); + BENCHMARK_TYPE(int8_t, 256); + BENCHMARK_TYPE(int8_t, 512); + + BENCHMARK_TYPE(uint8_t, 64); + BENCHMARK_TYPE(uint8_t, 128); + BENCHMARK_TYPE(uint8_t, 256); + BENCHMARK_TYPE(uint8_t, 512); + + BENCHMARK_TYPE(int64_t, 64); + BENCHMARK_TYPE(int64_t, 128); + BENCHMARK_TYPE(int64_t, 256); + BENCHMARK_TYPE(int64_t, 512); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_block_merge_sort" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - add_benchmarks(benchmark_kinds::sort_keys, "sort(keys)", benchmarks, stream, size); - add_benchmarks(benchmark_kinds::sort_pairs, "sort(keys, values)", benchmarks, stream, size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; + settings.size = 128 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + primbench::executor executor(argc, argv, settings); + + add_benchmarks(executor); + add_benchmarks(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_block_radix_rank.cpp b/projects/hipcub/benchmark/benchmark_block_radix_rank.cpp index ffcd1d775055..07c312fc4b94 100644 --- a/projects/hipcub/benchmark/benchmark_block_radix_rank.cpp +++ b/projects/hipcub/benchmark/benchmark_block_radix_rank.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,18 +20,15 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include #include #include #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 128; -#endif +constexpr unsigned int Trials = 10; enum class RadixRankAlgorithm { @@ -45,9 +42,9 @@ template -__global__ __launch_bounds__(BlockSize) void rank_kernel(const T* keys_input, int* ranks_output) + unsigned int ItemsPerThread> +__global__ __launch_bounds__(BlockSize) +void rank_kernel(const T* keys_input, int* ranks_output) { const unsigned int lid = hipThreadIdx_x; const unsigned int block_offset = hipBlockIdx_x * ItemsPerThread * BlockSize; @@ -70,7 +67,7 @@ __global__ __launch_bounds__(BlockSize) void rank_kernel(const T* keys_input, in Descending, BenchmarkKind == RadixRankAlgorithm::RADIX_RANK_MEMOIZE>>; -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(unsigned int key = 0; key < ItemsPerThread; key++) { unsigned_keys[key] = KeyTraits::TwiddleIn(unsigned_keys[key]); @@ -78,7 +75,7 @@ __global__ __launch_bounds__(BlockSize) void rank_kernel(const T* keys_input, in int ranks[ItemsPerThread]; -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { __shared__ typename RankType::TempStorage storage; @@ -99,144 +96,115 @@ __global__ __launch_bounds__(BlockSize) void rank_kernel(const T* keys_input, in hipcub::StoreDirectBlocked(lid, ranks_output + block_offset, ranks); } +inline const char* get_algorithm_name(RadixRankAlgorithm algorithm) +{ + switch(algorithm) + { + case RadixRankAlgorithm::RADIX_RANK_BASIC: return "basic"; + case RadixRankAlgorithm::RADIX_RANK_MATCH: return "match"; + case RadixRankAlgorithm::RADIX_RANK_MEMOIZE: return "memoize"; + } + + return "unknown algorithm"; +} + template -void run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) + unsigned int ItemsPerThread> +class block_radix_rank_benchmark : public primbench::benchmark_interface { - constexpr unsigned int items_per_block = BlockSize * ItemsPerThread; - const unsigned int size = items_per_block * ((N + items_per_block - 1) / items_per_block); - - std::vector input - = benchmark_utils::get_random_data(size, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); - - T* d_input; - int* d_output; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMalloc(&d_output, size * sizeof(int))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - for(auto _ : state) + primbench::json meta() const override { - auto start = std::chrono::high_resolution_clock::now(); - - hipLaunchKernelGGL( - HIP_KERNEL_NAME( - rank_kernel), - dim3(size / items_per_block), - dim3(BlockSize), - 0, - stream, - d_input, - d_output); - HIP_CHECK(hipPeekAtLastError()); + return primbench::json{} + .add("algo", "block_radix_rank") + .add("subalgo", get_algorithm_name(BenchmarkKind)) + .add("lvl", "block") + .add("data_type", primbench::name()) + .add("block_size", BlockSize) + .add("items_per_thread", ItemsPerThread); + } + + void run(primbench::state& state) override + { + const size_t input_items = state.size; + const auto& stream = state.stream; + + constexpr unsigned int items_per_block = BlockSize * ItemsPerThread; + const size_t items + = items_per_block * ((input_items + items_per_block - 1) / items_per_block); + + std::vector input + = benchmark_utils::get_random_data(items, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); + + T* d_input; + int* d_output; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK(hipMalloc(&d_output, items * sizeof(int))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + state.set_items(Trials * items); + state.add_writes(Trials * items); + + state.run( + [&] + { + hipLaunchKernelGGL( + HIP_KERNEL_NAME( + rank_kernel), + dim3(items / items_per_block), + dim3(BlockSize), + 0, + stream, + d_input, + d_output); + }); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); } - state.SetBytesProcessed(state.iterations() * Trials * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * Trials * size); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); -} +}; -#define CREATE_BENCHMARK(T, KIND, BS, IPT) \ - benchmark::RegisterBenchmark(std::string("block_radix_rank." \ - + name) \ - .c_str(), \ - &run_benchmark, \ - stream, \ - size) +#define CREATE_BENCHMARK(T, KIND, BS, IPT) \ + executor.queue>() -// clang-format off #define CREATE_BENCHMARK_KINDS(type, block, ipt) \ CREATE_BENCHMARK(type, RadixRankAlgorithm::RADIX_RANK_BASIC, block, ipt), \ CREATE_BENCHMARK(type, RadixRankAlgorithm::RADIX_RANK_MEMOIZE, block, ipt), \ CREATE_BENCHMARK(type, RadixRankAlgorithm::RADIX_RANK_MATCH, block, ipt) -#define BENCHMARK_TYPE(type, block) \ - CREATE_BENCHMARK_KINDS(type, block, 1), \ - CREATE_BENCHMARK_KINDS(type, block, 4), \ - CREATE_BENCHMARK_KINDS(type, block, 8), \ - CREATE_BENCHMARK_KINDS(type, block, 16), \ - CREATE_BENCHMARK_KINDS(type, block, 32) -// clang-format on - -void add_benchmarks(const std::string& name, - std::vector& benchmarks, - hipStream_t stream, - size_t size) -{ - std::vector bs = { - BENCHMARK_TYPE(int, 128), - BENCHMARK_TYPE(int, 256), - BENCHMARK_TYPE(int, 512), +#define BENCHMARK_TYPE(type, block) \ + CREATE_BENCHMARK_KINDS(type, block, 1), CREATE_BENCHMARK_KINDS(type, block, 4), \ + CREATE_BENCHMARK_KINDS(type, block, 8), CREATE_BENCHMARK_KINDS(type, block, 16), \ + CREATE_BENCHMARK_KINDS(type, block, 32) - BENCHMARK_TYPE(uint8_t, 128), - BENCHMARK_TYPE(uint8_t, 256), - BENCHMARK_TYPE(uint8_t, 512), +void add_benchmarks(primbench::executor& executor) +{ + BENCHMARK_TYPE(int, 128); + BENCHMARK_TYPE(int, 256); + BENCHMARK_TYPE(int, 512); - BENCHMARK_TYPE(long long, 128), - BENCHMARK_TYPE(long long, 256), - BENCHMARK_TYPE(long long, 512), - }; + BENCHMARK_TYPE(uint8_t, 128); + BENCHMARK_TYPE(uint8_t, 256); + BENCHMARK_TYPE(uint8_t, 512); - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + BENCHMARK_TYPE(int64_t, 128); + BENCHMARK_TYPE(int64_t, 256); + BENCHMARK_TYPE(int64_t, 512); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - - std::cout << "benchmark_block_radix_rank" << std::endl; - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - add_benchmarks("rank", benchmarks, stream, size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; + settings.size = 128 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + primbench::executor executor(argc, argv, settings); + + add_benchmarks(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_block_radix_sort.cpp b/projects/hipcub/benchmark/benchmark_block_radix_sort.cpp index 4b75c26910a4..7d6227737aa4 100644 --- a/projects/hipcub/benchmark/benchmark_block_radix_sort.cpp +++ b/projects/hipcub/benchmark/benchmark_block_radix_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,16 +20,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include #include #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 128; -#endif +constexpr unsigned int Trials = 10; enum class benchmark_kinds { @@ -39,29 +36,43 @@ enum class benchmark_kinds struct helper_blocked_blocked { + static const char* get_algorithm_name(benchmark_kinds algorithm) + { + switch(algorithm) + { + case benchmark_kinds::sort_keys: return "sort(keys)"; + case benchmark_kinds::sort_pairs: return "sort(keys, values)"; + } + + return "unknown algorithm"; + } + template - HIPCUB_DEVICE static void - load(int linear_id, InputIteratorT block_iter, T (&items)[ItemsPerThread]) + HIPCUB_DEVICE + static void load(int linear_id, InputIteratorT block_iter, T (&items)[ItemsPerThread]) { hipcub::LoadDirectStriped(linear_id, block_iter, items); } template - HIPCUB_DEVICE static void sort(T (&keys)[ItemsPerThread]) + HIPCUB_DEVICE + static void sort(T (&keys)[ItemsPerThread]) { hipcub::BlockRadixSort sort; sort.Sort(keys); } template - HIPCUB_DEVICE static void sort(T (&keys)[ItemsPerThread], T (&values)[ItemsPerThread]) + HIPCUB_DEVICE + static void sort(T (&keys)[ItemsPerThread], T (&values)[ItemsPerThread]) { hipcub::BlockRadixSort sort; sort.Sort(keys, values); } template - HIPCUB_DEVICE static void sort(benchmark_utils::custom_type (&keys)[ItemsPerThread]) + HIPCUB_DEVICE + static void sort(benchmark_utils::custom_type (&keys)[ItemsPerThread]) { using custom_t = benchmark_utils::custom_type; hipcub::BlockRadixSort sort; @@ -69,8 +80,9 @@ struct helper_blocked_blocked } template - HIPCUB_DEVICE static void sort(benchmark_utils::custom_type (&keys)[ItemsPerThread], - benchmark_utils::custom_type (&values)[ItemsPerThread]) + HIPCUB_DEVICE + static void sort(benchmark_utils::custom_type (&keys)[ItemsPerThread], + benchmark_utils::custom_type (&values)[ItemsPerThread]) { using custom_t = benchmark_utils::custom_type; hipcub::BlockRadixSort sort; @@ -80,29 +92,43 @@ struct helper_blocked_blocked struct helper_blocked_striped { + static const char* get_algorithm_name(benchmark_kinds algorithm) + { + switch(algorithm) + { + case benchmark_kinds::sort_keys: return "sort_to_striped(keys)"; + case benchmark_kinds::sort_pairs: return "sort_to_striped(keys, values)"; + } + + return "unknown algorithm"; + } + template - HIPCUB_DEVICE static void - load(int linear_id, InputIteratorT block_iter, T (&items)[ItemsPerThread]) + HIPCUB_DEVICE + static void load(int linear_id, InputIteratorT block_iter, T (&items)[ItemsPerThread]) { hipcub::LoadDirectBlocked(linear_id, block_iter, items); } template - HIPCUB_DEVICE static void sort(T (&keys)[ItemsPerThread]) + HIPCUB_DEVICE + static void sort(T (&keys)[ItemsPerThread]) { hipcub::BlockRadixSort sort; sort.SortBlockedToStriped(keys); } template - HIPCUB_DEVICE static void sort(T (&keys)[ItemsPerThread], T (&values)[ItemsPerThread]) + HIPCUB_DEVICE + static void sort(T (&keys)[ItemsPerThread], T (&values)[ItemsPerThread]) { hipcub::BlockRadixSort sort; sort.SortBlockedToStriped(keys, values); } template - HIPCUB_DEVICE static void sort(benchmark_utils::custom_type (&keys)[ItemsPerThread]) + HIPCUB_DEVICE + static void sort(benchmark_utils::custom_type (&keys)[ItemsPerThread]) { using custom_t = benchmark_utils::custom_type; hipcub::BlockRadixSort sort; @@ -110,8 +136,9 @@ struct helper_blocked_striped } template - HIPCUB_DEVICE static void sort(benchmark_utils::custom_type (&keys)[ItemsPerThread], - benchmark_utils::custom_type (&values)[ItemsPerThread]) + HIPCUB_DEVICE + static void sort(benchmark_utils::custom_type (&keys)[ItemsPerThread], + benchmark_utils::custom_type (&values)[ItemsPerThread]) { using custom_t = benchmark_utils::custom_type; hipcub::BlockRadixSort sort; @@ -121,12 +148,9 @@ struct helper_blocked_striped } }; -template -__global__ __launch_bounds__(BlockSize) void sort_keys_kernel(const T* input, T* output) +template +__global__ __launch_bounds__(BlockSize) +void sort_keys_kernel(const T* input, T* output) { const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; @@ -134,7 +158,7 @@ __global__ __launch_bounds__(BlockSize) void sort_keys_kernel(const T* input, T* T keys[ItemsPerThread]; Helper::template load(lid, input + block_offset, keys); -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { Helper::template sort(keys); @@ -143,12 +167,9 @@ __global__ __launch_bounds__(BlockSize) void sort_keys_kernel(const T* input, T* hipcub::StoreDirectStriped(lid, output + block_offset, keys); } -template -__global__ __launch_bounds__(BlockSize) void sort_pairs_kernel(const T* input, T* output) +template +__global__ __launch_bounds__(BlockSize) +void sort_pairs_kernel(const T* input, T* output) { const unsigned int lid = threadIdx.x; const unsigned int block_offset = blockIdx.x * ItemsPerThread * BlockSize; @@ -162,7 +183,7 @@ __global__ __launch_bounds__(BlockSize) void sort_pairs_kernel(const T* input, T values[i] = keys[i] + T(1); } -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { Helper::template sort(keys, values); @@ -178,157 +199,119 @@ __global__ __launch_bounds__(BlockSize) void sort_pairs_kernel(const T* input, T template -void run_benchmark(benchmark::State& state, - benchmark_kinds benchmark_kind, - hipStream_t stream, - size_t N) + unsigned int BlockSize, + unsigned int ItemsPerThread, + benchmark_kinds BenchmarkKind> +class block_radix_sort_benchmark : public primbench::benchmark_interface { - constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto size = items_per_block * ((N + items_per_block - 1) / items_per_block); - - std::vector input - = benchmark_utils::get_random_data(size, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); - - T* d_input; - T* d_output; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMalloc(&d_output, size * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - for(auto _ : state) + primbench::json meta() const override { - auto start = std::chrono::high_resolution_clock::now(); + return primbench::json{} + .add("algo", "block_radix_sort") + .add("subalgo", Helper::get_algorithm_name(BenchmarkKind)) + .add("lvl", "block") + .add("data_type", primbench::name()) + .add("block_size", BlockSize) + .add("items_per_thread", ItemsPerThread); + } - if(benchmark_kind == benchmark_kinds::sort_keys) - { - sort_keys_kernel - <<>>(d_input, d_output); - } else if(benchmark_kind == benchmark_kinds::sort_pairs) - { - sort_pairs_kernel - <<>>(d_input, d_output); - } - HIP_CHECK(hipPeekAtLastError()); + void run(primbench::state& state) override + { + const size_t input_items = state.size; + const auto& stream = state.stream; + + constexpr auto items_per_block = BlockSize * ItemsPerThread; + const size_t items + = items_per_block * ((input_items + items_per_block - 1) / items_per_block); + + std::vector input + = benchmark_utils::get_random_data(items, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); + + T* d_input; + T* d_output; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK(hipMalloc(&d_output, items * sizeof(T))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + state.set_items(Trials * items); + state.add_writes(Trials * items); + + state.run( + [&] + { + if constexpr(BenchmarkKind == benchmark_kinds::sort_keys) + { + sort_keys_kernel + <<>>(d_input, + d_output); + } + else if(BenchmarkKind == benchmark_kinds::sort_pairs) + { + sort_pairs_kernel + <<>>(d_input, + d_output); + } + }); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); } - state.SetBytesProcessed(state.iterations() * Trials * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * Trials * size); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); -} - -#define CREATE_BENCHMARK(T, BS, IPT) \ - benchmark::RegisterBenchmark(std::string("block_radix_sort.sub_algorithm_name:" \ - + name) \ - .c_str(), \ - &run_benchmark, \ - benchmark_kind, \ - stream, \ - size) - -// clang-format off -#define BENCHMARK_TYPE(type, block) \ - CREATE_BENCHMARK(type, block, 1), \ - CREATE_BENCHMARK(type, block, 3), \ - CREATE_BENCHMARK(type, block, 4), \ - CREATE_BENCHMARK(type, block, 8) -// clang-format on - -template -void add_benchmarks(benchmark_kinds benchmark_kind, - const std::string& name, - std::vector& benchmarks, - hipStream_t stream, - size_t size) -{ - using custom_int_t = benchmark_utils::custom_type; - - std::vector bs = { - BENCHMARK_TYPE(int, 64), BENCHMARK_TYPE(int, 128), - BENCHMARK_TYPE(int, 192), BENCHMARK_TYPE(int, 256), - BENCHMARK_TYPE(int, 320), BENCHMARK_TYPE(int, 512), - - BENCHMARK_TYPE(int8_t, 64), BENCHMARK_TYPE(int8_t, 128), - BENCHMARK_TYPE(int8_t, 192), BENCHMARK_TYPE(int8_t, 256), - BENCHMARK_TYPE(int8_t, 320), BENCHMARK_TYPE(int8_t, 512), +}; - BENCHMARK_TYPE(long long, 64), BENCHMARK_TYPE(long long, 128), - BENCHMARK_TYPE(long long, 192), BENCHMARK_TYPE(long long, 256), - BENCHMARK_TYPE(long long, 320), BENCHMARK_TYPE(long long, 512), +#define CREATE_BENCHMARK(T, BS, IPT) \ + executor.queue>() - BENCHMARK_TYPE(custom_int_t, 64), BENCHMARK_TYPE(custom_int_t, 128), - BENCHMARK_TYPE(custom_int_t, 192), BENCHMARK_TYPE(custom_int_t, 256), - BENCHMARK_TYPE(custom_int_t, 320), BENCHMARK_TYPE(custom_int_t, 512), - }; +#define BENCHMARK_TYPE(type, block) \ + CREATE_BENCHMARK(type, block, 1), CREATE_BENCHMARK(type, block, 3), \ + CREATE_BENCHMARK(type, block, 4), CREATE_BENCHMARK(type, block, 8) - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); +template +void add_benchmarks(primbench::executor& executor) +{ + BENCHMARK_TYPE(int, 64); + BENCHMARK_TYPE(int, 128); + BENCHMARK_TYPE(int, 192); + BENCHMARK_TYPE(int, 256); + BENCHMARK_TYPE(int, 320); + BENCHMARK_TYPE(int, 512); + + BENCHMARK_TYPE(int8_t, 64); + BENCHMARK_TYPE(int8_t, 128); + BENCHMARK_TYPE(int8_t, 192); + BENCHMARK_TYPE(int8_t, 256); + BENCHMARK_TYPE(int8_t, 320); + BENCHMARK_TYPE(int8_t, 512); + + BENCHMARK_TYPE(int64_t, 64); + BENCHMARK_TYPE(int64_t, 128); + BENCHMARK_TYPE(int64_t, 192); + BENCHMARK_TYPE(int64_t, 256); + BENCHMARK_TYPE(int64_t, 320); + BENCHMARK_TYPE(int64_t, 512); + + BENCHMARK_TYPE(custom_int_t, 64); + BENCHMARK_TYPE(custom_int_t, 128); + BENCHMARK_TYPE(custom_int_t, 192); + BENCHMARK_TYPE(custom_int_t, 256); + BENCHMARK_TYPE(custom_int_t, 320); + BENCHMARK_TYPE(custom_int_t, 512); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_block_radix_sort" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - // clang-format off - add_benchmarks( - benchmark_kinds::sort_keys, "sort(keys)", benchmarks, stream, size); - add_benchmarks( - benchmark_kinds::sort_pairs, "sort(keys, values)", benchmarks, stream, size); - add_benchmarks( - benchmark_kinds::sort_keys, "sort_to_striped(keys)", benchmarks, stream, size); - add_benchmarks( - benchmark_kinds::sort_pairs, "sort_to_striped(keys, values)", benchmarks, stream, size); - // clang-format on - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; + settings.size = 128 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + primbench::executor executor(argc, argv, settings); + + add_benchmarks(executor); + add_benchmarks(executor); + add_benchmarks(executor); + add_benchmarks(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_block_reduce.cpp b/projects/hipcub/benchmark/benchmark_block_reduce.cpp index fe4b815d50c1..23e4751d5787 100644 --- a/projects/hipcub/benchmark/benchmark_block_reduce.cpp +++ b/projects/hipcub/benchmark/benchmark_block_reduce.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,31 +20,40 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif +constexpr unsigned int Trials = 100; -template -__global__ __launch_bounds__(BlockSize) void kernel(const T* input, T* output) +template +__global__ __launch_bounds__(BlockSize) +void kernel(const T* input, T* output) { - Runner::template run(input, output); + Runner::template run(input, output); } template struct reduce { - template - __device__ static void run(const T* input, T* output) + static const char* get_algorithm_name() + { + switch(algorithm) + { + case hipcub::BlockReduceAlgorithm::BLOCK_REDUCE_RAKING: return "block_reduce_raking"; + case hipcub::BlockReduceAlgorithm::BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY: + return "block_reduce_raking_commutative_only"; + case hipcub::BlockReduceAlgorithm::BLOCK_REDUCE_WARP_REDUCTIONS: + return "block_reduce_warp_reductions"; + } + + return "unknown algorithm"; + } + + template + __device__ + static void run(const T* input, T* output) { const unsigned int i = hipBlockIdx_x * hipBlockDim_x + hipThreadIdx_x; @@ -58,10 +67,10 @@ struct reduce using breduce_t = hipcub::BlockReduce; __shared__ typename breduce_t::TempStorage storage; -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { - reduced_value = breduce_t(storage).Reduce(values, hipcub::Sum()); + reduced_value = breduce_t(storage).Reduce(values, benchmark_utils::plus{}); values[0] = reduced_value; } @@ -72,60 +81,60 @@ struct reduce } }; -template -void run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) +template +class block_reduce_benchmark : public primbench::benchmark_interface { - // Make sure size is a multiple of BlockSize - constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto size = items_per_block * ((N + items_per_block - 1) / items_per_block); - // Allocate and fill memory - std::vector input(size, T(1)); - T* d_input; - T* d_output; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMalloc(&d_output, size * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - for(auto _ : state) + primbench::json meta() const override { - auto start = std::chrono::high_resolution_clock::now(); - hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), - dim3(size / items_per_block), - dim3(BlockSize), - 0, - stream, - d_input, - d_output); - HIP_CHECK(hipPeekAtLastError()); - HIP_CHECK(hipDeviceSynchronize()); + return primbench::json{} + .add("algo", "block_reduce") + .add("subalgo", Benchmark::get_algorithm_name()) + .add("lvl", "block") + .add("data_type", primbench::name()) + .add("block_size", BlockSize) + .add("items_per_thread", ItemsPerThread); + } - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); + void run(primbench::state& state) override + { + const size_t input_items = state.size; + const auto& stream = state.stream; + + // Make sure size is a multiple of BlockSize + constexpr auto items_per_block = BlockSize * ItemsPerThread; + const auto items + = items_per_block * ((input_items + items_per_block - 1) / items_per_block); + + // Allocate and fill memory + std::vector input(items, T(1)); + T* d_input; + T* d_output; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK(hipMalloc(&d_output, items * sizeof(T))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK(hipDeviceSynchronize()); - state.SetIterationTime(elapsed_seconds.count()); + state.set_items(Trials * items); + state.add_writes(Trials * items); + + state.run( + [&] + { + hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), + dim3(items / items_per_block), + dim3(BlockSize), + 0, + stream, + d_input, + d_output); + }); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); } - state.SetBytesProcessed(state.iterations() * size * sizeof(T) * Trials); - state.SetItemsProcessed(state.iterations() * size * Trials); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); -} +}; -// IPT - items per thread -#define CREATE_BENCHMARK(T, BS, IPT) \ - benchmark::RegisterBenchmark(std::string("block_reduce.method_name:" + method_name) \ - .c_str(), \ - &run_benchmark, \ - stream, \ - size) +#define CREATE_BENCHMARK(T, BS, IPT) executor.queue>() #define BENCHMARK_TYPE(type, block) \ CREATE_BENCHMARK(type, block, 1), CREATE_BENCHMARK(type, block, 2), \ @@ -134,90 +143,42 @@ void run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) CREATE_BENCHMARK(type, block, 16) template -void add_benchmarks(std::vector& benchmarks, - const std::string& method_name, - const std::string& algorithm_name, - hipStream_t stream, - size_t size) +void add_benchmarks(primbench::executor& executor) { - - std::vector new_benchmarks = { - // When block size is less than or equal to warp size - BENCHMARK_TYPE(int, 64), - BENCHMARK_TYPE(float, 64), - BENCHMARK_TYPE(double, 64), - BENCHMARK_TYPE(int8_t, 64), - BENCHMARK_TYPE(uint8_t, 64), - - BENCHMARK_TYPE(int, 256), - BENCHMARK_TYPE(float, 256), - BENCHMARK_TYPE(double, 256), - BENCHMARK_TYPE(int8_t, 256), - BENCHMARK_TYPE(uint8_t, 256), - }; - benchmarks.insert(benchmarks.end(), new_benchmarks.begin(), new_benchmarks.end()); + BENCHMARK_TYPE(int, 64); + BENCHMARK_TYPE(float, 64); + BENCHMARK_TYPE(double, 64); + BENCHMARK_TYPE(int8_t, 64); + BENCHMARK_TYPE(uint8_t, 64); + + BENCHMARK_TYPE(int, 256); + BENCHMARK_TYPE(float, 256); + BENCHMARK_TYPE(double, 256); + BENCHMARK_TYPE(int8_t, 256); + BENCHMARK_TYPE(uint8_t, 256); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_block_reduce" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; + settings.noise_tolerance_percent = 2; + + primbench::executor executor(argc, argv, settings); + // using_warp_scan using reduce_uwr_t = reduce; - add_benchmarks(benchmarks, - "reduce", - "BLOCK_REDUCE_WARP_REDUCTIONS", - stream, - size); + add_benchmarks(executor); + // raking reduce using reduce_rr_t = reduce; - add_benchmarks(benchmarks, "reduce", "BLOCK_REDUCE_RAKING", stream, size); + add_benchmarks(executor); + // raking reduce commutative only using reduce_rrco_t = reduce; - add_benchmarks(benchmarks, - "reduce", - "BLOCK_REDUCE_RAKING_COMMUTATIVE_ONLY", - stream, - size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } - - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + add_benchmarks(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_block_run_length_decode.cpp b/projects/hipcub/benchmark/benchmark_block_run_length_decode.cpp index a42d3c480651..38146c3f70cf 100644 --- a/projects/hipcub/benchmark/benchmark_block_run_length_decode.cpp +++ b/projects/hipcub/benchmark/benchmark_block_run_length_decode.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,27 +20,24 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" #include #include #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif +constexpr unsigned int Trials = 100; template -__global__ - __launch_bounds__(BlockSize) void block_run_length_decode_kernel(const ItemT* d_run_items, - const OffsetT* d_run_offsets, - ItemT* d_decoded_items, - bool enable_store = false) + unsigned DecodedItemsPerThread> +__global__ __launch_bounds__(BlockSize) +void block_run_length_decode_kernel(const ItemT* d_run_items, + const OffsetT* d_run_offsets, + ItemT* d_decoded_items, + bool enable_store = false) { using BlockRunLengthDecodeT = hipcub::BlockRunLengthDecode; @@ -58,7 +55,7 @@ __global__ = d_run_offsets[(hipBlockIdx_x + 1) * BlockSize * RunsPerThread] - d_run_offsets[hipBlockIdx_x * BlockSize * RunsPerThread]; -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned i = 0; i < Trials; ++i) { OffsetT decoded_window_offset = 0; @@ -85,153 +82,120 @@ template -void run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) + unsigned DecodedItemsPerThread> +class block_run_length_decode_benchmark : public primbench::benchmark_interface { - constexpr auto runs_per_block = BlockSize * RunsPerThread; - const auto target_num_runs = 2 * N / (MinRunLength + MaxRunLength); - const auto num_runs - = runs_per_block * ((target_num_runs + runs_per_block - 1) / runs_per_block); - - std::vector run_items(num_runs); - std::vector run_offsets(num_runs + 1); - - std::default_random_engine prng(std::random_device{}()); - using ItemDistribution = std::conditional_t::value, - std::uniform_int_distribution, - std::uniform_real_distribution>; - ItemDistribution run_item_dist(0, 100); - std::uniform_int_distribution run_length_dist(MinRunLength, MaxRunLength); - - for(size_t i = 0; i < num_runs; ++i) + primbench::json meta() const override { - run_items[i] = run_item_dist(prng); + return primbench::json{} + .add("algo", "block_run_length_decode") + .add("lvl", "block") + .add("item_type", primbench::name()) + .add("offset_type", primbench::name()) + .add("min_run_length", MinRunLength) + .add("max_run_length", MaxRunLength) + .add("block_size", BlockSize) + .add("runs_per_thread", RunsPerThread) + .add("decoded_items_per_thread", DecodedItemsPerThread); } - for(size_t i = 1; i < num_runs + 1; ++i) - { - const OffsetT next_run_length = run_length_dist(prng); - run_offsets[i] = run_offsets[i - 1] + next_run_length; - } - const OffsetT output_length = run_offsets.back(); - - ItemT* d_run_items{}; - HIP_CHECK(hipMalloc(&d_run_items, run_items.size() * sizeof(ItemT))); - HIP_CHECK(hipMemcpy(d_run_items, - run_items.data(), - run_items.size() * sizeof(ItemT), - hipMemcpyHostToDevice)); - - OffsetT* d_run_offsets{}; - HIP_CHECK(hipMalloc(&d_run_offsets, run_offsets.size() * sizeof(OffsetT))); - HIP_CHECK(hipMemcpy(d_run_offsets, - run_offsets.data(), - run_offsets.size() * sizeof(OffsetT), - hipMemcpyHostToDevice)); - - ItemT* d_output{}; - HIP_CHECK(hipMalloc(&d_output, output_length * sizeof(ItemT))); - - for(auto _ : state) + + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); - hipLaunchKernelGGL(HIP_KERNEL_NAME(block_run_length_decode_kernel), - dim3(num_runs / runs_per_block), - dim3(BlockSize), - 0, - stream, - d_run_items, - d_run_offsets, - d_output); - HIP_CHECK(hipPeekAtLastError()); - HIP_CHECK(hipDeviceSynchronize()); - - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * output_length * sizeof(ItemT) * Trials); - state.SetItemsProcessed(state.iterations() * output_length * Trials); + const size_t input_items = state.size; + const auto& stream = state.stream; - HIP_CHECK(hipFree(d_run_items)); - HIP_CHECK(hipFree(d_run_offsets)); - HIP_CHECK(hipFree(d_output)); -} + constexpr auto runs_per_block = BlockSize * RunsPerThread; + const auto target_num_runs = 2 * input_items / (MinRunLength + MaxRunLength); + const auto num_runs + = runs_per_block * ((target_num_runs + runs_per_block - 1) / runs_per_block); -#define CREATE_BENCHMARK(IT, OT, MINRL, MAXRL, BS, RPT, DIPT) \ - benchmark::RegisterBenchmark( \ - std::string("block_run_length_decode.") \ - .c_str(), \ - &run_benchmark, \ - stream, \ - size) + std::vector run_items(num_runs); + std::vector run_offsets(num_runs + 1); -int main(int argc, char* argv[]) -{ - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_block_run_length_decode" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks{ - CREATE_BENCHMARK(int, int, 1, 5, 128, 2, 4), - CREATE_BENCHMARK(int, int, 1, 10, 128, 2, 4), - CREATE_BENCHMARK(int, int, 1, 50, 128, 2, 4), - CREATE_BENCHMARK(int, int, 1, 100, 128, 2, 4), - CREATE_BENCHMARK(int, int, 1, 500, 128, 2, 4), - CREATE_BENCHMARK(int, int, 1, 1000, 128, 2, 4), - CREATE_BENCHMARK(int, int, 1, 5000, 128, 2, 4), - - CREATE_BENCHMARK(double, long long, 1, 5, 128, 2, 4), - CREATE_BENCHMARK(double, long long, 1, 10, 128, 2, 4), - CREATE_BENCHMARK(double, long long, 1, 50, 128, 2, 4), - CREATE_BENCHMARK(double, long long, 1, 100, 128, 2, 4), - CREATE_BENCHMARK(double, long long, 1, 500, 128, 2, 4), - CREATE_BENCHMARK(double, long long, 1, 1000, 128, 2, 4), - CREATE_BENCHMARK(double, long long, 1, 5000, 128, 2, 4)}; - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + std::default_random_engine prng(std::random_device{}()); + using ItemDistribution = std::conditional_t::value, + std::uniform_int_distribution, + std::uniform_real_distribution>; + ItemDistribution run_item_dist(0, 100); + std::uniform_int_distribution run_length_dist(MinRunLength, MaxRunLength); - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) + for(size_t i = 0; i < num_runs; ++i) + { + run_items[i] = run_item_dist(prng); + } + for(size_t i = 1; i < num_runs + 1; ++i) { - b->Iterations(trials); + const OffsetT next_run_length = run_length_dist(prng); + run_offsets[i] = run_offsets[i - 1] + next_run_length; } + const OffsetT output_length = run_offsets.back(); + + ItemT* d_run_items{}; + HIP_CHECK(hipMalloc(&d_run_items, run_items.size() * sizeof(ItemT))); + HIP_CHECK(hipMemcpy(d_run_items, + run_items.data(), + run_items.size() * sizeof(ItemT), + hipMemcpyHostToDevice)); + + OffsetT* d_run_offsets{}; + HIP_CHECK(hipMalloc(&d_run_offsets, run_offsets.size() * sizeof(OffsetT))); + HIP_CHECK(hipMemcpy(d_run_offsets, + run_offsets.data(), + run_offsets.size() * sizeof(OffsetT), + hipMemcpyHostToDevice)); + + ItemT* d_output{}; + HIP_CHECK(hipMalloc(&d_output, output_length * sizeof(ItemT))); + + state.set_items(Trials * output_length); + state.add_writes(Trials * output_length); + + state.run( + [&] + { + hipLaunchKernelGGL( + HIP_KERNEL_NAME(block_run_length_decode_kernel), + dim3(num_runs / runs_per_block), + dim3(BlockSize), + 0, + stream, + d_run_items, + d_run_offsets, + d_output); + }); } +}; + +#define CREATE_BENCHMARK(IT, OT, MINRL, MAXRL, BS, RPT, DIPT) \ + executor.queue>() - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; +int main(int argc, char* argv[]) +{ + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; + + primbench::executor executor(argc, argv, settings); + + CREATE_BENCHMARK(int, int, 1, 5, 128, 2, 4); + CREATE_BENCHMARK(int, int, 1, 10, 128, 2, 4); + CREATE_BENCHMARK(int, int, 1, 50, 128, 2, 4); + CREATE_BENCHMARK(int, int, 1, 100, 128, 2, 4); + CREATE_BENCHMARK(int, int, 1, 500, 128, 2, 4); + CREATE_BENCHMARK(int, int, 1, 1000, 128, 2, 4); + CREATE_BENCHMARK(int, int, 1, 5000, 128, 2, 4); + + CREATE_BENCHMARK(double, int64_t, 1, 5, 128, 2, 4); + CREATE_BENCHMARK(double, int64_t, 1, 10, 128, 2, 4); + CREATE_BENCHMARK(double, int64_t, 1, 50, 128, 2, 4); + CREATE_BENCHMARK(double, int64_t, 1, 100, 128, 2, 4); + CREATE_BENCHMARK(double, int64_t, 1, 500, 128, 2, 4); + CREATE_BENCHMARK(double, int64_t, 1, 1000, 128, 2, 4); + CREATE_BENCHMARK(double, int64_t, 1, 5000, 128, 2, 4); + + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_block_scan.cpp b/projects/hipcub/benchmark/benchmark_block_scan.cpp index 51bf6c63fac7..603dea11a408 100644 --- a/projects/hipcub/benchmark/benchmark_block_scan.cpp +++ b/projects/hipcub/benchmark/benchmark_block_scan.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2022 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,30 +20,30 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// hipCUB API #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif +constexpr unsigned int Trials = 100; -template -__global__ __launch_bounds__(BlockSize) void kernel(const T* input, T* output, const T init) +template +__global__ __launch_bounds__(BlockSize) +void kernel(const T* input, T* output, const T init) { - Runner::template run(input, output, init); + Runner::template run(input, output, init); } -template +template struct inclusive_scan { - template - __device__ static void run(const T* input, T* output, const T init) + static const char* get_algorithm_name() + { + return Name::name; + } + + template + __device__ + static void run(const T* input, T* output, const T init) { (void)init; const unsigned int i = hipBlockIdx_x * hipBlockDim_x + hipThreadIdx_x; @@ -57,10 +57,10 @@ struct inclusive_scan using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage storage; -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { - bscan_t(storage).InclusiveScan(values, values, hipcub::Sum()); + bscan_t(storage).InclusiveScan(values, values, benchmark_utils::plus{}); } for(unsigned int k = 0; k < ItemsPerThread; k++) @@ -70,11 +70,17 @@ struct inclusive_scan } }; -template +template struct exclusive_scan { - template - __device__ static void run(const T* input, T* output, const T init) + static const char* get_algorithm_name() + { + return Name::name; + } + + template + __device__ + static void run(const T* input, T* output, const T init) { const unsigned int i = hipBlockIdx_x * hipBlockDim_x + hipThreadIdx_x; @@ -87,10 +93,10 @@ struct exclusive_scan using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage storage; -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { - bscan_t(storage).ExclusiveScan(values, values, init, hipcub::Sum()); + bscan_t(storage).ExclusiveScan(values, values, init, benchmark_utils::plus{}); } for(unsigned int k = 0; k < ItemsPerThread; k++) @@ -100,161 +106,146 @@ struct exclusive_scan } }; -template -void run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) +template +class block_scan_benchmark : public primbench::benchmark_interface { - // Make sure size is a multiple of BlockSize - constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto size = items_per_block * ((N + items_per_block - 1) / items_per_block); - // Allocate and fill memory - std::vector input(size, T(1)); - T* d_input; - T* d_output; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMalloc(&d_output, size * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - for(auto _ : state) + primbench::json meta() const override { - auto start = std::chrono::high_resolution_clock::now(); - hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), - dim3(size / items_per_block), - dim3(BlockSize), - 0, - stream, - d_input, - d_output, - input[0]); - HIP_CHECK(hipPeekAtLastError()); - HIP_CHECK(hipDeviceSynchronize()); + return primbench::json{} + .add("algo", "block_scan") + .add("subalgo", Benchmark::get_algorithm_name()) + .add("data_type", primbench::name()) + .add("block_size", BlockSize) + .add("items_per_thread", ItemsPerThread); + } - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); + void run(primbench::state& state) override + { + const size_t input_items = state.size; + const auto& stream = state.stream; + + constexpr auto items_per_block = BlockSize * ItemsPerThread; + const auto items + = items_per_block * ((input_items + items_per_block - 1) / items_per_block); + + std::vector input(items, T(1)); + T* d_input; + T* d_output; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK(hipMalloc(&d_output, items * sizeof(T))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK(hipDeviceSynchronize()); - state.SetIterationTime(elapsed_seconds.count()); + state.set_items(items * Trials); + state.add_writes(items * Trials); + + state.run( + [&] + { + hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), + dim3(items / items_per_block), + dim3(BlockSize), + 0, + stream, + d_input, + d_output, + input[0]); + }); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); } - state.SetBytesProcessed(state.iterations() * size * sizeof(T) * Trials); - state.SetItemsProcessed(state.iterations() * size * Trials); +}; - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); -} +#define CREATE_BENCHMARK(T, BS, IPT) executor.queue>() -// IPT - items per thread -#define CREATE_BENCHMARK(T, BS, IPT) \ - benchmark::RegisterBenchmark(std::string("block_scan.method_name:" + method_name) \ - .c_str(), \ - &run_benchmark, \ - stream, \ - size) - -// clang-format off #define BENCHMARK_TYPE(type, block) \ - CREATE_BENCHMARK(type, block, 1), \ - CREATE_BENCHMARK(type, block, 3), \ - CREATE_BENCHMARK(type, block, 4), \ - CREATE_BENCHMARK(type, block, 8), \ - CREATE_BENCHMARK(type, block, 11), \ + CREATE_BENCHMARK(type, block, 1); \ + CREATE_BENCHMARK(type, block, 3); \ + CREATE_BENCHMARK(type, block, 4); \ + CREATE_BENCHMARK(type, block, 8); \ + CREATE_BENCHMARK(type, block, 11); \ CREATE_BENCHMARK(type, block, 16) -// clang-format on template -void add_benchmarks(std::vector& benchmarks, - const std::string& method_name, - const std::string& algorithm_name, - hipStream_t stream, - size_t size) +void add_benchmarks(primbench::executor& executor) { - using custom_float2 = benchmark_utils::custom_type; - using custom_double2 = benchmark_utils::custom_type; - - std::vector new_benchmarks = { - // When block size is less than or equal to warp size - BENCHMARK_TYPE(int, 64), - BENCHMARK_TYPE(float, 64), - BENCHMARK_TYPE(double, 64), - BENCHMARK_TYPE(uint8_t, 64), - - BENCHMARK_TYPE(int, 256), - BENCHMARK_TYPE(float, 256), - BENCHMARK_TYPE(double, 256), - BENCHMARK_TYPE(uint8_t, 256), - - CREATE_BENCHMARK(custom_float2, 256, 1), - CREATE_BENCHMARK(custom_float2, 256, 4), - CREATE_BENCHMARK(custom_float2, 256, 8), - - CREATE_BENCHMARK(custom_double2, 256, 1), - CREATE_BENCHMARK(custom_double2, 256, 4), - CREATE_BENCHMARK(custom_double2, 256, 8), - }; - benchmarks.insert(benchmarks.end(), new_benchmarks.begin(), new_benchmarks.end()); + // When block size is less than or equal to warp size + BENCHMARK_TYPE(int, 64); + BENCHMARK_TYPE(float, 64); + BENCHMARK_TYPE(double, 64); + BENCHMARK_TYPE(uint8_t, 64); + + BENCHMARK_TYPE(int, 256); + BENCHMARK_TYPE(float, 256); + BENCHMARK_TYPE(double, 256); + BENCHMARK_TYPE(uint8_t, 256); + + CREATE_BENCHMARK(custom_float2, 256, 1); + CREATE_BENCHMARK(custom_float2, 256, 4); + CREATE_BENCHMARK(custom_float2, 256, 8); + + CREATE_BENCHMARK(custom_double2, 256, 1); + CREATE_BENCHMARK(custom_double2, 256, 4); + CREATE_BENCHMARK(custom_double2, 256, 8); } -int main(int argc, char* argv[]) +// At the time of writing, BLOCK_SCAN_RAKING and BLOCK_SCAN_RAKING_MEMOIZE have the same values, so we can't switch on them like a normal enum. +// So we have to pass the algorithm names here +struct inclusive_raking_tag { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_block_scan" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - // clang-format off - add_benchmarks>( - benchmarks, "inclusive_scan", "BLOCK_SCAN_RAKING", stream, size); - add_benchmarks>( - benchmarks, "inclusive_scan", "BLOCK_SCAN_RAKING_MEMOIZE", stream, size); - add_benchmarks>( - benchmarks, "inclusive_scan", "BLOCK_SCAN_WARP_SCANS", stream, size); - add_benchmarks>( - benchmarks, "exclusive_scan", "BLOCK_SCAN_RAKING", stream, size); - add_benchmarks>( - benchmarks, "exclusive_scan", "BLOCK_SCAN_RAKING_MEMOIZE", stream, size); - add_benchmarks>( - benchmarks, "exclusive_scan", "BLOCK_SCAN_WARP_SCANS", stream, size); - // clang-format on - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + static constexpr const char* name = "inclusive_scan(block_scan_raking)"; +}; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } +struct inclusive_raking_memoize_tag +{ + static constexpr const char* name = "inclusive_scan(block_scan_raking_memoize)"; +}; + +struct inclusive_warp_scans_tag +{ + static constexpr const char* name = "inclusive_scan(block_scan_warp_scans)"; +}; + +struct exclusive_raking_tag +{ + static constexpr const char* name = "exclusive_scan(block_scan_raking)"; +}; - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; +struct exclusive_raking_memoize_tag +{ + static constexpr const char* name = "exclusive_scan(block_scan_raking_memoize)"; +}; + +struct exclusive_warp_scans_tag +{ + static constexpr const char* name = "exclusive_scan(block_scan_warp_scans)"; +}; + +int main(int argc, char* argv[]) +{ + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; + settings.noise_tolerance_percent = 2; + + primbench::executor executor(argc, argv, settings); + + add_benchmarks< + inclusive_scan>( + executor); + add_benchmarks>(executor); + add_benchmarks>(executor); + + add_benchmarks< + exclusive_scan>( + executor); + add_benchmarks>(executor); + add_benchmarks>(executor); + + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_block_shuffle.cpp b/projects/hipcub/benchmark/benchmark_block_shuffle.cpp index 697d381c24dc..2854027558af 100644 --- a/projects/hipcub/benchmark/benchmark_block_shuffle.cpp +++ b/projects/hipcub/benchmark/benchmark_block_shuffle.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,31 +20,26 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif +constexpr unsigned int Trials = 100; -template -__global__ __launch_bounds__(BlockSize) void kernel(const T* input, T* output) +template +__global__ __launch_bounds__(BlockSize) +void kernel(const T* input, T* output) { - Runner::template run(input, output); + Runner::template run(input, output); } struct offset { - template - __device__ static void run(const T* input, T* output) + static constexpr const char* name = "offset"; + + template + __device__ + static void run(const T* input, T* output) { const unsigned int tid = hipBlockIdx_x * BlockSize + hipThreadIdx_x; @@ -53,7 +48,7 @@ struct offset using bshuffle_t = hipcub::BlockShuffle; __shared__ typename bshuffle_t::TempStorage storage; -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { bshuffle_t(storage).Offset(value, value, 1); @@ -71,11 +66,11 @@ struct offset struct rotate { - template - __device__ static void run(const T* input, T* output) + static constexpr const char* name = "rotate"; + + template + __device__ + static void run(const T* input, T* output) { const unsigned int tid = hipBlockIdx_x * BlockSize + hipThreadIdx_x; @@ -84,7 +79,7 @@ struct rotate using bshuffle_t = hipcub::BlockShuffle; __shared__ typename bshuffle_t::TempStorage storage; -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { bshuffle_t(storage).Rotate(value, value, 1); @@ -102,8 +97,11 @@ struct rotate struct up { - template - __device__ static void run(const T* input, T* output) + static constexpr const char* name = "up"; + + template + __device__ + static void run(const T* input, T* output) { const unsigned int tid = hipBlockIdx_x * BlockSize + hipThreadIdx_x; @@ -116,7 +114,7 @@ struct up using bshuffle_t = hipcub::BlockShuffle; __shared__ typename bshuffle_t::TempStorage storage; -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { bshuffle_t(storage).Up(values, values); @@ -137,8 +135,11 @@ struct up struct down { - template - __device__ static void run(const T* input, T* output) + static constexpr const char* name = "down"; + + template + __device__ + static void run(const T* input, T* output) { const unsigned int tid = hipBlockIdx_x * BlockSize + hipThreadIdx_x; @@ -151,7 +152,7 @@ struct down using bshuffle_t = hipcub::BlockShuffle; __shared__ typename bshuffle_t::TempStorage storage; -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { bshuffle_t(storage).Down(values, values); @@ -170,167 +171,105 @@ struct down static constexpr bool uses_ipt = true; }; -template -void run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) +template +class block_shuffle_benchmark : public primbench::benchmark_interface { - constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto size = items_per_block * ((N + items_per_block - 1) / items_per_block); - - std::vector input(size, T(1)); - T* d_input; - T* d_output; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMalloc(&d_output, size * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - for(auto _ : state) + primbench::json meta() const override { - auto start = std::chrono::high_resolution_clock::now(); - - hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), - dim3(size / items_per_block), - dim3(BlockSize), - 0, - stream, - d_input, - d_output); - HIP_CHECK(hipPeekAtLastError()); + return primbench::json{} + .add("algo", "block_shuffle") + .add("subalgo", Benchmark::name) + .add("lvl", "block") + .add("data_type", primbench::name()) + .add("block_size", BlockSize) + .add("items_per_thread", ItemsPerThread); + } + + void run(primbench::state& state) override + { + const size_t input_items = state.size; + const auto& stream = state.stream; + + constexpr auto items_per_block = BlockSize * ItemsPerThread; + const auto items + = items_per_block * ((input_items + items_per_block - 1) / items_per_block); + + std::vector input(items, T(1)); + T* d_input; + T* d_output; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK(hipMalloc(&d_output, items * sizeof(T))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + state.set_items(Trials * items); + state.add_writes(Trials * items); + + state.run( + [&] + { + hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), + dim3(items / items_per_block), + dim3(BlockSize), + 0, + stream, + d_input, + d_output); + }); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); } - state.SetBytesProcessed(state.iterations() * Trials * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * Trials * size); +}; - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); -} +#define CREATE_BENCHMARK_IPT(BS, IPT) \ + executor.queue>() -#define CREATE_BENCHMARK_IPT(BS, IPT) \ - benchmark::RegisterBenchmark( \ - ("block_shuffle.sub_algorithm_name:" + name) \ - .c_str(), \ - &run_benchmark, \ - stream, \ - size) - -#define CREATE_BENCHMARK(BS) \ - benchmark::RegisterBenchmark(("block_shuffle.sub_algorithm_name:" + name) \ - .c_str(), \ - &run_benchmark, \ - stream, \ - size) +#define CREATE_BENCHMARK(BS) executor.queue>() template = true> -void add_benchmarks_type(const std::string& name, - std::vector& benchmarks, - hipStream_t stream, - size_t size, - const std::string& type_name) +void add_benchmarks_type(primbench::executor& executor) { - std::vector bs = { - CREATE_BENCHMARK_IPT(256, 1), - CREATE_BENCHMARK_IPT(256, 3), - CREATE_BENCHMARK_IPT(256, 4), - CREATE_BENCHMARK_IPT(256, 8), - CREATE_BENCHMARK_IPT(256, 16), - CREATE_BENCHMARK_IPT(256, 32), - }; - - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + CREATE_BENCHMARK_IPT(256, 1); + CREATE_BENCHMARK_IPT(256, 3); + CREATE_BENCHMARK_IPT(256, 4); + CREATE_BENCHMARK_IPT(256, 8); + CREATE_BENCHMARK_IPT(256, 16); + CREATE_BENCHMARK_IPT(256, 32); } template = true> -void add_benchmarks_type(const std::string& name, - std::vector& benchmarks, - hipStream_t stream, - size_t size, - const std::string& type_name) +void add_benchmarks_type(primbench::executor& executor) { - std::vector bs = { - CREATE_BENCHMARK(256), - }; - - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + CREATE_BENCHMARK(256); } -#define CREATE_BENCHMARKS(T) add_benchmarks_type(name, benchmarks, stream, size, #T) +#define CREATE_BENCHMARKS(T) add_benchmarks_type(executor) template -void add_benchmarks(const std::string& name, - std::vector& benchmarks, - hipStream_t stream, - size_t size) +void add_benchmarks(primbench::executor& executor) { - using custom_float2 = benchmark_utils::custom_type; - using custom_double2 = benchmark_utils::custom_type; - CREATE_BENCHMARKS(int); CREATE_BENCHMARKS(float); CREATE_BENCHMARKS(double); CREATE_BENCHMARKS(int8_t); - CREATE_BENCHMARKS(long long); + CREATE_BENCHMARKS(int64_t); CREATE_BENCHMARKS(custom_float2); CREATE_BENCHMARKS(custom_double2); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_block_shuffle" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - add_benchmarks("offset", benchmarks, stream, size); - add_benchmarks("rotate", benchmarks, stream, size); - add_benchmarks("up", benchmarks, stream, size); - add_benchmarks("down", benchmarks, stream, size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + primbench::executor executor(argc, argv, settings); + + add_benchmarks(executor); + add_benchmarks(executor); + add_benchmarks(executor); + add_benchmarks(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_adjacent_difference.cpp b/projects/hipcub/benchmark/benchmark_device_adjacent_difference.cpp index 335144c0248e..7e3f72e096ae 100644 --- a/projects/hipcub/benchmark/benchmark_device_adjacent_difference.cpp +++ b/projects/hipcub/benchmark/benchmark_device_adjacent_difference.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,14 +20,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -// CUB's implementation of DeviceRunLengthEncode has unused parameters, -// disable the warning because all warnings are threated as errors: - -#include "common_benchmark_header.hpp" - -#include - -#include "cmdparser.hpp" +#include "benchmark_utils.hpp" #include @@ -42,18 +35,11 @@ namespace { -#ifndef DEFAULT_N -constexpr std::size_t DEFAULT_N = 1024 * 1024 * 128; -#endif - -constexpr unsigned int batch_size = 10; -constexpr unsigned int warmup_size = 5; - template auto dispatch_adjacent_difference(std::true_type /*left*/, std::true_type /*copy*/, void* const temporary_storage, - std::size_t& storage_size, + size_t& storage_size, const InputIt input, const OutputIt output, Args&&... args) @@ -69,7 +55,7 @@ template auto dispatch_adjacent_difference(std::false_type /*left*/, std::true_type /*copy*/, void* const temporary_storage, - std::size_t& storage_size, + size_t& storage_size, const InputIt input, const OutputIt output, Args&&... args) @@ -85,7 +71,7 @@ template auto dispatch_adjacent_difference(std::true_type /*left*/, std::false_type /*copy*/, void* const temporary_storage, - std::size_t& storage_size, + size_t& storage_size, const InputIt input, const OutputIt /*output*/, Args&&... args) @@ -100,7 +86,7 @@ template auto dispatch_adjacent_difference(std::false_type /*left*/, std::false_type /*copy*/, void* const temporary_storage, - std::size_t& storage_size, + size_t& storage_size, const InputIt input, const OutputIt /*output*/, Args&&... args) @@ -112,160 +98,106 @@ auto dispatch_adjacent_difference(std::false_type /*left*/, } template -void run_benchmark(benchmark::State& state, const std::size_t size, const hipStream_t stream) +class device_adjacent_difference_benchmark : public primbench::benchmark_interface { - using output_type = T; - - // Generate data - const std::vector input = benchmark_utils::get_random_data(size, 1, 100); - - T* d_input; - output_type* d_output = nullptr; - HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(input[0]))); - HIP_CHECK( - hipMemcpy(d_input, input.data(), input.size() * sizeof(input[0]), hipMemcpyHostToDevice)); - - if(copy) + primbench::json meta() const override { - HIP_CHECK(hipMalloc(&d_output, size * sizeof(output_type))); + return primbench::json{} + .add("algo", "device_adjacent_difference") + .add("lvl", "device") + .add("data_type", primbench::name()) + .add("subalgo", + "subtract_" + std::string(left ? "left" : "right") + + std::string(copy ? "_copy" : "")); } - static constexpr std::integral_constant left_tag; - static constexpr std::integral_constant copy_tag; + void run(primbench::state& state) override + { + using output_type = T; - // Allocate temporary storage - std::size_t temp_storage_size{}; - void* d_temp_storage = nullptr; + const size_t items = state.size; + const auto& stream = state.stream; - const auto launch = [&] - { - return dispatch_adjacent_difference(left_tag, - copy_tag, - d_temp_storage, - temp_storage_size, - d_input, - d_output, - size, - hipcub::Sum{}, - stream); - }; - HIP_CHECK(launch()); - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size)); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) - { - HIP_CHECK(launch()); - } - HIP_CHECK(hipDeviceSynchronize()); + // Generate data + const std::vector input = benchmark_utils::get_random_data(items, 1, 100); - // Run - for(auto _ : state) - { - auto start = std::chrono::high_resolution_clock::now(); + T* d_input; + output_type* d_output = nullptr; + HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(input[0]))); + HIP_CHECK(hipMemcpy(d_input, + input.data(), + input.size() * sizeof(input[0]), + hipMemcpyHostToDevice)); - for(size_t i = 0; i < batch_size; i++) + if constexpr(copy) { - HIP_CHECK(launch()); + HIP_CHECK(hipMalloc(&d_output, items * sizeof(output_type))); } - HIP_CHECK(hipStreamSynchronize(stream)); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * batch_size * size); + static constexpr std::integral_constant left_tag; + static constexpr std::integral_constant copy_tag; - HIP_CHECK(hipFree(d_input)); - if(copy) - { - HIP_CHECK(hipFree(d_output)); - } - HIP_CHECK(hipFree(d_temp_storage)); -} + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; -} // namespace + const auto launch = [&] + { + HIP_CHECK(dispatch_adjacent_difference(left_tag, + copy_tag, + d_temp_storage, + temp_storage_bytes, + d_input, + d_output, + items, + benchmark_utils::plus{}, + stream)); + }; -using namespace std::string_literals; - -#define CREATE_BENCHMARK(T, left, copy) \ - benchmark::RegisterBenchmark(std::string("device_adjacent_difference" \ - "." \ - "sub_algorithm_name:subtract_" \ - + std::string(left ? "left" : "right") \ - + std::string(copy ? "_copy" : "")) \ - .c_str(), \ - &run_benchmark, \ - size, \ - stream) - -// clang-format off -#define CREATE_BENCHMARKS(T) \ - CREATE_BENCHMARK(T, true, false), \ - CREATE_BENCHMARK(T, true, true), \ - CREATE_BENCHMARK(T, false, false), \ - CREATE_BENCHMARK(T, false, true) -// clang-format on + launch(); -int main(int argc, char* argv[]) -{ - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); + state.set_items(items); + state.add_writes(items); - // HIP - const hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); + state.run(launch); - std::cout << "benchmark_device_adjacent_difference" << std::endl; - std::cout << "[HIP] Device name: " << devProp.name << std::endl; + HIP_CHECK(hipFree(d_input)); + if(copy) + { + HIP_CHECK(hipFree(d_output)); + } + HIP_CHECK(hipFree(d_temp_storage)); + } +}; - using custom_float2 = benchmark_utils::custom_type; - using custom_double2 = benchmark_utils::custom_type; +} // namespace - // Add benchmarks - const std::vector benchmarks = { - CREATE_BENCHMARKS(int), - CREATE_BENCHMARKS(std::int64_t), +#define CREATE_BENCHMARK(T, left, copy) \ + executor.queue>() - CREATE_BENCHMARKS(uint8_t), +#define CREATE_BENCHMARKS(T) \ + CREATE_BENCHMARK(T, true, false), CREATE_BENCHMARK(T, true, true), \ + CREATE_BENCHMARK(T, false, false), CREATE_BENCHMARK(T, false, true) - CREATE_BENCHMARKS(float), - CREATE_BENCHMARKS(double), +int main(int argc, char* argv[]) +{ + primbench::settings settings; + settings.size = 128 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; - CREATE_BENCHMARKS(custom_float2), - CREATE_BENCHMARKS(custom_double2), - }; + primbench::executor executor(argc, argv, settings); - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + CREATE_BENCHMARKS(int); + CREATE_BENCHMARKS(int64_t); - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + CREATE_BENCHMARKS(uint8_t); + + CREATE_BENCHMARKS(float); + CREATE_BENCHMARKS(double); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); + CREATE_BENCHMARKS(custom_float2); + CREATE_BENCHMARKS(custom_double2); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_batch_copy.cpp b/projects/hipcub/benchmark/benchmark_device_batch_copy.cpp index 909c50b564ca..bc393cadc245 100644 --- a/projects/hipcub/benchmark/benchmark_device_batch_copy.cpp +++ b/projects/hipcub/benchmark/benchmark_device_batch_copy.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,9 +20,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "benchmark/benchmark.h" -#include "cmdparser.hpp" -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" #include #include @@ -41,8 +39,7 @@ #include #include -constexpr uint32_t warmup_size = 5; -constexpr int32_t max_size = 1024 * 1024; +constexpr int32_t max_size = 1024 * 1024; constexpr int32_t wlev_min_size = 128; constexpr int32_t blev_min_size = 1024; @@ -90,12 +87,10 @@ std::vector shuffled_exclusive_scan(const std::vector& input, RandomGenera return result; } -using offset_type = size_t; - template struct BatchCopyData { - size_t total_num_elements = 0; + size_t items = 0; ValueType* d_input = nullptr; ValueType* d_output = nullptr; ValueType** d_buffer_srcs = nullptr; @@ -106,7 +101,7 @@ struct BatchCopyData BatchCopyData(const BatchCopyData&) = delete; BatchCopyData(BatchCopyData&& other) - : total_num_elements{std::exchange(other.total_num_elements, 0)} + : items{std::exchange(other.items, 0)} , d_input{std::exchange(other.d_input, nullptr)} , d_output{std::exchange(other.d_output, nullptr)} , d_buffer_srcs{std::exchange(other.d_buffer_srcs, nullptr)} @@ -116,7 +111,7 @@ struct BatchCopyData BatchCopyData& operator=(BatchCopyData&& other) { - total_num_elements = std::exchange(other.total_num_elements, 0); + items = std::exchange(other.items, 0); d_input = std::exchange(other.d_input, nullptr); d_output = std::exchange(other.d_output, nullptr); d_buffer_srcs = std::exchange(other.d_buffer_srcs, nullptr); @@ -127,9 +122,9 @@ struct BatchCopyData BatchCopyData& operator=(const BatchCopyData&) = delete; - size_t total_num_bytes() const + size_t get_bytes() const { - return total_num_elements * sizeof(ValueType); + return items * sizeof(ValueType); } ~BatchCopyData() @@ -185,27 +180,28 @@ BatchCopyData prepare_data(const int32_t num_tlev_buf // Shuffle the sizes so that size classes aren't clustered std::shuffle(h_buffer_num_elements.begin(), h_buffer_num_elements.end(), rng); - result.total_num_elements + result.items = std::accumulate(h_buffer_num_elements.begin(), h_buffer_num_elements.end(), size_t{0}); // Generate data. - std::independent_bits_engine bits_engine{rng}; + std::independent_bits_engine bits_engine{rng}; - const size_t num_ints - = benchmark_utils::ceiling_div(result.total_num_bytes(), sizeof(uint64_t)); - auto h_input = std::make_unique(num_ints * sizeof(uint64_t)); + const size_t num_ints = benchmark_utils::ceiling_div(result.get_bytes(), sizeof(unsigned long long)); + auto h_input = std::make_unique(num_ints * sizeof(unsigned long long)); - std::for_each(reinterpret_cast(h_input.get()), - reinterpret_cast(h_input.get() + num_ints * sizeof(uint64_t)), - [&bits_engine](uint64_t& elem) { ::new(&elem) uint64_t{bits_engine()}; }); + std::for_each(reinterpret_cast(h_input.get()), + reinterpret_cast(h_input.get() + num_ints * sizeof(unsigned long long)), + [&bits_engine](unsigned long long& elem) { ::new(&elem) unsigned long long{bits_engine()}; }); - HIP_CHECK(hipMalloc(&result.d_input, result.total_num_bytes())); - HIP_CHECK(hipMalloc(&result.d_output, result.total_num_bytes())); + HIP_CHECK(hipMalloc(&result.d_input, result.get_bytes())); + HIP_CHECK(hipMalloc(&result.d_output, result.get_bytes())); HIP_CHECK(hipMalloc(&result.d_buffer_srcs, num_buffers * sizeof(ValueType*))); HIP_CHECK(hipMalloc(&result.d_buffer_dsts, num_buffers * sizeof(ValueType*))); HIP_CHECK(hipMalloc(&result.d_buffer_sizes, num_buffers * sizeof(BufferSizeType))); + using offset_type = size_t; + // Generate the source and shuffled destination offsets. std::vector src_offsets; std::vector dst_offsets; @@ -214,7 +210,8 @@ BatchCopyData prepare_data(const int32_t num_tlev_buf { src_offsets = shuffled_exclusive_scan(h_buffer_num_elements, rng); dst_offsets = shuffled_exclusive_scan(h_buffer_num_elements, rng); - } else + } + else { src_offsets = std::vector(num_buffers); dst_offsets = std::vector(num_buffers); @@ -240,8 +237,7 @@ BatchCopyData prepare_data(const int32_t num_tlev_buf } // Prepare the batch copy. - HIP_CHECK( - hipMemcpy(result.d_input, h_input.get(), result.total_num_bytes(), hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(result.d_input, h_input.get(), result.get_bytes(), hipMemcpyHostToDevice)); HIP_CHECK(hipMemcpy(result.d_buffer_srcs, h_buffer_srcs.data(), h_buffer_srcs.size() * sizeof(ValueType*), @@ -258,91 +254,69 @@ BatchCopyData prepare_data(const int32_t num_tlev_buf return result; } -template -void run_benchmark(benchmark::State& state, - hipStream_t stream, - const int32_t num_tlev_buffers = 1024, - const int32_t num_wlev_buffers = 1024, - const int32_t num_blev_buffers = 1024) +template +class batch_copy_benchmark : public primbench::benchmark_interface { - const size_t num_buffers = num_tlev_buffers + num_wlev_buffers + num_blev_buffers; + using ValueType = benchmark_utils::custom_aligned_type; - size_t temp_storage_bytes = 0; - BatchCopyData data; - HIP_CHECK(hipcub::DeviceCopy::Batched(nullptr, - temp_storage_bytes, - data.d_buffer_srcs, - data.d_buffer_dsts, - data.d_buffer_sizes, - num_buffers)); + primbench::json meta() const override + { + return primbench::json{} + .add("algo", "device_batch_copy") + .add("lvl", "device") + .add("item_size", ItemSize) + .add("item_alignment", ItemAlignment) + .add("data_type", primbench::name()) + .add("number_of_tlev", NumTlevBuffers) + .add("number_of_wlev", NumWlevBuffers) + .add("number_of_blev", NumBlevBuffers); + } - void* d_temp_storage = nullptr; - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + void run(primbench::state& state) override + { + const auto& stream = state.stream; - data = prepare_data(num_tlev_buffers, - num_wlev_buffers, - num_blev_buffers); + BatchCopyData data + = prepare_data(NumTlevBuffers, + NumWlevBuffers, + NumBlevBuffers); - // Warm-up - for(size_t i = 0; i < warmup_size; i++) - { - HIP_CHECK(hipcub::DeviceCopy::Batched(d_temp_storage, - temp_storage_bytes, - data.d_buffer_srcs, - data.d_buffer_dsts, - data.d_buffer_sizes, - num_buffers, - stream)); - } - HIP_CHECK(hipDeviceSynchronize()); + const size_t num_buffers = NumTlevBuffers + NumWlevBuffers + NumBlevBuffers; - // HIP events creation - hipEvent_t start, stop; - HIP_CHECK(hipEventCreate(&start)); - HIP_CHECK(hipEventCreate(&stop)); + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; - for(auto _ : state) - { - // Record start event - HIP_CHECK(hipEventRecord(start, stream)); - - HIP_CHECK(hipcub::DeviceCopy::Batched(d_temp_storage, - temp_storage_bytes, - data.d_buffer_srcs, - data.d_buffer_dsts, - data.d_buffer_sizes, - num_buffers, - stream)); - - // Record stop event and wait until it completes - HIP_CHECK(hipEventRecord(stop, stream)); - HIP_CHECK(hipEventSynchronize(stop)); - - float elapsed_mseconds; - HIP_CHECK(hipEventElapsedTime(&elapsed_mseconds, start, stop)); - state.SetIterationTime(elapsed_mseconds / 1000); - } - state.SetBytesProcessed(state.iterations() * data.total_num_bytes()); - state.SetItemsProcessed(state.iterations() * data.total_num_elements); + const auto launch = [&] + { + HIP_CHECK(hipcub::DeviceCopy::Batched(d_temp_storage, + temp_storage_bytes, + data.d_buffer_srcs, + data.d_buffer_dsts, + data.d_buffer_sizes, + num_buffers, + stream)); + }; - HIP_CHECK(hipFree(d_temp_storage)); -} + launch(); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + + state.set_items(data.items); + state.add_writes(data.items); + + state.run(launch); + + HIP_CHECK(hipFree(d_temp_storage)); + } +}; -#define CREATE_BENCHMARK(IS, IA, T, num_tlev, num_wlev, num_blev) \ - benchmark::RegisterBenchmark( \ - std::string("device_batch_copy" \ - ".") \ - .c_str(), \ - [=](benchmark::State& state) \ - { \ - run_benchmark, T>(state, \ - stream, \ - num_tlev, \ - num_wlev, \ - num_blev); \ - }) +#define CREATE_BENCHMARK(IS, IA, T, num_tlev, num_wlev, num_blev) \ + executor.queue>() #define BENCHMARK_TYPE(item_size, item_alignment) \ CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 100000, 0, 0), \ @@ -352,66 +326,21 @@ void run_benchmark(benchmark::State& state, int32_t main(int32_t argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", 1024, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.set_optional("name_format", - "name_format", - "human", - "either: json,human,txt"); - - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int32_t trials = parser.get("trials"); - - // HIP - hipStream_t stream = hipStreamDefault; // default - - hipDeviceProp_t devProp; - int device_id = 0; - - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - - std::cout << "benchmark_device_batch_copy" << std::endl; - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Benchmark info - benchmark::AddCustomContext("size", std::to_string(size)); - - // Add benchmarks - std::vector benchmarks; - - benchmarks = {BENCHMARK_TYPE(1, 1), - BENCHMARK_TYPE(1, 2), - BENCHMARK_TYPE(1, 4), - BENCHMARK_TYPE(1, 8), - BENCHMARK_TYPE(2, 2), - BENCHMARK_TYPE(4, 4), - BENCHMARK_TYPE(8, 8)}; - - - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + // The size is set to 1, as prepare_data() calculates it later. + settings.size = 1; + settings.min_gpu_ms_per_batch = 100; + + primbench::executor executor(argc, argv, settings); + + BENCHMARK_TYPE(1, 1); + BENCHMARK_TYPE(1, 2); + BENCHMARK_TYPE(1, 4); + BENCHMARK_TYPE(1, 8); + BENCHMARK_TYPE(2, 2); + BENCHMARK_TYPE(4, 4); + BENCHMARK_TYPE(8, 8); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_batch_memcpy.cpp b/projects/hipcub/benchmark/benchmark_device_batch_memcpy.cpp index 8de4f8e7bdc8..7aec677761d1 100644 --- a/projects/hipcub/benchmark/benchmark_device_batch_memcpy.cpp +++ b/projects/hipcub/benchmark/benchmark_device_batch_memcpy.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,9 +20,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "benchmark/benchmark.h" -#include "cmdparser.hpp" -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" #include #include @@ -45,8 +43,7 @@ #include -constexpr uint32_t warmup_size = 5; -constexpr int32_t max_size = 1024 * 1024; +constexpr int32_t max_size = 1024 * 1024; constexpr int32_t wlev_min_size = 128; constexpr int32_t blev_min_size = 1024; @@ -94,12 +91,10 @@ std::vector shuffled_exclusive_scan(const std::vector& input, RandomGenera return result; } -using offset_type = size_t; - template struct BatchMemcpyData { - size_t total_num_elements = 0; + size_t items = 0; ValueType* d_input = nullptr; ValueType* d_output = nullptr; ValueType** d_buffer_srcs = nullptr; @@ -110,7 +105,7 @@ struct BatchMemcpyData BatchMemcpyData(const BatchMemcpyData&) = delete; BatchMemcpyData(BatchMemcpyData&& other) - : total_num_elements{std::exchange(other.total_num_elements, 0)} + : items{std::exchange(other.items, 0)} , d_input{std::exchange(other.d_input, nullptr)} , d_output{std::exchange(other.d_output, nullptr)} , d_buffer_srcs{std::exchange(other.d_buffer_srcs, nullptr)} @@ -120,7 +115,7 @@ struct BatchMemcpyData BatchMemcpyData& operator=(BatchMemcpyData&& other) { - total_num_elements = std::exchange(other.total_num_elements, 0); + items = std::exchange(other.items, 0); d_input = std::exchange(other.d_input, nullptr); d_output = std::exchange(other.d_output, nullptr); d_buffer_srcs = std::exchange(other.d_buffer_srcs, nullptr); @@ -131,9 +126,9 @@ struct BatchMemcpyData BatchMemcpyData& operator=(const BatchMemcpyData&) = delete; - size_t total_num_bytes() const + size_t get_bytes() const { - return total_num_elements * sizeof(ValueType); + return items * sizeof(ValueType); } ~BatchMemcpyData() @@ -196,27 +191,28 @@ BatchMemcpyData prepare_data(const int32_t num_tlev_b h_buffer_num_bytes[i] = h_buffer_num_elements[i] * sizeof(ValueType); } - result.total_num_elements + result.items = std::accumulate(h_buffer_num_elements.begin(), h_buffer_num_elements.end(), size_t{0}); // Generate data. - std::independent_bits_engine bits_engine{rng}; + std::independent_bits_engine bits_engine{rng}; - const size_t num_ints - = benchmark_utils::ceiling_div(result.total_num_bytes(), sizeof(uint64_t)); - auto h_input = std::make_unique(num_ints * sizeof(uint64_t)); + const size_t num_ints = benchmark_utils::ceiling_div(result.get_bytes(), sizeof(unsigned long long)); + auto h_input = std::make_unique(num_ints * sizeof(unsigned long long)); - std::for_each(reinterpret_cast(h_input.get()), - reinterpret_cast(h_input.get() + num_ints * sizeof(uint64_t)), - [&bits_engine](uint64_t& elem) { ::new(&elem) uint64_t{bits_engine()}; }); + std::for_each(reinterpret_cast(h_input.get()), + reinterpret_cast(h_input.get() + num_ints * sizeof(unsigned long long)), + [&bits_engine](unsigned long long& elem) { ::new(&elem) unsigned long long{bits_engine()}; }); - HIP_CHECK(hipMalloc(&result.d_input, result.total_num_bytes())); - HIP_CHECK(hipMalloc(&result.d_output, result.total_num_bytes())); + HIP_CHECK(hipMalloc(&result.d_input, result.get_bytes())); + HIP_CHECK(hipMalloc(&result.d_output, result.get_bytes())); HIP_CHECK(hipMalloc(&result.d_buffer_srcs, num_buffers * sizeof(ValueType*))); HIP_CHECK(hipMalloc(&result.d_buffer_dsts, num_buffers * sizeof(ValueType*))); HIP_CHECK(hipMalloc(&result.d_buffer_sizes, num_buffers * sizeof(BufferSizeType))); + using offset_type = size_t; + // Generate the source and shuffled destination offsets. std::vector src_offsets; std::vector dst_offsets; @@ -225,7 +221,8 @@ BatchMemcpyData prepare_data(const int32_t num_tlev_b { src_offsets = shuffled_exclusive_scan(h_buffer_num_elements, rng); dst_offsets = shuffled_exclusive_scan(h_buffer_num_elements, rng); - } else + } + else { src_offsets = std::vector(num_buffers); dst_offsets = std::vector(num_buffers); @@ -251,8 +248,7 @@ BatchMemcpyData prepare_data(const int32_t num_tlev_b } // Prepare the batch memcpy. - HIP_CHECK( - hipMemcpy(result.d_input, h_input.get(), result.total_num_bytes(), hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(result.d_input, h_input.get(), result.get_bytes(), hipMemcpyHostToDevice)); HIP_CHECK(hipMemcpy(result.d_buffer_srcs, h_buffer_srcs.data(), h_buffer_srcs.size() * sizeof(ValueType*), @@ -269,90 +265,69 @@ BatchMemcpyData prepare_data(const int32_t num_tlev_b return result; } -template -void run_benchmark(benchmark::State& state, - hipStream_t stream, - const int32_t num_tlev_buffers = 1024, - const int32_t num_wlev_buffers = 1024, - const int32_t num_blev_buffers = 1024) +template +class batch_memcpy_benchmark : public primbench::benchmark_interface { - const size_t num_buffers = num_tlev_buffers + num_wlev_buffers + num_blev_buffers; + using ValueType = benchmark_utils::custom_aligned_type; - size_t temp_storage_bytes = 0; - BatchMemcpyData data; - HIP_CHECK(hipcub::DeviceMemcpy::Batched(nullptr, - temp_storage_bytes, - data.d_buffer_srcs, - data.d_buffer_dsts, - data.d_buffer_sizes, - num_buffers)); + primbench::json meta() const override + { + return primbench::json{} + .add("algo", "device_batch_memcpy") + .add("data_type", primbench::name()) + .add("lvl", "device") + .add("item_size", ItemSize) + .add("item_alignment", ItemAlignment) + .add("number_of_tlev", NumTlevBuffers) + .add("number_of_wlev", NumWlevBuffers) + .add("number_of_blev", NumBlevBuffers); + } - void* d_temp_storage = nullptr; - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + void run(primbench::state& state) override + { + const auto& stream = state.stream; - data = prepare_data(num_tlev_buffers, - num_wlev_buffers, - num_blev_buffers); + BatchMemcpyData data + = prepare_data(NumTlevBuffers, + NumWlevBuffers, + NumBlevBuffers); - // Warm-up - for(size_t i = 0; i < warmup_size; i++) - { - HIP_CHECK(hipcub::DeviceMemcpy::Batched(d_temp_storage, - temp_storage_bytes, - data.d_buffer_srcs, - data.d_buffer_dsts, - data.d_buffer_sizes, - num_buffers, - stream)); - } - HIP_CHECK(hipDeviceSynchronize()); + constexpr size_t num_buffers = NumTlevBuffers + NumWlevBuffers + NumBlevBuffers; - // HIP events creation - hipEvent_t start, stop; - HIP_CHECK(hipEventCreate(&start)); - HIP_CHECK(hipEventCreate(&stop)); + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; - for(auto _ : state) - { - // Record start event - HIP_CHECK(hipEventRecord(start, stream)); - - HIP_CHECK(hipcub::DeviceMemcpy::Batched(d_temp_storage, - temp_storage_bytes, - data.d_buffer_srcs, - data.d_buffer_dsts, - data.d_buffer_sizes, - num_buffers, - stream)); - - // Record stop event and wait until it completes - HIP_CHECK(hipEventRecord(stop, stream)); - HIP_CHECK(hipEventSynchronize(stop)); - - float elapsed_mseconds; - HIP_CHECK(hipEventElapsedTime(&elapsed_mseconds, start, stop)); - state.SetIterationTime(elapsed_mseconds / 1000); - } - state.SetBytesProcessed(state.iterations() * data.total_num_bytes()); - state.SetItemsProcessed(state.iterations() * data.total_num_elements); + const auto launch = [&] + { + HIP_CHECK(hipcub::DeviceMemcpy::Batched(d_temp_storage, + temp_storage_bytes, + data.d_buffer_srcs, + data.d_buffer_dsts, + data.d_buffer_sizes, + num_buffers, + stream)); + }; - HIP_CHECK(hipFree(d_temp_storage)); -} + launch(); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + + state.set_items(data.items); + state.add_writes(data.items); + + state.run(launch); + + HIP_CHECK(hipFree(d_temp_storage)); + } +}; -#define CREATE_BENCHMARK(IS, IA, T, num_tlev, num_wlev, num_blev) \ - benchmark::RegisterBenchmark( \ - std::string("device_batch_memcpy.") \ - .c_str(), \ - [=](benchmark::State& state) \ - { \ - run_benchmark, T>(state, \ - stream, \ - num_tlev, \ - num_wlev, \ - num_blev); \ - }) +#define CREATE_BENCHMARK(IS, IA, T, num_tlev, num_wlev, num_blev) \ + executor.queue>() #define BENCHMARK_TYPE(item_size, item_alignment) \ CREATE_BENCHMARK(item_size, item_alignment, uint32_t, 100000, 0, 0), \ @@ -362,63 +337,21 @@ void run_benchmark(benchmark::State& state, int32_t main(int32_t argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", 1024, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.set_optional("name_format", - "name_format", - "human", - "either: json,human,txt"); - - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int32_t trials = parser.get("trials"); - - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - - std::cout << "benchmark_device_adjacent_difference" << std::endl; - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // HIP - hipStream_t stream = hipStreamDefault; // default - - // Benchmark info - benchmark::AddCustomContext("size", std::to_string(size)); - - // Add benchmarks - std::vector benchmarks; - - benchmarks = {BENCHMARK_TYPE(1, 1), - BENCHMARK_TYPE(1, 2), - BENCHMARK_TYPE(1, 4), - BENCHMARK_TYPE(1, 8), - BENCHMARK_TYPE(2, 2), - BENCHMARK_TYPE(4, 4), - BENCHMARK_TYPE(8, 8)}; - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + // The size is set to 1, as prepare_data() calculates it later. + settings.size = 1; + settings.min_gpu_ms_per_batch = 100; + + primbench::executor executor(argc, argv, settings); + + BENCHMARK_TYPE(1, 1); + BENCHMARK_TYPE(1, 2); + BENCHMARK_TYPE(1, 4); + BENCHMARK_TYPE(1, 8); + BENCHMARK_TYPE(2, 2); + BENCHMARK_TYPE(4, 4); + BENCHMARK_TYPE(8, 8); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_for.cpp b/projects/hipcub/benchmark/benchmark_device_for.cpp deleted file mode 100644 index 4b7a7d75fdfa..000000000000 --- a/projects/hipcub/benchmark/benchmark_device_for.cpp +++ /dev/null @@ -1,160 +0,0 @@ -// MIT License -// -// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -// CUB's implementation of single_pass_scan_operators has maybe uninitialized parameters, -// disable the warning because all warnings are threated as errors: - -#include "common_benchmark_header.hpp" - -// HIP API -#include - -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif - -const unsigned int batch_size = 10; -const unsigned int warmup_size = 5; - -template -struct op_t -{ - unsigned int* d_count; - - HIPCUB_DEVICE - void operator()(T i) - { - // The data is non zero so atomic will never be activated. - if(i == 0) - { - atomicAdd(d_count, 1); - } - } -}; - -template -void run_benchmark(benchmark::State& state, hipStream_t stream, size_t size) -{ - using T = Value; - - // Generate data - std::vector values_input(size, 4); - - T* d_input; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, values_input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - - unsigned int* d_count; - HIP_CHECK(hipMalloc(&d_count, sizeof(T))); - HIP_CHECK(hipMemset(d_count, 0, sizeof(T))); - op_t device_op{d_count}; - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) - { - HIP_CHECK(hipcub::DeviceFor::ForEach(d_input, d_input + size, device_op, stream)); - } - HIP_CHECK(hipDeviceSynchronize()); - - for(auto _ : state) - { - auto start = std::chrono::high_resolution_clock::now(); - - for(size_t i = 0; i < batch_size; i++) - { - HIP_CHECK(hipcub::DeviceFor::ForEach(d_input, d_input + size, device_op, stream)); - } - HIP_CHECK(hipStreamSynchronize(stream)); - - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * batch_size * size); - - HIP_CHECK(hipFree(d_count)); - HIP_CHECK(hipFree(d_input)); -} - -#define CREATE_BENCHMARK(Value) \ - benchmark::RegisterBenchmark(("for_each"), \ - &run_benchmark, \ - stream, \ - size) - -int main(int argc, char* argv[]) -{ - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_device_reduce_by_key" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - using custom_double2 = benchmark_utils::custom_type; - - // Add benchmarks - std::vector benchmarks = { - CREATE_BENCHMARK(float), - CREATE_BENCHMARK(double), - CREATE_BENCHMARK(custom_double2), - CREATE_BENCHMARK(int8_t), - CREATE_BENCHMARK(float), - CREATE_BENCHMARK(double), - CREATE_BENCHMARK(long long), - }; - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } - - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } - - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; -} diff --git a/projects/hipcub/benchmark/benchmark_device_for_each.cpp b/projects/hipcub/benchmark/benchmark_device_for_each.cpp new file mode 100644 index 000000000000..7051b9ef224c --- /dev/null +++ b/projects/hipcub/benchmark/benchmark_device_for_each.cpp @@ -0,0 +1,103 @@ +// MIT License +// +// Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "benchmark_utils.hpp" + +#include + +template +struct op_t +{ + unsigned int* d_count; + + HIPCUB_DEVICE + void operator()(T i) + { + // The data is non zero so atomic will never be activated. + if(i == 0) + { + atomicAdd(d_count, 1); + } + } +}; + +template +class for_each_benchmark : public primbench::benchmark_interface +{ + primbench::json meta() const override + { + return primbench::json{} + .add("algo", "device_for_each") + .add("lvl", "device") + .add("data_type", primbench::name()); + } + + void run(primbench::state& state) override + { + const size_t items = state.size; + const auto& stream = state.stream; + + // Generate data + std::vector values_input(items, 4); + + T* d_input; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK( + hipMemcpy(d_input, values_input.data(), items * sizeof(T), hipMemcpyHostToDevice)); + + unsigned int* d_count; + HIP_CHECK(hipMalloc(&d_count, sizeof(*d_count))); + HIP_CHECK(hipMemset(d_count, 0, sizeof(*d_count))); + op_t device_op{d_count}; + + state.set_items(items); + state.add_reads(items); + + state.run( + [&] { + HIP_CHECK(hipcub::DeviceFor::ForEach(d_input, d_input + items, device_op, stream)); + }); + + HIP_CHECK(hipFree(d_count)); + HIP_CHECK(hipFree(d_input)); + } +}; + +#define CREATE_BENCHMARK(Value) executor.queue>() + +int main(int argc, char* argv[]) +{ + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; + + primbench::executor executor(argc, argv, settings); + + // Add benchmarks + CREATE_BENCHMARK(float); + CREATE_BENCHMARK(double); + CREATE_BENCHMARK(custom_double2); + CREATE_BENCHMARK(int8_t); + CREATE_BENCHMARK(int64_t); + + executor.run(); +} diff --git a/projects/hipcub/benchmark/benchmark_device_histogram.cpp b/projects/hipcub/benchmark/benchmark_device_histogram.cpp index ded31e28f8cb..faa518c48884 100644 --- a/projects/hipcub/benchmark/benchmark_device_histogram.cpp +++ b/projects/hipcub/benchmark/benchmark_device_histogram.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -26,35 +26,26 @@ #pragma GCC diagnostic ignored "-Wunused-parameter" #endif -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include -#include - -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif - -const unsigned int batch_size = 10; -const unsigned int warmup_size = 5; template std::vector - generate(size_t size, int entropy_reduction, long long lower_level, long long upper_level) + generate(size_t items, int entropy_reduction, int64_t lower_level, int64_t upper_level) { if(entropy_reduction >= 5) { - return std::vector(size, (lower_level + upper_level) / 2); + return std::vector(items, (lower_level + upper_level) / 2); } const size_t max_random_size = 1024 * 1024; std::random_device rd; std::default_random_engine gen(rd()); - std::vector data(size); + std::vector data(items); std::generate(data.begin(), - data.begin() + std::min(size, max_random_size), + data.begin() + _HIPCUB_STD::min(items, max_random_size), [&]() { // Reduce entropy by applying bitwise AND to random bits @@ -67,9 +58,9 @@ std::vector } return T(lower_level + v % (upper_level - lower_level)); }); - for(size_t i = max_random_size; i < size; i += max_random_size) + for(size_t i = max_random_size; i < items; i += max_random_size) { - std::copy_n(data.begin(), std::min(size - i, max_random_size), data.begin() + i); + std::copy_n(data.begin(), _HIPCUB_STD::min(items - i, max_random_size), data.begin() + i); } return data; } @@ -89,382 +80,334 @@ int get_entropy_percents(int entropy_reduction) const int entropy_reductions[] = {0, 2, 4, 6}; -template -void run_even_benchmark(benchmark::State& state, - size_t bins, - size_t scale, - int entropy_reduction, - hipStream_t stream, - size_t size) +template +class even_benchmark : public primbench::benchmark_interface { - using counter_type = unsigned int; - - const T lower_level = 0; - // casting for compilation with CUB backend because - // there is no casting from size_t (aka unsigned long) to __half - const T upper_level = static_cast(bins * scale); - - // Generate data - std::vector input = generate(size, entropy_reduction, lower_level, upper_level); - - T* d_input; - counter_type* d_histogram; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMalloc(&d_histogram, size * sizeof(counter_type))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - HIP_CHECK(hipcub::DeviceHistogram::HistogramEven(d_temporary_storage, - temporary_storage_bytes, - d_input, - d_histogram, - bins + 1, - lower_level, - upper_level, - int(size), - stream)); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) +public: + even_benchmark(int entropy_reduction) : m_entropy_reduction(entropy_reduction) {} + +private: + primbench::json meta() const override { - HIP_CHECK(hipcub::DeviceHistogram::HistogramEven(d_temporary_storage, - temporary_storage_bytes, - d_input, - d_histogram, - bins + 1, - lower_level, - upper_level, - int(size), - stream)); + return primbench::json{} + .add("algo", "device_histogram") + .add("subalgo", "even") + .add("lvl", "device") + .add("data_type", primbench::name()) + .add("bin_count", Bins) + .add("entropy_percent", std::to_string(get_entropy_percents(m_entropy_reduction))); } - HIP_CHECK(hipDeviceSynchronize()); - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); + using counter_type = unsigned int; + + const T lower_level = 0; + // casting for compilation with CUB backend because + // there is no casting from size_t (aka unsigned long) to __half + const T upper_level = static_cast(Bins * Scale); + + const size_t items = state.size; + const auto& stream = state.stream; - for(size_t i = 0; i < batch_size; i++) + // Generate data + std::vector input = generate(items, m_entropy_reduction, lower_level, upper_level); + + T* d_input; + counter_type* d_histogram; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK(hipMalloc(&d_histogram, items * sizeof(counter_type))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { - HIP_CHECK(hipcub::DeviceHistogram::HistogramEven(d_temporary_storage, - temporary_storage_bytes, + HIP_CHECK(hipcub::DeviceHistogram::HistogramEven(d_temp_storage, + temp_storage_bytes, d_input, d_histogram, - bins + 1, + Bins + 1, lower_level, upper_level, - int(size), + int(items), stream)); - } + }; + + launch(); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + state.set_items(items); + state.add_writes(items); + + state.run(launch); + + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_histogram)); } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * batch_size * size); - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_histogram)); -} + int m_entropy_reduction; +}; -template -void run_multi_even_benchmark(benchmark::State& state, - size_t bins, - size_t scale, - int entropy_reduction, - hipStream_t stream, - size_t size) +template +class multi_even_benchmark : public primbench::benchmark_interface { - using counter_type = unsigned int; +public: + multi_even_benchmark(int entropy_reduction) : m_entropy_reduction(entropy_reduction) {} - int num_levels[ActiveChannels]; - int lower_level[ActiveChannels]; - int upper_level[ActiveChannels]; - for(unsigned int channel = 0; channel < ActiveChannels; channel++) +private: + primbench::json meta() const override { - lower_level[channel] = 0; - upper_level[channel] = bins * scale; - num_levels[channel] = bins + 1; + return primbench::json{} + .add("algo", "device_histogram") + .add("subalgo", "multi_even") + .add("lvl", "device") + .add("data_type", primbench::name()) + .add("bin_count", Bins) + .add("channels", Channels) + .add("active_channels", ActiveChannels) + .add("entropy_percent", std::to_string(get_entropy_percents(m_entropy_reduction))); } - // Generate data - std::vector input - = generate(size * Channels, entropy_reduction, lower_level[0], upper_level[0]); - - T* d_input; - counter_type* d_histogram[ActiveChannels]; - HIP_CHECK(hipMalloc(&d_input, size * Channels * sizeof(T))); - for(unsigned int channel = 0; channel < ActiveChannels; channel++) + void run(primbench::state& state) override { - HIP_CHECK(hipMalloc(&d_histogram[channel], bins * sizeof(counter_type))); - } - HIP_CHECK(hipMemcpy(d_input, input.data(), size * Channels * sizeof(T), hipMemcpyHostToDevice)); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - HIP_CHECK((hipcub::DeviceHistogram::MultiHistogramEven( - d_temporary_storage, - temporary_storage_bytes, - d_input, - d_histogram, - num_levels, - lower_level, - upper_level, - int(size), - stream))); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) - { - HIP_CHECK((hipcub::DeviceHistogram::MultiHistogramEven( - d_temporary_storage, - temporary_storage_bytes, - d_input, - d_histogram, - num_levels, - lower_level, - upper_level, - int(size), - stream))); - } - HIP_CHECK(hipDeviceSynchronize()); + const size_t items = state.size; + const auto& stream = state.stream; - for(auto _ : state) - { - auto start = std::chrono::high_resolution_clock::now(); + using counter_type = unsigned int; + + int num_levels[ActiveChannels]; + int lower_level[ActiveChannels]; + int upper_level[ActiveChannels]; + for(unsigned int channel = 0; channel < ActiveChannels; channel++) + { + lower_level[channel] = 0; + upper_level[channel] = Bins * Scale; + num_levels[channel] = Bins + 1; + } + + // Generate data + std::vector input + = generate(items * Channels, m_entropy_reduction, lower_level[0], upper_level[0]); - for(size_t i = 0; i < batch_size; i++) + T* d_input; + counter_type* d_histogram[ActiveChannels]; + HIP_CHECK(hipMalloc(&d_input, items * Channels * sizeof(T))); + for(unsigned int channel = 0; channel < ActiveChannels; channel++) + { + HIP_CHECK(hipMalloc(&d_histogram[channel], Bins * sizeof(counter_type))); + } + HIP_CHECK( + hipMemcpy(d_input, input.data(), items * Channels * sizeof(T), hipMemcpyHostToDevice)); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { HIP_CHECK((hipcub::DeviceHistogram::MultiHistogramEven( - d_temporary_storage, - temporary_storage_bytes, + d_temp_storage, + temp_storage_bytes, d_input, d_histogram, num_levels, lower_level, upper_level, - int(size), + int(items), stream))); - } + }; + + launch(); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * size * Channels * sizeof(T)); - state.SetItemsProcessed(state.iterations() * batch_size * size * Channels); + state.set_items(items * Channels); + state.add_writes(items * Channels); - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_input)); - for(unsigned int channel = 0; channel < ActiveChannels; channel++) - { - HIP_CHECK(hipFree(d_histogram[channel])); + state.run(launch); + + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_input)); + for(unsigned int channel = 0; channel < ActiveChannels; channel++) + { + HIP_CHECK(hipFree(d_histogram[channel])); + } } -} -template -void run_range_benchmark(benchmark::State& state, size_t bins, hipStream_t stream, size_t size) + int m_entropy_reduction; +}; + +template +class range_benchmark : public primbench::benchmark_interface { - using counter_type = unsigned int; - - // Generate data - std::vector input = benchmark_utils::get_random_data(size, 0, bins); - - std::vector levels(bins + 1); - std::iota(levels.begin(), levels.end(), static_cast(0)); - - T* d_input; - T* d_levels; - counter_type* d_histogram; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMalloc(&d_levels, (bins + 1) * sizeof(T))); - HIP_CHECK(hipMalloc(&d_histogram, size * sizeof(counter_type))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_levels, levels.data(), (bins + 1) * sizeof(T), hipMemcpyHostToDevice)); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - HIP_CHECK(hipcub::DeviceHistogram::HistogramRange(d_temporary_storage, - temporary_storage_bytes, - d_input, - d_histogram, - bins + 1, - d_levels, - int(size), - stream)); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) + primbench::json meta() const override { - HIP_CHECK(hipcub::DeviceHistogram::HistogramRange(d_temporary_storage, - temporary_storage_bytes, - d_input, - d_histogram, - bins + 1, - d_levels, - int(size), - stream)); + return primbench::json{} + .add("algo", "device_histogram") + .add("subalgo", "range") + .add("lvl", "device") + .add("data_type", primbench::name()) + .add("bin_count", Bins); } - HIP_CHECK(hipDeviceSynchronize()); - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); + const size_t items = state.size; + const auto& stream = state.stream; - for(size_t i = 0; i < batch_size; i++) + using counter_type = unsigned int; + + // Generate data + std::vector input = benchmark_utils::get_random_data(items, 0, Bins); + + std::vector levels(Bins + 1); + std::iota(levels.begin(), levels.end(), static_cast(0)); + + T* d_input; + T* d_levels; + counter_type* d_histogram; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK(hipMalloc(&d_levels, (Bins + 1) * sizeof(T))); + HIP_CHECK(hipMalloc(&d_histogram, items * sizeof(counter_type))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK( + hipMemcpy(d_levels, levels.data(), (Bins + 1) * sizeof(T), hipMemcpyHostToDevice)); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { - HIP_CHECK(hipcub::DeviceHistogram::HistogramRange(d_temporary_storage, - temporary_storage_bytes, + HIP_CHECK(hipcub::DeviceHistogram::HistogramRange(d_temp_storage, + temp_storage_bytes, d_input, d_histogram, - bins + 1, + Bins + 1, d_levels, - int(size), + int(items), stream)); - } - HIP_CHECK(hipDeviceSynchronize()); + }; - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * batch_size * size); + launch(); - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_levels)); - HIP_CHECK(hipFree(d_histogram)); -} + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); -template -void run_multi_range_benchmark(benchmark::State& state, - size_t bins, - hipStream_t stream, - size_t size) -{ - using counter_type = unsigned int; + state.set_items(items); + state.add_writes(items); - // Number of levels for a single channel - const int num_levels_channel = bins + 1; - int num_levels[ActiveChannels]; - std::vector levels[ActiveChannels]; - for(unsigned int channel = 0; channel < ActiveChannels; channel++) - { - levels[channel].resize(num_levels_channel); - std::iota(levels[channel].begin(), levels[channel].end(), static_cast(0)); - num_levels[channel] = num_levels_channel; - } - - // Generate data - std::vector input = benchmark_utils::get_random_data(size * Channels, 0, bins); + state.run(launch); - T* d_input; - T* d_levels[ActiveChannels]; - counter_type* d_histogram[ActiveChannels]; - HIP_CHECK(hipMalloc(&d_input, size * Channels * sizeof(T))); - for(unsigned int channel = 0; channel < ActiveChannels; channel++) - { - HIP_CHECK(hipMalloc(&d_levels[channel], num_levels_channel * sizeof(T))); - HIP_CHECK(hipMalloc(&d_histogram[channel], size * sizeof(counter_type))); + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_levels)); + HIP_CHECK(hipFree(d_histogram)); } +}; - HIP_CHECK(hipMemcpy(d_input, input.data(), size * Channels * sizeof(T), hipMemcpyHostToDevice)); - for(unsigned int channel = 0; channel < ActiveChannels; channel++) +template +class multi_range_benchmark : public primbench::benchmark_interface +{ + primbench::json meta() const override { - HIP_CHECK(hipMemcpy(d_levels[channel], - levels[channel].data(), - num_levels_channel * sizeof(T), - hipMemcpyHostToDevice)); + return primbench::json{} + .add("algo", "device_histogram") + .add("subalgo", "multi_range") + .add("lvl", "device") + .add("data_type", primbench::name()) + .add("channels", Channels) + .add("active_channels", ActiveChannels) + .add("bin_count", Bins); } - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - HIP_CHECK((hipcub::DeviceHistogram::MultiHistogramRange( - d_temporary_storage, - temporary_storage_bytes, - d_input, - d_histogram, - num_levels, - d_levels, - int(size), - stream))); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) + void run(primbench::state& state) override { - HIP_CHECK((hipcub::DeviceHistogram::MultiHistogramRange( - d_temporary_storage, - temporary_storage_bytes, - d_input, - d_histogram, - num_levels, - d_levels, - int(size), - stream))); - } - HIP_CHECK(hipDeviceSynchronize()); + const size_t items = state.size; + const auto& stream = state.stream; - for(auto _ : state) - { - auto start = std::chrono::high_resolution_clock::now(); + using counter_type = unsigned int; + + // Number of levels for a single channel + const int num_levels_channel = Bins + 1; + int num_levels[ActiveChannels]; + std::vector levels[ActiveChannels]; + for(unsigned int channel = 0; channel < ActiveChannels; channel++) + { + levels[channel].resize(num_levels_channel); + std::iota(levels[channel].begin(), levels[channel].end(), static_cast(0)); + num_levels[channel] = num_levels_channel; + } + + // Generate data + std::vector input = benchmark_utils::get_random_data(items * Channels, 0, Bins); + + T* d_input; + T* d_levels[ActiveChannels]; + counter_type* d_histogram[ActiveChannels]; + HIP_CHECK(hipMalloc(&d_input, items * Channels * sizeof(T))); + for(unsigned int channel = 0; channel < ActiveChannels; channel++) + { + HIP_CHECK(hipMalloc(&d_levels[channel], num_levels_channel * sizeof(T))); + HIP_CHECK(hipMalloc(&d_histogram[channel], items * sizeof(counter_type))); + } + + HIP_CHECK( + hipMemcpy(d_input, input.data(), items * Channels * sizeof(T), hipMemcpyHostToDevice)); + for(unsigned int channel = 0; channel < ActiveChannels; channel++) + { + HIP_CHECK(hipMemcpy(d_levels[channel], + levels[channel].data(), + num_levels_channel * sizeof(T), + hipMemcpyHostToDevice)); + } + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; - for(size_t i = 0; i < batch_size; i++) + const auto launch = [&] { HIP_CHECK((hipcub::DeviceHistogram::MultiHistogramRange( - d_temporary_storage, - temporary_storage_bytes, + d_temp_storage, + temp_storage_bytes, d_input, d_histogram, num_levels, d_levels, - int(size), + int(items), stream))); - } + }; + + launch(); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * size * Channels * sizeof(T)); - state.SetItemsProcessed(state.iterations() * batch_size * size * Channels); + state.set_items(items * Channels); + state.add_writes(items * Channels); - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_input)); - for(unsigned int channel = 0; channel < ActiveChannels; channel++) - { - HIP_CHECK(hipFree(d_levels[channel])); - HIP_CHECK(hipFree(d_histogram[channel])); + state.run(launch); + + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_input)); + for(unsigned int channel = 0; channel < ActiveChannels; channel++) + { + HIP_CHECK(hipFree(d_levels[channel])); + HIP_CHECK(hipFree(d_histogram[channel])); + } } -} +}; template struct num_limits { static constexpr T max() { - return std::numeric_limits::max(); + return _HIPCUB_STD::numeric_limits::max(); }; }; @@ -477,18 +420,10 @@ struct num_limits<__half> }; }; -#define CREATE_EVEN_BENCHMARK(VECTOR, T, BINS, SCALE) \ - if(num_limits::max() > BINS * SCALE) \ - { \ - VECTOR.push_back(benchmark::RegisterBenchmark( \ - std::string("device_histogram_even" \ - "." \ - "(entropy_percent:" \ - + std::to_string(get_entropy_percents(entropy_reduction)) \ - + "%,bin_count:" + std::to_string(BINS) + " bins)") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_even_benchmark(state, BINS, SCALE, entropy_reduction, stream, size); })); \ +#define CREATE_EVEN_BENCHMARK(VECTOR, T, BINS, SCALE) \ + if(num_limits::max() > BINS * SCALE) \ + { \ + executor.queue>(entropy_reduction); \ } #define BENCHMARK_TYPE(VECTOR, T) \ @@ -499,13 +434,11 @@ struct num_limits<__half> CREATE_EVEN_BENCHMARK(VECTOR, T, 256, 10); \ CREATE_EVEN_BENCHMARK(VECTOR, T, 65536, 1) -void add_even_benchmarks(std::vector& benchmarks, - hipStream_t stream, - size_t size) +void add_even_benchmarks(primbench::executor& executor) { for(int entropy_reduction : entropy_reductions) { - BENCHMARK_TYPE(benchmarks, long long); + BENCHMARK_TYPE(benchmarks, int64_t); BENCHMARK_TYPE(benchmarks, int); BENCHMARK_TYPE(benchmarks, unsigned short); BENCHMARK_TYPE(benchmarks, uint8_t); @@ -519,140 +452,68 @@ void add_even_benchmarks(std::vector& benchmark }; } -#define CREATE_MULTI_EVEN_BENCHMARK(CHANNELS, ACTIVE_CHANNELS, T, BINS, SCALE) \ - benchmark::RegisterBenchmark( \ - std::string("device_multi_histogram_even" \ - "." \ - "(entropy_percent:" \ - + std::to_string(get_entropy_percents(entropy_reduction)) \ - + "%,bin_count:" + std::to_string(BINS) + " bins)") \ - .c_str(), \ - [=](benchmark::State& state) \ - { \ - run_multi_even_benchmark(state, \ - BINS, \ - SCALE, \ - entropy_reduction, \ - stream, \ - size); \ - }) - -void add_multi_even_benchmarks(std::vector& benchmarks, - hipStream_t stream, - size_t size) +#define CREATE_MULTI_EVEN_BENCHMARK(CHANNELS, ACTIVE_CHANNELS, T, BINS, SCALE) \ + executor.queue>( \ + entropy_reduction); + +void add_multi_even_benchmarks(primbench::executor& executor) { for(int entropy_reduction : entropy_reductions) { - std::vector bs = { - CREATE_MULTI_EVEN_BENCHMARK(4, 3, int, 10, 1234), - CREATE_MULTI_EVEN_BENCHMARK(4, 3, int, 100, 1234), + CREATE_MULTI_EVEN_BENCHMARK(4, 3, int, 10, 1234); + CREATE_MULTI_EVEN_BENCHMARK(4, 3, int, 100, 1234); - CREATE_MULTI_EVEN_BENCHMARK(4, 3, unsigned char, 16, 10), - CREATE_MULTI_EVEN_BENCHMARK(4, 3, unsigned char, 256, 1), + CREATE_MULTI_EVEN_BENCHMARK(4, 3, unsigned char, 16, 10); + CREATE_MULTI_EVEN_BENCHMARK(4, 3, unsigned char, 256, 1); - CREATE_MULTI_EVEN_BENCHMARK(4, 3, unsigned short, 16, 10), - CREATE_MULTI_EVEN_BENCHMARK(4, 3, unsigned short, 256, 10), - CREATE_MULTI_EVEN_BENCHMARK(4, 3, unsigned short, 65536, 1), - }; - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + CREATE_MULTI_EVEN_BENCHMARK(4, 3, unsigned short, 16, 10); + CREATE_MULTI_EVEN_BENCHMARK(4, 3, unsigned short, 256, 10); + CREATE_MULTI_EVEN_BENCHMARK(4, 3, unsigned short, 65536, 1); }; } -#define CREATE_RANGE_BENCHMARK(T, BINS) \ - benchmark::RegisterBenchmark(std::string("device_histogram_range" \ - "." \ - "(bin_count:" \ - + std::to_string(BINS) + " bins)") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_range_benchmark(state, BINS, stream, size); }) - -#define BENCHMARK_RANGE_TYPE(T) \ - CREATE_RANGE_BENCHMARK(T, 10), CREATE_RANGE_BENCHMARK(T, 100), \ - CREATE_RANGE_BENCHMARK(T, 1000), CREATE_RANGE_BENCHMARK(T, 10000), \ - CREATE_RANGE_BENCHMARK(T, 100000), CREATE_RANGE_BENCHMARK(T, 1000000) - -void add_range_benchmarks(std::vector& benchmarks, - hipStream_t stream, - size_t size) +#define CREATE_RANGE_BENCHMARK(T, BINS) executor.queue>(); + +#define BENCHMARK_RANGE_TYPE(T) \ + CREATE_RANGE_BENCHMARK(T, 10); \ + CREATE_RANGE_BENCHMARK(T, 100); \ + CREATE_RANGE_BENCHMARK(T, 1000); \ + CREATE_RANGE_BENCHMARK(T, 10000); \ + CREATE_RANGE_BENCHMARK(T, 100000); \ + CREATE_RANGE_BENCHMARK(T, 1000000) + +void add_range_benchmarks(primbench::executor& executor) { - std::vector bs - = {BENCHMARK_RANGE_TYPE(float), BENCHMARK_RANGE_TYPE(double)}; - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + BENCHMARK_RANGE_TYPE(float); + BENCHMARK_RANGE_TYPE(double); } -#define CREATE_MULTI_RANGE_BENCHMARK(CHANNELS, ACTIVE_CHANNELS, T, BINS) \ - benchmark::RegisterBenchmark( \ - std::string("device_multi_histogram_range" \ - ".(bin_count:" \ - + std::to_string(BINS) + " bins)") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_multi_range_benchmark(state, BINS, stream, size); }) - -void add_multi_range_benchmarks(std::vector& benchmarks, - hipStream_t stream, - size_t size) +#define CREATE_MULTI_RANGE_BENCHMARK(CHANNELS, ACTIVE_CHANNELS, T, BINS) \ + executor.queue>() + +void add_multi_range_benchmarks(primbench::executor& executor) { - std::vector bs = { - CREATE_MULTI_RANGE_BENCHMARK(4, 3, float, 10), - CREATE_MULTI_RANGE_BENCHMARK(4, 3, float, 100), - CREATE_MULTI_RANGE_BENCHMARK(4, 3, float, 1000), - CREATE_MULTI_RANGE_BENCHMARK(4, 3, float, 10000), - CREATE_MULTI_RANGE_BENCHMARK(4, 3, float, 100000), - CREATE_MULTI_RANGE_BENCHMARK(4, 3, float, 1000000), - }; - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + CREATE_MULTI_RANGE_BENCHMARK(4, 3, float, 10); + CREATE_MULTI_RANGE_BENCHMARK(4, 3, float, 100); + CREATE_MULTI_RANGE_BENCHMARK(4, 3, float, 1000); + CREATE_MULTI_RANGE_BENCHMARK(4, 3, float, 10000); + CREATE_MULTI_RANGE_BENCHMARK(4, 3, float, 100000); + CREATE_MULTI_RANGE_BENCHMARK(4, 3, float, 1000000); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_device_histogram" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; - // Add benchmarks - std::vector benchmarks; - add_even_benchmarks(benchmarks, stream, size); - add_multi_even_benchmarks(benchmarks, stream, size); - add_range_benchmarks(benchmarks, stream, size); - add_multi_range_benchmarks(benchmarks, stream, size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::executor executor(argc, argv, settings); - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + // Add benchmarks + add_even_benchmarks(executor); + add_multi_even_benchmarks(executor); + add_range_benchmarks(executor); + add_multi_range_benchmarks(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_memory.cpp b/projects/hipcub/benchmark/benchmark_device_memory.cpp index 1e62167a7a1a..c33826265a7b 100644 --- a/projects/hipcub/benchmark/benchmark_device_memory.cpp +++ b/projects/hipcub/benchmark/benchmark_device_memory.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,7 +20,7 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" #include #include @@ -81,12 +81,12 @@ struct operation (void)storage; (void)global_mem_output; -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(unsigned int i = 0; i < ItemsPerThread; i++) { input[i] = input[i] + 666; constexpr unsigned int repeats = 30; -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(unsigned int j = 0; j < repeats; j++) { input[i] = input[i] * (input[j % ItemsPerThread]); @@ -113,7 +113,7 @@ struct operation // sync before re-using shared memory from load __syncthreads(); - block_scan_type(storage).InclusiveScan(input, input, hipcub::Sum()); + block_scan_type(storage).InclusiveScan(input, input, benchmark_utils::plus{}); } }; @@ -134,7 +134,7 @@ struct operation const unsigned int index = threadIdx.x * ItemsPerThread + blockIdx.x * blockDim.x * ItemsPerThread; -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(unsigned int i = 0; i < ItemsPerThread; i++) { atomicAdd(&global_mem_output[index + i], T(666)); @@ -159,7 +159,7 @@ struct operation const unsigned int index = (threadIdx.x % warpSize) * ItemsPerThread + blockIdx.x * blockDim.x * ItemsPerThread; -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(unsigned int i = 0; i < ItemsPerThread; i++) { atomicAdd(&global_mem_output[index + i], T(666)); @@ -183,7 +183,7 @@ struct operation (void)input; const unsigned int index = threadIdx.x * ItemsPerThread; -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(unsigned int i = 0; i < ItemsPerThread; i++) { atomicAdd(&global_mem_output[index + i], T(666)); @@ -245,7 +245,8 @@ template -__global__ __launch_bounds__(BlockSize) void operation_kernel(T* input, T* output, CustomOp op) +__global__ __launch_bounds__(BlockSize) +void operation_kernel(T* input, T* output, CustomOp op) { using mem_op = memory_operation; using load_type = hipcub::BlockLoad; @@ -265,236 +266,187 @@ __global__ __launch_bounds__(BlockSize) void operation_kernel(T* input, T* outpu load_type(storage.load).Load(input + offset, items); op(storage.operand, items, output); + // sync before re-using shared memory from load or from operand __syncthreads(); store_type(storage.store).Store(output + offset, items); } +inline const char* get_name(memory_operation_method method) +{ + switch(method) + { + case memory_operation_method::direct: return "direct"; + case memory_operation_method::striped: return "striped"; + case memory_operation_method::vectorize: return "vectorize"; + case memory_operation_method::transpose: return "transpose"; + case memory_operation_method::warp_transpose: return "warp_transpose"; + } + + return "unknown memory operation method"; +} + +inline const char* get_name(kernel_operation method) +{ + switch(method) + { + case kernel_operation::no_operation: return "no_kernel_op"; + case kernel_operation::block_scan: return "block_scan"; + case kernel_operation::custom_operation: return "custom_kernel_op"; + case kernel_operation::atomics_no_collision: return "atomics_no_collision"; + case kernel_operation::atomics_inter_block_collision: + return "atomics_inter_block_collision"; + case kernel_operation::atomics_inter_warp_collision: return "atomics_inter_warp_collision"; + } + + return "unknown kernel operation method"; +} + template -void run_benchmark(benchmark::State& state, size_t size, const hipStream_t stream) +class device_memory_benchmark : public primbench::benchmark_interface { - const size_t grid_size = size / (BlockSize * ItemsPerThread); - std::vector input - = benchmark_utils::get_random_data(size, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); - - T* d_input; - T* d_output; - HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(T))); - HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), size * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - operation selected_operation; - - // Warm-up - for(size_t i = 0; i < 10; i++) + primbench::json meta() const override { - hipLaunchKernelGGL(HIP_KERNEL_NAME(operation_kernel), - dim3(grid_size), - dim3(BlockSize), - 0, - stream, - d_input, - d_output, - selected_operation); + return primbench::json{} + .add("algo", "device_memory") + .add("subalgo", get_name(MemOp)) + .add("lvl", "device") + .add("data_type", primbench::name()) + .add("items_per_thread", ItemsPerThread) + .add("kernel_op", get_name(KernelOp)) + .add("block_size", BlockSize); + ; } - HIP_CHECK(hipDeviceSynchronize()); - - // HIP events creation - hipEvent_t start, stop; - HIP_CHECK(hipEventCreate(&start)); - HIP_CHECK(hipEventCreate(&stop)); - const unsigned int batch_size = 10; - for(auto _ : state) + void run(primbench::state& state) override { - // Record start event - HIP_CHECK(hipEventRecord(start, stream)); + const size_t bytes = state.size; + const auto& stream = state.stream; - for(size_t i = 0; i < batch_size; i++) - { - hipLaunchKernelGGL( - HIP_KERNEL_NAME(operation_kernel), - dim3(grid_size), - dim3(BlockSize), - 0, - stream, - d_input, - d_output, - selected_operation); - } + const size_t items = bytes / sizeof(T); - // Record stop event and wait until it completes - HIP_CHECK(hipEventRecord(stop, stream)); - HIP_CHECK(hipEventSynchronize(stop)); + const size_t grid_size = items / size_t(BlockSize * ItemsPerThread); + std::vector input + = benchmark_utils::get_random_data(items, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); - float elapsed_mseconds; - HIP_CHECK(hipEventElapsedTime(&elapsed_mseconds, start, stop)); - state.SetIterationTime(elapsed_mseconds / 1000); - } + T* d_input; + T* d_output; + HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), items * sizeof(T))); + HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), items * sizeof(T))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK(hipDeviceSynchronize()); - // Destroy HIP events - HIP_CHECK(hipEventDestroy(start)); - HIP_CHECK(hipEventDestroy(stop)); + operation selected_operation; - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * batch_size * size); + state.set_items(items); + state.add_writes(items); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); -} + state.run( + [&] + { + hipLaunchKernelGGL( + HIP_KERNEL_NAME(operation_kernel), + dim3(grid_size), + dim3(BlockSize), + 0, + stream, + d_input, + d_output, + selected_operation); + }); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); + } +}; template -void run_benchmark_memcpy(benchmark::State& state, size_t size, const hipStream_t stream) +class memcpy_benchmark : public primbench::benchmark_interface { - // Allocate device buffers - // Note: since this benchmark only tests memcpy performance between device - // buffers, we don't really need to copy data into these from the host - - // whatever happens to be in memory will suffice. - T* d_input; - T* d_output; - HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), size * sizeof(T))); - HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), size * sizeof(T))); - - // Warm-up - for(size_t i = 0; i < 10; i++) + primbench::json meta() const override { - HIP_CHECK(hipMemcpy(d_output, d_input, size * sizeof(T), hipMemcpyDeviceToDevice)); + return primbench::json{} + .add("algo", "device_memory") + .add("subalgo", "memcpy") + .add("lvl", "device") + .add("data_type", primbench::name()); } - HIP_CHECK(hipDeviceSynchronize()); - // HIP events creation - hipEvent_t start, stop; - HIP_CHECK(hipEventCreate(&start)); - HIP_CHECK(hipEventCreate(&stop)); - - const unsigned int batch_size = 10; - for(auto _ : state) + void run(primbench::state& state) override { - // Record start event - HIP_CHECK(hipEventRecord(start, stream)); - - for(size_t i = 0; i < batch_size; i++) - { - HIP_CHECK(hipMemcpy(d_output, d_input, size * sizeof(T), hipMemcpyDeviceToDevice)); - } - - // Record stop event and wait until it completes - HIP_CHECK(hipEventRecord(stop, stream)); - HIP_CHECK(hipEventSynchronize(stop)); - - float elapsed_mseconds; - HIP_CHECK(hipEventElapsedTime(&elapsed_mseconds, start, stop)); - state.SetIterationTime(elapsed_mseconds / 1000); + const size_t bytes = state.size; + const auto& stream = state.stream; + + const size_t items = bytes / sizeof(T); + + // Allocate device buffers + // Note: since this benchmark only tests memcpy performance between device + // buffers, we don't really need to copy data into these from the host - + // whatever happens to be in memory will suffice. + T* d_input; + T* d_output; + HIP_CHECK(hipMalloc(reinterpret_cast(&d_input), items * sizeof(T))); + HIP_CHECK(hipMalloc(reinterpret_cast(&d_output), items * sizeof(T))); + HIP_CHECK(hipDeviceSynchronize()); + + state.set_items(items); + state.add_writes(items); + + state.run( + [&] + { + // We deliberately call hipMemcpyAsync() instead of hipMemcpy() here, + // since hipMemcpy() uses the slow default legacy stream (stream 0). + HIP_CHECK(hipMemcpyAsync(d_output, + d_input, + items * sizeof(T), + hipMemcpyDeviceToDevice, + stream)); + }); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); } +}; - // Destroy HIP events - HIP_CHECK(hipEventDestroy(start)); - HIP_CHECK(hipEventDestroy(stop)); +#define QUEUE_IPT(METHOD, OPERATION, IPT) \ + executor.queue>() - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * batch_size * size); +#define QUEUE_BLOCK_SIZE(MEM_OP, OP) \ + QUEUE_IPT(MEM_OP, OP, 1); \ + QUEUE_IPT(MEM_OP, OP, 2); \ + QUEUE_IPT(MEM_OP, OP, 4); \ + QUEUE_IPT(MEM_OP, OP, 8) - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); -} - -#define CREATE_BENCHMARK_IPT(METHOD, OPERATION, T, SIZE, BS, IPT) \ - benchmarks.push_back(benchmark::RegisterBenchmark( \ - std::string("device_memory.") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_benchmark(state, SIZE, stream); })); - -#define CREATE_BENCHMARK_MEMCPY(T, SIZE) \ - benchmarks.push_back(benchmark::RegisterBenchmark( \ - std::string("device_memory_memcpy.").c_str(), \ - [=](benchmark::State& state) { run_benchmark_memcpy(state, SIZE, stream); })); - -// clang-format off -#define CREATE_BENCHMARK_BLOCK_SIZE(MEM_OP, OP, TYPE, SIZE, BLOCK_SIZE) \ - CREATE_BENCHMARK_IPT(MEM_OP, OP, TYPE, SIZE, BLOCK_SIZE, 1) \ - CREATE_BENCHMARK_IPT(MEM_OP, OP, TYPE, SIZE, BLOCK_SIZE, 2) \ - CREATE_BENCHMARK_IPT(MEM_OP, OP, TYPE, SIZE, BLOCK_SIZE, 4) \ - CREATE_BENCHMARK_IPT(MEM_OP, OP, TYPE, SIZE, BLOCK_SIZE, 8) - -#define CREATE_BENCHMARK_MEM_OP(MEM_OP, OP, TYPE, SIZE) \ - CREATE_BENCHMARK_BLOCK_SIZE(MEM_OP, OP, TYPE, SIZE, 256) - -#define CREATE_BENCHMARK(OP, TYPE, SIZE) \ - CREATE_BENCHMARK_MEM_OP(direct, OP, TYPE, SIZE) \ - CREATE_BENCHMARK_MEM_OP(striped, OP, TYPE, SIZE) \ - CREATE_BENCHMARK_MEM_OP(vectorize, OP, TYPE, SIZE) \ - CREATE_BENCHMARK_MEM_OP(transpose, OP, TYPE, SIZE) \ - CREATE_BENCHMARK_MEM_OP(warp_transpose, OP, TYPE, SIZE) -// clang-format on - -template -constexpr unsigned int megabytes(unsigned int size) -{ - return (size * (1024 * 1024 / sizeof(T))); -} +#define QUEUE(OP) \ + QUEUE_BLOCK_SIZE(direct, OP); \ + QUEUE_BLOCK_SIZE(striped, OP); \ + QUEUE_BLOCK_SIZE(vectorize, OP); \ + QUEUE_BLOCK_SIZE(transpose, OP); \ + QUEUE_BLOCK_SIZE(warp_transpose, OP) int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const int trials = parser.get("trials"); - - std::cout << "benchmark_device_memory" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - - // Simple memory copy from device to device, not running a kernel - CREATE_BENCHMARK_MEMCPY(int, megabytes(128)) - - // clang-format off - CREATE_BENCHMARK(no_operation, int, megabytes(128)) - CREATE_BENCHMARK(block_scan, int, megabytes(128)) - CREATE_BENCHMARK(custom_operation, int, megabytes(128)) - CREATE_BENCHMARK(atomics_no_collision, int, megabytes(128)) - CREATE_BENCHMARK(atomics_inter_block_collision, int, megabytes(128)) - CREATE_BENCHMARK(atomics_inter_warp_collision, int, megabytes(128)) - // clang-format on - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; + settings.size = 128 * primbench::MiB; // In bytes + settings.min_gpu_ms_per_batch = 100; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + primbench::executor executor(argc, argv, settings); + + executor.queue>(); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); + QUEUE(no_operation); + QUEUE(block_scan); + QUEUE(custom_operation); + QUEUE(atomics_no_collision); + QUEUE(atomics_inter_block_collision); + QUEUE(atomics_inter_warp_collision); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_merge.cpp b/projects/hipcub/benchmark/benchmark_device_merge.cpp index e22d6ea925fd..92fdf1ae2383 100644 --- a/projects/hipcub/benchmark/benchmark_device_merge.cpp +++ b/projects/hipcub/benchmark/benchmark_device_merge.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,345 +20,248 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif - -const unsigned int batch_size = 10; -const unsigned int warmup_size = 5; - -template +template struct CompareFunction { HIPCUB_HOST_DEVICE - inline constexpr bool - operator()(const key_type& a, const key_type& b) + inline constexpr bool operator()(const Key& a, const Key& b) { return a < b; } }; template -void run_merge_keys_benchmark(benchmark::State& state, hipStream_t stream, size_t size) +class merge_keys_benchmark : public primbench::benchmark_interface { - using key_type = Key; - - CompareFunction compare_function; - - const size_t size1 = size / 2; - const size_t size2 = size - size1; - - std::vector keys_input1 = benchmark_utils::get_random_data( - size1, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); - - std::vector keys_input2 = benchmark_utils::get_random_data( - size2, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); - - std::sort(keys_input1.begin(), keys_input1.end(), compare_function); - std::sort(keys_input2.begin(), keys_input2.end(), compare_function); - - key_type* d_keys_input1; - HIP_CHECK(hipMalloc(&d_keys_input1, size1 * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_keys_input1, - keys_input1.data(), - size1 * sizeof(key_type), - hipMemcpyHostToDevice)); - - key_type* d_keys_input2; - HIP_CHECK(hipMalloc(&d_keys_input2, size2 * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_keys_input2, - keys_input2.data(), - size2 * sizeof(key_type), - hipMemcpyHostToDevice)); - - key_type* d_keys_output; - HIP_CHECK(hipMalloc(&d_keys_output, size * sizeof(key_type))); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - HIP_CHECK(hipcub::DeviceMerge::MergeKeys(d_temporary_storage, - temporary_storage_bytes, - d_keys_input1, - size1, - d_keys_input2, - size2, - d_keys_output, - compare_function, - stream)); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) + primbench::json meta() const override { - HIP_CHECK(hipcub::DeviceMerge::MergeKeys(d_temporary_storage, - temporary_storage_bytes, - d_keys_input1, - size1, - d_keys_input2, - size2, - d_keys_output, - compare_function, - stream)); + return primbench::json{} + .add("algo", "device_merge") + .add("subalgo", "merge_keys") + .add("lvl", "device") + .add("key_data_type", primbench::name()); } - HIP_CHECK(hipDeviceSynchronize()); - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); + const size_t items = state.size; + const auto& stream = state.stream; + + CompareFunction compare_function; + + const size_t items1 = items / 2; + const size_t items2 = items - items1; + + std::vector keys_input1 + = benchmark_utils::get_random_data(items1, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); + + std::vector keys_input2 + = benchmark_utils::get_random_data(items2, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); + + std::sort(keys_input1.begin(), keys_input1.end(), compare_function); + std::sort(keys_input2.begin(), keys_input2.end(), compare_function); + + Key* d_keys_input1; + HIP_CHECK(hipMalloc(&d_keys_input1, items1 * sizeof(Key))); + HIP_CHECK(hipMemcpy(d_keys_input1, + keys_input1.data(), + items1 * sizeof(Key), + hipMemcpyHostToDevice)); - for(size_t i = 0; i < batch_size; i++) + Key* d_keys_input2; + HIP_CHECK(hipMalloc(&d_keys_input2, items2 * sizeof(Key))); + HIP_CHECK(hipMemcpy(d_keys_input2, + keys_input2.data(), + items2 * sizeof(Key), + hipMemcpyHostToDevice)); + + Key* d_keys_output; + HIP_CHECK(hipMalloc(&d_keys_output, items * sizeof(Key))); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { - HIP_CHECK(hipcub::DeviceMerge::MergeKeys(d_temporary_storage, - temporary_storage_bytes, + HIP_CHECK(hipcub::DeviceMerge::MergeKeys(d_temp_storage, + temp_storage_bytes, d_keys_input1, - size1, + items1, d_keys_input2, - size2, + items2, d_keys_output, compare_function, stream)); - } + }; + + launch(); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(key_type)); - state.SetItemsProcessed(state.iterations() * batch_size * size); + state.set_items(items); + state.add_writes(items); - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_keys_input1)); - HIP_CHECK(hipFree(d_keys_input2)); - HIP_CHECK(hipFree(d_keys_output)); -} + state.run(launch); + + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_keys_input1)); + HIP_CHECK(hipFree(d_keys_input2)); + HIP_CHECK(hipFree(d_keys_output)); + } +}; template -void run_merge_pairs_benchmark(benchmark::State& state, hipStream_t stream, size_t size) +class merge_pairs_benchmark : public primbench::benchmark_interface { - using key_type = Key; - using value_type = Value; - - CompareFunction compare_function; - - const size_t size1 = size / 2; - const size_t size2 = size - size1; - - std::vector keys_input1 = benchmark_utils::get_random_data( - size1, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); - std::vector keys_input2 = benchmark_utils::get_random_data( - size2, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); - - std::sort(keys_input1.begin(), keys_input1.end(), compare_function); - std::sort(keys_input2.begin(), keys_input2.end(), compare_function); - - key_type* d_keys_input1; - HIP_CHECK(hipMalloc(&d_keys_input1, size1 * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_keys_input1, - keys_input1.data(), - size1 * sizeof(key_type), - hipMemcpyHostToDevice)); - - key_type* d_keys_input2; - HIP_CHECK(hipMalloc(&d_keys_input2, size2 * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_keys_input2, - keys_input2.data(), - size2 * sizeof(key_type), - hipMemcpyHostToDevice)); - - key_type* d_keys_output; - HIP_CHECK(hipMalloc(&d_keys_output, size * sizeof(key_type))); - - std::vector values_input1(size1); - std::iota(values_input1.begin(), values_input1.end(), 0); - value_type* d_values_input1; - HIP_CHECK(hipMalloc(&d_values_input1, size1 * sizeof(value_type))); - HIP_CHECK(hipMemcpy(d_values_input1, - values_input1.data(), - size1 * sizeof(value_type), - hipMemcpyHostToDevice)); - - std::vector values_input2(size2); - std::iota(values_input2.begin(), values_input2.end(), size1); - value_type* d_values_input2; - HIP_CHECK(hipMalloc(&d_values_input2, size2 * sizeof(value_type))); - HIP_CHECK(hipMemcpy(d_values_input2, - values_input2.data(), - size2 * sizeof(value_type), - hipMemcpyHostToDevice)); - - value_type* d_values_output; - HIP_CHECK(hipMalloc(&d_values_output, size * sizeof(value_type))); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - HIP_CHECK(hipcub::DeviceMerge::MergePairs(d_temporary_storage, - temporary_storage_bytes, - d_keys_input1, - d_values_input1, - size1, - d_keys_input2, - d_values_input2, - size2, - d_keys_output, - d_values_output, - compare_function, - stream)); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) + primbench::json meta() const override { - HIP_CHECK(hipcub::DeviceMerge::MergePairs(d_temporary_storage, - temporary_storage_bytes, - d_keys_input1, - d_values_input1, - size1, - d_keys_input2, - d_values_input2, - size2, - d_keys_output, - d_values_output, - compare_function, - stream)); + return primbench::json{} + .add("algo", "device_merge") + .add("subalgo", "merge_pairs") + .add("lvl", "device") + .add("key_data_type", primbench::name()) + .add("value_data_type", primbench::name()); } - HIP_CHECK(hipDeviceSynchronize()); - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); - - for(size_t i = 0; i < batch_size; i++) + const size_t items = state.size; + const auto& stream = state.stream; + + CompareFunction compare_function; + + const size_t items1 = items / 2; + const size_t items2 = items - items1; + + std::vector keys_input1 + = benchmark_utils::get_random_data(items1, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); + std::vector keys_input2 + = benchmark_utils::get_random_data(items2, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); + + std::sort(keys_input1.begin(), keys_input1.end(), compare_function); + std::sort(keys_input2.begin(), keys_input2.end(), compare_function); + + Key* d_keys_input1; + HIP_CHECK(hipMalloc(&d_keys_input1, items1 * sizeof(Key))); + HIP_CHECK(hipMemcpy(d_keys_input1, + keys_input1.data(), + items1 * sizeof(Key), + hipMemcpyHostToDevice)); + + Key* d_keys_input2; + HIP_CHECK(hipMalloc(&d_keys_input2, items2 * sizeof(Key))); + HIP_CHECK(hipMemcpy(d_keys_input2, + keys_input2.data(), + items2 * sizeof(Key), + hipMemcpyHostToDevice)); + + Key* d_keys_output; + HIP_CHECK(hipMalloc(&d_keys_output, items * sizeof(Key))); + + std::vector values_input1(items1); + std::iota(values_input1.begin(), values_input1.end(), 0); + Value* d_values_input1; + HIP_CHECK(hipMalloc(&d_values_input1, items1 * sizeof(Value))); + HIP_CHECK(hipMemcpy(d_values_input1, + values_input1.data(), + items1 * sizeof(Value), + hipMemcpyHostToDevice)); + + std::vector values_input2(items2); + std::iota(values_input2.begin(), values_input2.end(), items1); + Value* d_values_input2; + HIP_CHECK(hipMalloc(&d_values_input2, items2 * sizeof(Value))); + HIP_CHECK(hipMemcpy(d_values_input2, + values_input2.data(), + items2 * sizeof(Value), + hipMemcpyHostToDevice)); + + Value* d_values_output; + HIP_CHECK(hipMalloc(&d_values_output, items * sizeof(Value))); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { - HIP_CHECK(hipcub::DeviceMerge::MergePairs(d_temporary_storage, - temporary_storage_bytes, + HIP_CHECK(hipcub::DeviceMerge::MergePairs(d_temp_storage, + temp_storage_bytes, d_keys_input1, d_values_input1, - size1, + items1, d_keys_input2, d_values_input2, - size2, + items2, d_keys_output, d_values_output, compare_function, stream)); - } + }; + + launch(); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + state.set_items(items); + state.add_writes(items); + state.add_writes(items); + + state.run(launch); + + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_keys_input1)); + HIP_CHECK(hipFree(d_keys_input2)); + HIP_CHECK(hipFree(d_keys_output)); + HIP_CHECK(hipFree(d_values_input1)); + HIP_CHECK(hipFree(d_values_input2)); + HIP_CHECK(hipFree(d_values_output)); } - state.SetBytesProcessed(state.iterations() * batch_size * size - * (sizeof(key_type) + sizeof(value_type))); - state.SetItemsProcessed(state.iterations() * batch_size * size); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_keys_input1)); - HIP_CHECK(hipFree(d_keys_input2)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_values_input1)); - HIP_CHECK(hipFree(d_values_input2)); - HIP_CHECK(hipFree(d_values_output)); -} +}; -#define CREATE_MERGE_KEYS_BENCHMARK(T) \ - benchmarks.push_back(benchmark::RegisterBenchmark( \ - std::string("device_merge_keys" \ - ".") \ - .c_str(), \ - [=](benchmark::State& state) { run_merge_keys_benchmark(state, stream, size); })); +#define CREATE_MERGE_KEYS_BENCHMARK(T) executor.queue>() -#define CREATE_MERGE_PAIRS_BENCHMARK(T, V) \ - benchmarks.push_back(benchmark::RegisterBenchmark( \ - std::string("device_merge_pairs<" \ - ",key_data_type:" #T ",value_data_type:" #V ">.") \ - .c_str(), \ - [=](benchmark::State& state) { run_merge_pairs_benchmark(state, stream, size); })); +#define CREATE_MERGE_PAIRS_BENCHMARK(T, V) executor.queue>() int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - - std::cout << "benchmark_device_merge" << std::endl; - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - - using custom_float2 = benchmark_utils::custom_type; - using custom_double2 = benchmark_utils::custom_type; - using custom_char_double = benchmark_utils::custom_type; - using custom_double_char = benchmark_utils::custom_type; - - CREATE_MERGE_KEYS_BENCHMARK(int) - CREATE_MERGE_KEYS_BENCHMARK(long long) - CREATE_MERGE_KEYS_BENCHMARK(int8_t) - CREATE_MERGE_KEYS_BENCHMARK(uint8_t) - CREATE_MERGE_KEYS_BENCHMARK(short) - CREATE_MERGE_KEYS_BENCHMARK(double) - CREATE_MERGE_KEYS_BENCHMARK(float) - CREATE_MERGE_KEYS_BENCHMARK(custom_float2) - CREATE_MERGE_KEYS_BENCHMARK(custom_double2) - - CREATE_MERGE_PAIRS_BENCHMARK(int, int) - CREATE_MERGE_PAIRS_BENCHMARK(long long, long long) - CREATE_MERGE_PAIRS_BENCHMARK(int8_t, int8_t) - CREATE_MERGE_PAIRS_BENCHMARK(uint8_t, uint8_t) - CREATE_MERGE_PAIRS_BENCHMARK(short, short) - CREATE_MERGE_PAIRS_BENCHMARK(custom_char_double, custom_char_double) - CREATE_MERGE_PAIRS_BENCHMARK(int, custom_double_char) - CREATE_MERGE_PAIRS_BENCHMARK(custom_double2, custom_double2) - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } - - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } - - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; + + primbench::executor executor(argc, argv, settings); + + CREATE_MERGE_KEYS_BENCHMARK(int); + CREATE_MERGE_KEYS_BENCHMARK(int64_t); + CREATE_MERGE_KEYS_BENCHMARK(int8_t); + CREATE_MERGE_KEYS_BENCHMARK(uint8_t); + CREATE_MERGE_KEYS_BENCHMARK(short); + CREATE_MERGE_KEYS_BENCHMARK(double); + CREATE_MERGE_KEYS_BENCHMARK(float); + CREATE_MERGE_KEYS_BENCHMARK(custom_float2); + CREATE_MERGE_KEYS_BENCHMARK(custom_double2); + + CREATE_MERGE_PAIRS_BENCHMARK(int, int); + CREATE_MERGE_PAIRS_BENCHMARK(int64_t, int64_t); + CREATE_MERGE_PAIRS_BENCHMARK(int8_t, int8_t); + CREATE_MERGE_PAIRS_BENCHMARK(uint8_t, uint8_t); + CREATE_MERGE_PAIRS_BENCHMARK(short, short); + CREATE_MERGE_PAIRS_BENCHMARK(custom_char_double, custom_char_double); + CREATE_MERGE_PAIRS_BENCHMARK(int, custom_double_char); + CREATE_MERGE_PAIRS_BENCHMARK(custom_double2, custom_double2); + + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_merge_sort.cpp b/projects/hipcub/benchmark/benchmark_device_merge_sort.cpp index 284892cb0b3a..a1e209c0e628 100644 --- a/projects/hipcub/benchmark/benchmark_device_merge_sort.cpp +++ b/projects/hipcub/benchmark/benchmark_device_merge_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,299 +20,206 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 32 << 20; -#endif - -const unsigned int batch_size = 10; -const unsigned int warmup_size = 5; - -template +template struct CompareFunction { HIPCUB_DEVICE - inline constexpr bool - operator()(const key_type& a, const key_type& b) + inline constexpr bool operator()(const Key& a, const Key& b) { return a < b; } }; template -void run_sort_keys_benchmark(benchmark::State& state, hipStream_t stream, size_t size) +class sort_keys_benchmark : public primbench::benchmark_interface { - using key_type = Key; - - CompareFunction compare_function; - - std::vector keys_input = benchmark_utils::get_random_data( - size, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); - - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(hipMalloc(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK(hipMalloc(&d_keys_output, size * sizeof(key_type))); - HIP_CHECK( - hipMemcpy(d_keys_input, keys_input.data(), size * sizeof(key_type), hipMemcpyHostToDevice)); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - HIP_CHECK(hipcub::DeviceMergeSort::SortKeysCopy(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - size, - compare_function, - stream)); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) + primbench::json meta() const override { - HIP_CHECK(hipcub::DeviceMergeSort::SortKeysCopy(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - size, - compare_function, - stream)); + return primbench::json{} + .add("algo", "device_merge_sort") + .add("subalgo", "sort_keys") + .add("lvl", "device") + .add("key_data_type", primbench::name()); } - HIP_CHECK(hipDeviceSynchronize()); - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); + const size_t items = state.size; + const auto& stream = state.stream; + + CompareFunction compare_function; + + std::vector keys_input + = benchmark_utils::get_random_data(items, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); + + Key* d_keys_input; + Key* d_keys_output; + HIP_CHECK(hipMalloc(&d_keys_input, items * sizeof(Key))); + HIP_CHECK(hipMalloc(&d_keys_output, items * sizeof(Key))); + HIP_CHECK( + hipMemcpy(d_keys_input, keys_input.data(), items * sizeof(Key), hipMemcpyHostToDevice)); - for(size_t i = 0; i < batch_size; i++) + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { - HIP_CHECK(hipcub::DeviceMergeSort::SortKeysCopy(d_temporary_storage, - temporary_storage_bytes, + HIP_CHECK(hipcub::DeviceMergeSort::SortKeysCopy(d_temp_storage, + temp_storage_bytes, d_keys_input, d_keys_output, - size, + items, compare_function, stream)); - } - HIP_CHECK(hipDeviceSynchronize()); + }; - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(key_type)); - state.SetItemsProcessed(state.iterations() * batch_size * size); + launch(); - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_keys_output)); -} - -template -void run_sort_pairs_benchmark(benchmark::State& state, hipStream_t stream, size_t size) -{ - using key_type = Key; - using value_type = Value; + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); - CompareFunction compare_function; + state.set_items(items); + state.add_writes(items); - std::vector keys_input = benchmark_utils::get_random_data( - size, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); + state.run(launch); - std::vector values_input(size); - for(size_t i = 0; i < size; i++) - { - values_input[i] = value_type(i); + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_keys_input)); + HIP_CHECK(hipFree(d_keys_output)); } +}; - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(hipMalloc(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK(hipMalloc(&d_keys_output, size * sizeof(key_type))); - HIP_CHECK( - hipMemcpy(d_keys_input, keys_input.data(), size * sizeof(key_type), hipMemcpyHostToDevice)); - - value_type* d_values_input; - value_type* d_values_output; - HIP_CHECK(hipMalloc(&d_values_input, size * sizeof(value_type))); - HIP_CHECK(hipMalloc(&d_values_output, size * sizeof(value_type))); - HIP_CHECK(hipMemcpy(d_values_input, - values_input.data(), - size * sizeof(value_type), - hipMemcpyHostToDevice)); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - HIP_CHECK(hipcub::DeviceMergeSort::SortPairsCopy(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_values_input, - d_keys_output, - d_values_output, - size, - compare_function, - stream)); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) +template +class sort_pairs_benchmark : public primbench::benchmark_interface +{ + primbench::json meta() const override { - HIP_CHECK(hipcub::DeviceMergeSort::SortPairsCopy(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_values_input, - d_keys_output, - d_values_output, - size, - compare_function, - stream)); + return primbench::json{} + .add("algo", "device_merge_sort") + .add("subalgo", "sort_pairs") + .add("lvl", "device") + .add("key_data_type", primbench::name()) + .add("value_data_type", primbench::name()); } - HIP_CHECK(hipDeviceSynchronize()); - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); + const size_t items = state.size; + const auto& stream = state.stream; + + CompareFunction compare_function; + + std::vector keys_input + = benchmark_utils::get_random_data(items, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); - for(size_t i = 0; i < batch_size; i++) + std::vector values_input(items); + for(size_t i = 0; i < items; i++) { - HIP_CHECK(hipcub::DeviceMergeSort::SortPairsCopy(d_temporary_storage, - temporary_storage_bytes, + values_input[i] = Value(i); + } + + Key* d_keys_input; + Key* d_keys_output; + HIP_CHECK(hipMalloc(&d_keys_input, items * sizeof(Key))); + HIP_CHECK(hipMalloc(&d_keys_output, items * sizeof(Key))); + HIP_CHECK( + hipMemcpy(d_keys_input, keys_input.data(), items * sizeof(Key), hipMemcpyHostToDevice)); + + Value* d_values_input; + Value* d_values_output; + HIP_CHECK(hipMalloc(&d_values_input, items * sizeof(Value))); + HIP_CHECK(hipMalloc(&d_values_output, items * sizeof(Value))); + HIP_CHECK(hipMemcpy(d_values_input, + values_input.data(), + items * sizeof(Value), + hipMemcpyHostToDevice)); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] + { + HIP_CHECK(hipcub::DeviceMergeSort::SortPairsCopy(d_temp_storage, + temp_storage_bytes, d_keys_input, d_values_input, d_keys_output, d_values_output, - size, + items, compare_function, stream)); - } - HIP_CHECK(hipDeviceSynchronize()); + }; - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + launch(); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + + state.set_items(items); + state.add_writes(items); + state.add_writes(items); + + state.run(launch); + + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_keys_input)); + HIP_CHECK(hipFree(d_keys_output)); + HIP_CHECK(hipFree(d_values_input)); + HIP_CHECK(hipFree(d_values_output)); } - state.SetBytesProcessed(state.iterations() * batch_size * size - * (sizeof(key_type) + sizeof(value_type))); - state.SetItemsProcessed(state.iterations() * batch_size * size); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_values_output)); -} +}; + +#define CREATE_SORT_KEYS_BENCHMARK(T) executor.queue>() + +#define CREATE_SORT_PAIRS_BENCHMARK(T, V) executor.queue>() -#define CREATE_SORT_KEYS_BENCHMARK(T) \ - benchmarks.push_back(benchmark::RegisterBenchmark( \ - std::string("device_merge_sort_sort_keys" \ - ".") \ - .c_str(), \ - [=](benchmark::State& state) { run_sort_keys_benchmark(state, stream, size); })); - -#define CREATE_SORT_PAIRS_BENCHMARK(T, V) \ - benchmarks.push_back(benchmark::RegisterBenchmark( \ - std::string("device_merge_sort_sort_pairs<" \ - ",key_data_type:" #T ",value_data_type:" #V ">.") \ - .c_str(), \ - [=](benchmark::State& state) { run_sort_pairs_benchmark(state, stream, size); })); - -void add_sort_keys_benchmarks(std::vector& benchmarks, - hipStream_t stream, - size_t size) +void add_sort_keys_benchmarks(primbench::executor& executor) { - CREATE_SORT_KEYS_BENCHMARK(int) - CREATE_SORT_KEYS_BENCHMARK(long long) - CREATE_SORT_KEYS_BENCHMARK(int8_t) - CREATE_SORT_KEYS_BENCHMARK(uint8_t) - CREATE_SORT_KEYS_BENCHMARK(short) + CREATE_SORT_KEYS_BENCHMARK(int); + CREATE_SORT_KEYS_BENCHMARK(int64_t); + CREATE_SORT_KEYS_BENCHMARK(int8_t); + CREATE_SORT_KEYS_BENCHMARK(uint8_t); + CREATE_SORT_KEYS_BENCHMARK(short); } -void add_sort_pairs_benchmarks(std::vector& benchmarks, - hipStream_t stream, - size_t size) +void add_sort_pairs_benchmarks(primbench::executor& executor) { - using custom_float2 = benchmark_utils::custom_type; - using custom_double2 = benchmark_utils::custom_type; - using custom_char_double = benchmark_utils::custom_type; - using custom_double_char = benchmark_utils::custom_type; - - CREATE_SORT_PAIRS_BENCHMARK(int, float) - CREATE_SORT_PAIRS_BENCHMARK(int, double) - CREATE_SORT_PAIRS_BENCHMARK(int, custom_float2) - CREATE_SORT_PAIRS_BENCHMARK(int, custom_double2) - CREATE_SORT_PAIRS_BENCHMARK(int, custom_char_double) - CREATE_SORT_PAIRS_BENCHMARK(int, custom_double_char) - - CREATE_SORT_PAIRS_BENCHMARK(long long, float) - CREATE_SORT_PAIRS_BENCHMARK(long long, double) - CREATE_SORT_PAIRS_BENCHMARK(long long, custom_float2) - CREATE_SORT_PAIRS_BENCHMARK(long long, custom_char_double) - CREATE_SORT_PAIRS_BENCHMARK(long long, custom_double_char) - CREATE_SORT_PAIRS_BENCHMARK(long long, custom_double2) - - CREATE_SORT_PAIRS_BENCHMARK(int8_t, int8_t) - CREATE_SORT_PAIRS_BENCHMARK(uint8_t, uint8_t) + CREATE_SORT_PAIRS_BENCHMARK(int, float); + CREATE_SORT_PAIRS_BENCHMARK(int, double); + CREATE_SORT_PAIRS_BENCHMARK(int, custom_float2); + CREATE_SORT_PAIRS_BENCHMARK(int, custom_double2); + CREATE_SORT_PAIRS_BENCHMARK(int, custom_char_double); + CREATE_SORT_PAIRS_BENCHMARK(int, custom_double_char); + + CREATE_SORT_PAIRS_BENCHMARK(int64_t, float); + CREATE_SORT_PAIRS_BENCHMARK(int64_t, double); + CREATE_SORT_PAIRS_BENCHMARK(int64_t, custom_float2); + CREATE_SORT_PAIRS_BENCHMARK(int64_t, custom_char_double); + CREATE_SORT_PAIRS_BENCHMARK(int64_t, custom_double_char); + CREATE_SORT_PAIRS_BENCHMARK(int64_t, custom_double2); + + CREATE_SORT_PAIRS_BENCHMARK(int8_t, int8_t); + CREATE_SORT_PAIRS_BENCHMARK(uint8_t, uint8_t); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - - std::cout << "benchmark_device_merge_sort" << std::endl; - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - add_sort_keys_benchmarks(benchmarks, stream, size); - add_sort_pairs_benchmarks(benchmarks, stream, size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + primbench::executor executor(argc, argv, settings); + + add_sort_keys_benchmarks(executor); + add_sort_pairs_benchmarks(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_partition.cpp b/projects/hipcub/benchmark/benchmark_device_partition.cpp index f925e422cba2..24b6a6f2cdba 100644 --- a/projects/hipcub/benchmark/benchmark_device_partition.cpp +++ b/projects/hipcub/benchmark/benchmark_device_partition.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,21 +20,13 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include #include #include -#ifndef DEFAULT_N -constexpr size_t DEFAULT_N = 1024 * 1024 * 32; -#endif - -constexpr unsigned int batch_size = 10; -constexpr unsigned int warmup_size = 5; - namespace { template @@ -42,7 +34,8 @@ struct LessOp { HIPCUB_HOST_DEVICE LessOp(const T& pivot) : pivot_{pivot} {} - HIPCUB_HOST_DEVICE bool operator()(const T& val) const + HIPCUB_HOST_DEVICE + bool operator()(const T& val) const { return val < pivot_; } @@ -52,68 +45,49 @@ struct LessOp }; } // namespace -template -void run_flagged(benchmark::State& state, - const hipStream_t stream, - const T threshold, - const size_t size) +template +class flagged_benchmark : public primbench::benchmark_interface { - const auto select_op = LessOp{threshold}; - const auto input - = benchmark_utils::get_random_data(size, static_cast(0), static_cast(100)); - - std::vector flags(size); - for(unsigned int i = 0; i < size; i++) +private: + primbench::json meta() const override { - flags[i] = static_cast(select_op(input[i])); + return primbench::json{} + .add("algo", "device_partition") + .add("subalgo", "flagged") + .add("lvl", "device") + .add("data_type", primbench::name()) + .add("flag_type", primbench::name()) + .add("split_threshold", std::to_string(Threshold) + "%"); } - T* d_input = nullptr; - F* d_flags = nullptr; - T* d_output = nullptr; - unsigned int* d_num_selected_output = nullptr; - HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_flags, input.size() * sizeof(F))); - HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_num_selected_output, sizeof(unsigned int))); - - // Allocate temporary storage - void* d_temp_storage = nullptr; - size_t temp_storage_bytes = 0; - HIP_CHECK(hipcub::DevicePartition::Flagged(nullptr, - temp_storage_bytes, - d_input, - d_flags, - d_output, - d_num_selected_output, - static_cast(input.size()), - stream)); - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); - - // Warm-up - HIP_CHECK(hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_flags, flags.data(), flags.size() * sizeof(F), hipMemcpyHostToDevice)); - for(unsigned int i = 0; i < warmup_size; ++i) + void run(primbench::state& state) override { - HIP_CHECK(hipcub::DevicePartition::Flagged(d_temp_storage, - temp_storage_bytes, - d_input, - d_flags, - d_output, - d_num_selected_output, - static_cast(input.size()), - stream)); - } - HIP_CHECK(hipDeviceSynchronize()); + const size_t items = state.size; + const auto& stream = state.stream; - // Run benchmark - for(auto _ : state) - { - namespace chrono = std::chrono; - using clock = chrono::high_resolution_clock; + const auto select_op = LessOp{T(Threshold)}; + const auto input + = benchmark_utils::get_random_data(items, static_cast(0), static_cast(100)); + + std::vector flags(items); + for(unsigned int i = 0; i < items; i++) + { + flags[i] = static_cast(select_op(input[i])); + } + + T* d_input = nullptr; + F* d_flags = nullptr; + T* d_output = nullptr; + unsigned int* d_num_selected_output = nullptr; + HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_flags, input.size() * sizeof(F))); + HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_num_selected_output, sizeof(unsigned int))); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; - const auto start = clock::now(); - for(unsigned int i = 0; i < batch_size; ++i) + const auto launch = [&] { HIP_CHECK(hipcub::DevicePartition::Flagged(d_temp_storage, temp_storage_bytes, @@ -123,81 +97,61 @@ void run_flagged(benchmark::State& state, d_num_selected_output, static_cast(input.size()), stream)); - } - HIP_CHECK(hipDeviceSynchronize()); + }; - const auto end = clock::now(); - using seconds_d = chrono::duration; - const auto elapsed_seconds = chrono::duration_cast(end - start); + launch(); - state.SetIterationTime(elapsed_seconds.count()); - } + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); - state.SetItemsProcessed(state.iterations() * batch_size * input.size()); - state.SetBytesProcessed( - static_cast(state.iterations() * batch_size * input.size() * sizeof(input[0]))); + state.set_items(input.size()); + state.add_writes(input.size()); - HIP_CHECK(hipFree(d_temp_storage)); - HIP_CHECK(hipFree(d_num_selected_output)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_flags)); - HIP_CHECK(hipFree(d_input)); -} + state.run(launch); -template -void run_predicate(benchmark::State& state, - const hipStream_t stream, - const T threshold, - const size_t size) + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_num_selected_output)); + HIP_CHECK(hipFree(d_output)); + HIP_CHECK(hipFree(d_flags)); + HIP_CHECK(hipFree(d_input)); + } +}; + +template +class predicate_benchmark : public primbench::benchmark_interface { - const auto input - = benchmark_utils::get_random_data(size, static_cast(0), static_cast(100)); - - T* d_input = nullptr; - T* d_output = nullptr; - unsigned int* d_num_selected_output = nullptr; - HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_num_selected_output, sizeof(unsigned int))); - - const auto select_op = LessOp{threshold}; - - // Allocate temporary storage - void* d_temp_storage = nullptr; - size_t temp_storage_bytes = 0; - HIP_CHECK(hipcub::DevicePartition::If(nullptr, - temp_storage_bytes, - d_input, - d_output, - d_num_selected_output, - static_cast(input.size()), - select_op, - stream)); - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); - - // Warm-up - HIP_CHECK(hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); - for(unsigned int i = 0; i < warmup_size; ++i) +private: + primbench::json meta() const override { - HIP_CHECK(hipcub::DevicePartition::If(d_temp_storage, - temp_storage_bytes, - d_input, - d_output, - d_num_selected_output, - static_cast(input.size()), - select_op, - stream)); + return primbench::json{} + .add("algo", "device_partition") + .add("subalgo", "predicate") + .add("lvl", "device") + .add("data_type", primbench::name()) + .add("split_threshold", std::to_string(Threshold) + "%"); } - HIP_CHECK(hipDeviceSynchronize()); - // Run benchmark - for(auto _ : state) + void run(primbench::state& state) override { - namespace chrono = std::chrono; - using clock = chrono::high_resolution_clock; + const size_t items = state.size; + const auto& stream = state.stream; - const auto start = clock::now(); - for(unsigned int i = 0; i < batch_size; ++i) + const auto input + = benchmark_utils::get_random_data(items, static_cast(0), static_cast(100)); + + T* d_input = nullptr; + T* d_output = nullptr; + unsigned int* d_num_selected_output = nullptr; + HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_num_selected_output, sizeof(unsigned int))); + + const auto select_op = LessOp{T(Threshold)}; + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { HIP_CHECK(hipcub::DevicePartition::If(d_temp_storage, temp_storage_bytes, @@ -207,92 +161,66 @@ void run_predicate(benchmark::State& state, static_cast(input.size()), select_op, stream)); - } - HIP_CHECK(hipDeviceSynchronize()); + }; - const auto end = clock::now(); - using seconds_d = chrono::duration; - const auto elapsed_seconds = chrono::duration_cast(end - start); + launch(); - state.SetIterationTime(elapsed_seconds.count()); - } + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); - state.SetItemsProcessed(state.iterations() * batch_size * input.size()); - state.SetBytesProcessed( - static_cast(state.iterations() * batch_size * input.size() * sizeof(input[0]))); + state.set_items(input.size()); + state.add_writes(input.size()); - HIP_CHECK(hipFree(d_temp_storage)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_num_selected_output)); -} + state.run(launch); -template -void run_threeway(benchmark::State& state, - const hipStream_t stream, - const T small_threshold, - const T large_threshold, - const size_t size) + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); + HIP_CHECK(hipFree(d_num_selected_output)); + } +}; + +template +class threeway_benchmark : public primbench::benchmark_interface { - const auto input - = benchmark_utils::get_random_data(size, static_cast(0), static_cast(100)); - - T* d_input = nullptr; - T* d_first_output = nullptr; - T* d_second_output = nullptr; - T* d_unselected_output = nullptr; - unsigned int* d_num_selected_output = nullptr; - HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_first_output, input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_second_output, input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_unselected_output, input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_num_selected_output, 2 * sizeof(unsigned int))); - - const auto select_first_part_op = LessOp{small_threshold}; - const auto select_second_part_op = LessOp{large_threshold}; - - // Allocate temporary storage - void* d_temp_storage = nullptr; - size_t temp_storage_bytes = 0; - HIP_CHECK(hipcub::DevicePartition::If(nullptr, - temp_storage_bytes, - d_input, - d_first_output, - d_second_output, - d_unselected_output, - d_num_selected_output, - static_cast(input.size()), - select_first_part_op, - select_second_part_op, - stream)); - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); - - // Warm-up - HIP_CHECK(hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); - for(unsigned int i = 0; i < warmup_size; ++i) +private: + primbench::json meta() const override { - HIP_CHECK(hipcub::DevicePartition::If(d_temp_storage, - temp_storage_bytes, - d_input, - d_first_output, - d_second_output, - d_unselected_output, - d_num_selected_output, - static_cast(input.size()), - select_first_part_op, - select_second_part_op, - stream)); + return primbench::json{} + .add("algo", "device_partition") + .add("subalgo", "three_way") + .add("lvl", "device") + .add("data_type", primbench::name()) + .add("small_threshold", std::to_string(SmallThreshold) + "%") + .add("large_threshold", std::to_string(LargeThreshold) + "%"); } - HIP_CHECK(hipDeviceSynchronize()); - // Run benchmark - for(auto _ : state) + void run(primbench::state& state) override { - namespace chrono = std::chrono; - using clock = chrono::high_resolution_clock; - - const auto start = clock::now(); - for(unsigned int i = 0; i < batch_size; ++i) + const size_t items = state.size; + const auto& stream = state.stream; + + const auto input + = benchmark_utils::get_random_data(items, static_cast(0), static_cast(100)); + + T* d_input = nullptr; + T* d_first_output = nullptr; + T* d_second_output = nullptr; + T* d_unselected_output = nullptr; + unsigned int* d_num_selected_output = nullptr; + HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_first_output, input.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_second_output, input.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_unselected_output, input.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_num_selected_output, 2 * sizeof(unsigned int))); + + const auto select_first_part_op = LessOp{T(SmallThreshold)}; + const auto select_second_part_op = LessOp{T(LargeThreshold)}; + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { HIP_CHECK(hipcub::DevicePartition::If(d_temp_storage, temp_storage_bytes, @@ -305,142 +233,84 @@ void run_threeway(benchmark::State& state, select_first_part_op, select_second_part_op, stream)); - } - HIP_CHECK(hipDeviceSynchronize()); + }; - const auto end = clock::now(); - using seconds_d = chrono::duration; - const auto elapsed_seconds = chrono::duration_cast(end - start); + launch(); - state.SetIterationTime(elapsed_seconds.count()); - } + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); - state.SetItemsProcessed(state.iterations() * batch_size * input.size()); - state.SetBytesProcessed( - static_cast(state.iterations() * batch_size * input.size() * sizeof(input[0]))); + state.set_items(input.size()); + state.add_writes(input.size()); - HIP_CHECK(hipFree(d_temp_storage)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_first_output)); - HIP_CHECK(hipFree(d_second_output)); - HIP_CHECK(hipFree(d_unselected_output)); - HIP_CHECK(hipFree(d_num_selected_output)); -} + state.run(launch); -#define CREATE_BENCHMARK_FLAGGED(T, T_FLAG, SPLIT_T) \ - benchmark::RegisterBenchmark(std::string("device_parition_flagged.(split_threshold:" #SPLIT_T \ - "%)") \ - .c_str(), \ - &run_flagged, \ - stream, \ - static_cast(SPLIT_T), \ - size) - -#define CREATE_BENCHMARK_PREDICATE(T, SPLIT_T) \ - benchmark::RegisterBenchmark( \ - std::string("device_parition_predicate.(split_threshold:" #SPLIT_T "%)") \ - .c_str(), \ - &run_predicate, \ - stream, \ - static_cast(SPLIT_T), \ - size) - -#define CREATE_BENCHMARK_THREEWAY(T, SMALL_T, LARGE_T) \ - benchmark::RegisterBenchmark(std::string("device_parition_three_way" \ - ".(small_threshold:" #SMALL_T \ - "%,large_threshold:" #LARGE_T "%)") \ - .c_str(), \ - &run_threeway, \ - stream, \ - static_cast(SMALL_T), \ - static_cast(LARGE_T), \ - size) - -#define BENCHMARK_FLAGGED_TYPE(type, flag_type) \ - CREATE_BENCHMARK_FLAGGED(type, flag_type, 33), CREATE_BENCHMARK_FLAGGED(type, flag_type, 50), \ - CREATE_BENCHMARK_FLAGGED(type, flag_type, 60), \ - CREATE_BENCHMARK_FLAGGED(type, flag_type, 90) - -#define BENCHMARK_PREDICATE_TYPE(type) \ - CREATE_BENCHMARK_PREDICATE(type, 33), CREATE_BENCHMARK_PREDICATE(type, 50), \ - CREATE_BENCHMARK_PREDICATE(type, 60), CREATE_BENCHMARK_PREDICATE(type, 90) - -#define BENCHMARK_THREEWAY_TYPE(type) \ - CREATE_BENCHMARK_THREEWAY(type, 33, 66), CREATE_BENCHMARK_THREEWAY(type, 10, 66), \ - CREATE_BENCHMARK_THREEWAY(type, 50, 60), CREATE_BENCHMARK_THREEWAY(type, 50, 90) + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_first_output)); + HIP_CHECK(hipFree(d_second_output)); + HIP_CHECK(hipFree(d_unselected_output)); + HIP_CHECK(hipFree(d_num_selected_output)); + } +}; -int main(int argc, char* argv[]) -{ - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); +#define CREATE_BENCHMARK_FLAGGED(T, T_FLAG, SPLIT_T) \ + executor.queue>() - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); +#define CREATE_BENCHMARK_PREDICATE(T, SPLIT_T) executor.queue>() - std::cout << "benchmark_device_partition" << std::endl; +#define CREATE_BENCHMARK_THREEWAY(T, SMALL_T, LARGE_T) \ + executor.queue>() - // HIP - const hipStream_t stream = 0; // default - { - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - } +#define BENCHMARK_FLAGGED_TYPE(type, flag_type) \ + CREATE_BENCHMARK_FLAGGED(type, flag_type, 33); \ + CREATE_BENCHMARK_FLAGGED(type, flag_type, 50); \ + CREATE_BENCHMARK_FLAGGED(type, flag_type, 60); \ + CREATE_BENCHMARK_FLAGGED(type, flag_type, 90) - using custom_float2 = benchmark_utils::custom_type; - using custom_double2 = benchmark_utils::custom_type; - - // Add benchmarks - std::vector benchmarks = { - BENCHMARK_FLAGGED_TYPE(int8_t, unsigned char), - BENCHMARK_FLAGGED_TYPE(int, unsigned char), - BENCHMARK_FLAGGED_TYPE(float, unsigned char), - BENCHMARK_FLAGGED_TYPE(long long, uint8_t), - BENCHMARK_FLAGGED_TYPE(double, int8_t), - BENCHMARK_FLAGGED_TYPE(custom_float2, int8_t), - BENCHMARK_FLAGGED_TYPE(custom_double2, unsigned char), - - BENCHMARK_PREDICATE_TYPE(int8_t), - BENCHMARK_PREDICATE_TYPE(int), - BENCHMARK_PREDICATE_TYPE(float), - BENCHMARK_PREDICATE_TYPE(long long), - BENCHMARK_PREDICATE_TYPE(double), - BENCHMARK_PREDICATE_TYPE(custom_float2), - BENCHMARK_PREDICATE_TYPE(custom_double2), - - BENCHMARK_THREEWAY_TYPE(int8_t), - BENCHMARK_THREEWAY_TYPE(int), - BENCHMARK_THREEWAY_TYPE(float), - BENCHMARK_THREEWAY_TYPE(long long), - BENCHMARK_THREEWAY_TYPE(double), - BENCHMARK_THREEWAY_TYPE(custom_float2), - BENCHMARK_THREEWAY_TYPE(custom_double2), - }; - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } +#define BENCHMARK_PREDICATE_TYPE(type) \ + CREATE_BENCHMARK_PREDICATE(type, 33); \ + CREATE_BENCHMARK_PREDICATE(type, 50); \ + CREATE_BENCHMARK_PREDICATE(type, 60); \ + CREATE_BENCHMARK_PREDICATE(type, 90) - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } +#define BENCHMARK_THREEWAY_TYPE(type) \ + CREATE_BENCHMARK_THREEWAY(type, 33, 66); \ + CREATE_BENCHMARK_THREEWAY(type, 10, 66); \ + CREATE_BENCHMARK_THREEWAY(type, 50, 60); \ + CREATE_BENCHMARK_THREEWAY(type, 50, 90) - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; +int main(int argc, char* argv[]) +{ + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; + + primbench::executor executor(argc, argv, settings); + + BENCHMARK_FLAGGED_TYPE(int8_t, unsigned char); + BENCHMARK_FLAGGED_TYPE(int, unsigned char); + BENCHMARK_FLAGGED_TYPE(float, unsigned char); + BENCHMARK_FLAGGED_TYPE(int64_t, uint8_t); + BENCHMARK_FLAGGED_TYPE(double, int8_t); + BENCHMARK_FLAGGED_TYPE(custom_float2, int8_t); + BENCHMARK_FLAGGED_TYPE(custom_double2, unsigned char); + + BENCHMARK_PREDICATE_TYPE(int8_t); + BENCHMARK_PREDICATE_TYPE(int); + BENCHMARK_PREDICATE_TYPE(float); + BENCHMARK_PREDICATE_TYPE(int64_t); + BENCHMARK_PREDICATE_TYPE(double); + BENCHMARK_PREDICATE_TYPE(custom_float2); + BENCHMARK_PREDICATE_TYPE(custom_double2); + + BENCHMARK_THREEWAY_TYPE(int8_t); + BENCHMARK_THREEWAY_TYPE(int); + BENCHMARK_THREEWAY_TYPE(float); + BENCHMARK_THREEWAY_TYPE(int64_t); + BENCHMARK_THREEWAY_TYPE(double); + BENCHMARK_THREEWAY_TYPE(custom_float2); + BENCHMARK_THREEWAY_TYPE(custom_double2); + + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_radix_sort.cpp b/projects/hipcub/benchmark/benchmark_device_radix_sort.cpp index 2e40b3b64a71..54324252db63 100644 --- a/projects/hipcub/benchmark/benchmark_device_radix_sort.cpp +++ b/projects/hipcub/benchmark/benchmark_device_radix_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,38 +20,19 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" #include #include -// HIP API #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif - -const unsigned int batch_size = 10; -const unsigned int warmup_size = 5; - -template -std::vector generate_keys(size_t size) -{ - using key_type = Key; - - return benchmark_utils::get_random_data( - size, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); -} - template auto invoke_sort_keys(void* d_temp_storage, size_t& temp_storage_bytes, Key* d_keys_input, Key* d_keys_output, - size_t size, + size_t items, hipStream_t stream) -> std::enable_if_t::value, hipError_t> { @@ -59,7 +40,7 @@ auto invoke_sort_keys(void* d_temp_storage, temp_storage_bytes, d_keys_input, d_keys_output, - size, + items, 0, sizeof(Key) * 8, stream); @@ -70,7 +51,7 @@ auto invoke_sort_keys(void* d_temp_storage, size_t& temp_storage_bytes, Key* d_keys_input, Key* d_keys_output, - size_t size, + size_t items, hipStream_t stream) -> std::enable_if_t::value, hipError_t> { @@ -78,7 +59,7 @@ auto invoke_sort_keys(void* d_temp_storage, temp_storage_bytes, d_keys_input, d_keys_output, - size, + items, 0, sizeof(Key) * 8, stream); @@ -89,7 +70,7 @@ auto invoke_sort_keys(void* d_temp_storage, size_t& temp_storage_bytes, Key* d_keys_input, Key* d_keys_output, - size_t size, + size_t items, hipStream_t stream) -> std::enable_if_t::value, hipError_t> { @@ -97,7 +78,7 @@ auto invoke_sort_keys(void* d_temp_storage, temp_storage_bytes, d_keys_input, d_keys_output, - size, + items, benchmark_utils::custom_type_decomposer{}, stream); } @@ -107,7 +88,7 @@ auto invoke_sort_keys(void* d_temp_storage, size_t& temp_storage_bytes, Key* d_keys_input, Key* d_keys_output, - size_t size, + size_t items, hipStream_t stream) -> std::enable_if_t::value, hipError_t> { @@ -116,78 +97,69 @@ auto invoke_sort_keys(void* d_temp_storage, temp_storage_bytes, d_keys_input, d_keys_output, - size, + items, benchmark_utils::custom_type_decomposer{}, stream); } template -void run_sort_keys_benchmark(benchmark::State& state, - hipStream_t stream, - size_t size, - std::shared_ptr> keys_input) +class sort_keys_benchmark : public primbench::benchmark_interface { - using key_type = Key; - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(hipMalloc(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK(hipMalloc(&d_keys_output, size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_keys_input, - keys_input->data(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - HIP_CHECK(invoke_sort_keys(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - size, - stream)); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) + primbench::json meta() const override { - HIP_CHECK(invoke_sort_keys(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - size, - stream)); + return primbench::json{} + .add("algo", "device_radix_sort") + .add("subalgo", "sort_keys") + .add("lvl", "device") + .add("key_data_type", primbench::name()) + .add("descending", Descending); } - HIP_CHECK(hipDeviceSynchronize()); - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); + const size_t items = state.size; + const auto& stream = state.stream; + + std::vector keys_input + = benchmark_utils::get_random_data(items, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); + + Key* d_keys_input; + Key* d_keys_output; + HIP_CHECK(hipMalloc(&d_keys_input, items * sizeof(Key))); + HIP_CHECK(hipMalloc(&d_keys_output, items * sizeof(Key))); + HIP_CHECK( + hipMemcpy(d_keys_input, keys_input.data(), items * sizeof(Key), hipMemcpyHostToDevice)); - for(size_t i = 0; i < batch_size; i++) + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { - HIP_CHECK(invoke_sort_keys(d_temporary_storage, - temporary_storage_bytes, + HIP_CHECK(invoke_sort_keys(d_temp_storage, + temp_storage_bytes, d_keys_input, d_keys_output, - size, + items, stream)); - } + }; + + launch(); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(key_type)); - state.SetItemsProcessed(state.iterations() * batch_size * size); + state.set_items(items); + state.add_writes(items); - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_keys_output)); -} + state.run(launch); + + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_keys_input)); + HIP_CHECK(hipFree(d_keys_output)); + } +}; template auto invoke_sort_pairs(void* d_temp_storage, @@ -196,7 +168,7 @@ auto invoke_sort_pairs(void* d_temp_storage, Key* d_keys_output, Value* d_values_input, Value* d_values_output, - size_t size, + size_t items, hipStream_t stream) -> std::enable_if_t::value, hipError_t> { @@ -206,7 +178,7 @@ auto invoke_sort_pairs(void* d_temp_storage, d_keys_output, d_values_input, d_values_output, - size, + items, 0, sizeof(Key) * 8, stream); @@ -219,7 +191,7 @@ auto invoke_sort_pairs(void* d_temp_storage, Key* d_keys_output, Value* d_values_input, Value* d_values_output, - size_t size, + size_t items, hipStream_t stream) -> std::enable_if_t::value, hipError_t> { @@ -229,7 +201,7 @@ auto invoke_sort_pairs(void* d_temp_storage, d_keys_output, d_values_input, d_values_output, - size, + items, 0, sizeof(Key) * 8, stream); @@ -242,7 +214,7 @@ auto invoke_sort_pairs(void* d_temp_storage, Key* d_keys_output, Value* d_values_input, Value* d_values_output, - size_t size, + size_t items, hipStream_t stream) -> std::enable_if_t::value, hipError_t> { @@ -252,7 +224,7 @@ auto invoke_sort_pairs(void* d_temp_storage, d_keys_output, d_values_input, d_values_output, - size, + items, benchmark_utils::custom_type_decomposer{}, stream); } @@ -264,7 +236,7 @@ auto invoke_sort_pairs(void* d_temp_storage, Key* d_keys_output, Value* d_values_input, Value* d_values_output, - size_t size, + size_t items, hipStream_t stream) -> std::enable_if_t::value, hipError_t> { @@ -275,225 +247,132 @@ auto invoke_sort_pairs(void* d_temp_storage, d_keys_output, d_values_input, d_values_output, - size, + items, benchmark_utils::custom_type_decomposer{}, stream); } template -void run_sort_pairs_benchmark(benchmark::State& state, - hipStream_t stream, - size_t size, - std::shared_ptr> keys_input) +class sort_pairs_benchmark : public primbench::benchmark_interface { - using key_type = Key; - using value_type = Value; - std::vector values_input(size); - for(size_t i = 0; i < size; i++) + primbench::json meta() const override { - values_input[i] = value_type(i); + return primbench::json{} + .add("algo", "device_radix_sort") + .add("subalgo", "sort_pairs") + .add("lvl", "device") + .add("key_data_type", primbench::name()) + .add("value_data_type", primbench::name()) + .add("descending", Descending); } - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(hipMalloc(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK(hipMalloc(&d_keys_output, size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_keys_input, - keys_input->data(), - size * sizeof(key_type), - hipMemcpyHostToDevice)); - - value_type* d_values_input; - value_type* d_values_output; - HIP_CHECK(hipMalloc(&d_values_input, size * sizeof(value_type))); - HIP_CHECK(hipMalloc(&d_values_output, size * sizeof(value_type))); - HIP_CHECK(hipMemcpy(d_values_input, - values_input.data(), - size * sizeof(value_type), - hipMemcpyHostToDevice)); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - HIP_CHECK(invoke_sort_pairs(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, - size, - stream)); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) + void run(primbench::state& state) override { - HIP_CHECK(invoke_sort_pairs(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, - size, - stream)); - } - HIP_CHECK(hipDeviceSynchronize()); + const size_t items = state.size; + const auto& stream = state.stream; - for(auto _ : state) - { - auto start = std::chrono::high_resolution_clock::now(); + std::vector keys_input + = benchmark_utils::get_random_data(items, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); - for(size_t i = 0; i < batch_size; i++) + std::vector values_input(items); + for(size_t i = 0; i < items; i++) { - HIP_CHECK(invoke_sort_pairs(d_temporary_storage, - temporary_storage_bytes, + values_input[i] = Value(i); + } + + Key* d_keys_input; + Key* d_keys_output; + HIP_CHECK(hipMalloc(&d_keys_input, items * sizeof(Key))); + HIP_CHECK(hipMalloc(&d_keys_output, items * sizeof(Key))); + HIP_CHECK( + hipMemcpy(d_keys_input, keys_input.data(), items * sizeof(Key), hipMemcpyHostToDevice)); + + Value* d_values_input; + Value* d_values_output; + HIP_CHECK(hipMalloc(&d_values_input, items * sizeof(Value))); + HIP_CHECK(hipMalloc(&d_values_output, items * sizeof(Value))); + HIP_CHECK(hipMemcpy(d_values_input, + values_input.data(), + items * sizeof(Value), + hipMemcpyHostToDevice)); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] + { + HIP_CHECK(invoke_sort_pairs(d_temp_storage, + temp_storage_bytes, d_keys_input, d_keys_output, d_values_input, d_values_output, - size, + items, stream)); - } + }; + + launch(); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * size - * (sizeof(key_type) + sizeof(value_type))); - state.SetItemsProcessed(state.iterations() * batch_size * size); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_values_output)); -} + state.set_items(items); + state.add_writes(items); + state.add_writes(items); -#define CREATE_SORT_KEYS_BENCHMARK(Key) \ - { \ - auto keys_input = std::make_shared>(generate_keys(size)); \ - benchmarks.push_back(benchmark::RegisterBenchmark( \ - std::string("device_radix_sort_keys_ascending" \ - ".") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_sort_keys_benchmark(state, stream, size, keys_input); })); \ - benchmarks.push_back(benchmark::RegisterBenchmark( \ - std::string("device_radix_sort_keys_descending" \ - ".") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_sort_keys_benchmark(state, stream, size, keys_input); })); \ - } + state.run(launch); -#define CREATE_SORT_PAIRS_BENCHMARK(Key, Value) \ - { \ - auto keys_input = std::make_shared>(generate_keys(size)); \ - benchmarks.push_back(benchmark::RegisterBenchmark( \ - std::string("device_radix_sort_pairs_ascending" \ - ".") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_sort_pairs_benchmark(state, stream, size, keys_input); })); \ - benchmarks.push_back(benchmark::RegisterBenchmark( \ - std::string("device_radix_sort_pairs_descending" \ - ".") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_sort_pairs_benchmark(state, stream, size, keys_input); })); \ + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_keys_input)); + HIP_CHECK(hipFree(d_keys_output)); + HIP_CHECK(hipFree(d_values_input)); + HIP_CHECK(hipFree(d_values_output)); } +}; -void add_sort_keys_benchmarks(std::vector& benchmarks, - hipStream_t stream, - size_t size) -{ - using custom_int_t = benchmark_utils::custom_type; - CREATE_SORT_KEYS_BENCHMARK(int) - CREATE_SORT_KEYS_BENCHMARK(long long) - CREATE_SORT_KEYS_BENCHMARK(int8_t) - CREATE_SORT_KEYS_BENCHMARK(uint8_t) - CREATE_SORT_KEYS_BENCHMARK(short) - CREATE_SORT_KEYS_BENCHMARK(custom_int_t) -} +#define QUEUE_KEY(Key) \ + executor.queue>(); \ + executor.queue>(); -void add_sort_pairs_benchmarks(std::vector& benchmarks, - hipStream_t stream, - size_t size) -{ - using custom_float2 = benchmark_utils::custom_type; - using custom_double2 = benchmark_utils::custom_type; - using custom_char_double = benchmark_utils::custom_type; - using custom_double_char = benchmark_utils::custom_type; - using custom_int_t = benchmark_utils::custom_type; - - CREATE_SORT_PAIRS_BENCHMARK(int, float) - CREATE_SORT_PAIRS_BENCHMARK(int, double) - CREATE_SORT_PAIRS_BENCHMARK(int, custom_float2) - CREATE_SORT_PAIRS_BENCHMARK(int, custom_double2) - CREATE_SORT_PAIRS_BENCHMARK(int, custom_char_double) - CREATE_SORT_PAIRS_BENCHMARK(int, custom_double_char) - - CREATE_SORT_PAIRS_BENCHMARK(long long, float) - CREATE_SORT_PAIRS_BENCHMARK(long long, double) - CREATE_SORT_PAIRS_BENCHMARK(long long, custom_float2) - CREATE_SORT_PAIRS_BENCHMARK(long long, custom_char_double) - CREATE_SORT_PAIRS_BENCHMARK(long long, custom_double_char) - CREATE_SORT_PAIRS_BENCHMARK(long long, custom_double2) - - CREATE_SORT_PAIRS_BENCHMARK(int8_t, int8_t) - CREATE_SORT_PAIRS_BENCHMARK(uint8_t, uint8_t) - - CREATE_SORT_PAIRS_BENCHMARK(custom_int_t, float) -} +#define QUEUE_PAIR(Key, Value) \ + executor.queue>(); \ + executor.queue>(); int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_device_radix_sort" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - add_sort_keys_benchmarks(benchmarks, stream, size); - add_sort_pairs_benchmarks(benchmarks, stream, size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } - - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } - - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; + + primbench::executor executor(argc, argv); + + QUEUE_KEY(int) + QUEUE_KEY(int64_t) + QUEUE_KEY(int8_t) + QUEUE_KEY(uint8_t) + QUEUE_KEY(short) + QUEUE_KEY(custom_int_t) + + QUEUE_PAIR(int, float); + QUEUE_PAIR(int, double); + QUEUE_PAIR(int, custom_float2); + QUEUE_PAIR(int, custom_double2); + QUEUE_PAIR(int, custom_char_double); + QUEUE_PAIR(int, custom_double_char); + + QUEUE_PAIR(int64_t, float); + QUEUE_PAIR(int64_t, double); + QUEUE_PAIR(int64_t, custom_float2); + QUEUE_PAIR(int64_t, custom_char_double); + QUEUE_PAIR(int64_t, custom_double_char); + QUEUE_PAIR(int64_t, custom_double2); + + QUEUE_PAIR(int8_t, int8_t); + QUEUE_PAIR(uint8_t, uint8_t); + + QUEUE_PAIR(custom_int_t, float); + + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_reduce.cpp b/projects/hipcub/benchmark/benchmark_device_reduce.cpp index 8dcd96861a96..2a138f2040af 100644 --- a/projects/hipcub/benchmark/benchmark_device_reduce.cpp +++ b/projects/hipcub/benchmark/benchmark_device_reduce.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,182 +20,142 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" + #include "hipcub/config.hpp" -// HIP API #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 128; -#endif - -const unsigned int batch_size = 10; -const unsigned int warmup_size = 5; - template -void run_benchmark(benchmark::State& state, - size_t size, - const hipStream_t stream, - ReduceKernel reduce) +class reduce_benchmark : public primbench::benchmark_interface { - std::vector input = benchmark_utils::get_random_data(size, T(0), T(1000)); - - T* d_input; - OutputT* d_output; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMalloc(&d_output, sizeof(OutputT))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - // Allocate temporary storage memory - size_t temp_storage_size_bytes = 0; - void* d_temp_storage = nullptr; - // Get size of d_temp_storage - HIP_CHECK(reduce(d_temp_storage, temp_storage_size_bytes, d_input, d_output, size, stream)); - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - for(size_t i = 0; i < warmup_size; i++) + primbench::json meta() const override { - HIP_CHECK(reduce(d_temp_storage, temp_storage_size_bytes, d_input, d_output, size, stream)); + return primbench::json{} + .add("algo", "device_reduce") + .add("lvl", "device") + .add("data_type", primbench::name()) + .add("op", ReduceKernel::name); } - HIP_CHECK(hipDeviceSynchronize()); - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); - - for(size_t i = 0; i < batch_size; i++) - { - HIP_CHECK( - reduce(d_temp_storage, temp_storage_size_bytes, d_input, d_output, size, stream)); - } - HIP_CHECK(hipStreamSynchronize(stream)); - - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * batch_size * size); + const size_t items = state.size; + const auto& stream = state.stream; - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); -} + std::vector input = benchmark_utils::get_random_data(items, T(0), T(1000)); + + T* d_input; + OutputT* d_output; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK(hipMalloc(&d_output, sizeof(OutputT))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); + + auto reduce = ReduceKernel::kernel; + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { + HIP_CHECK(reduce(d_temp_storage, temp_storage_bytes, d_input, d_output, items, stream)); + }; + + launch(); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); + + state.set_items(items); + state.add_writes(items); + + state.run(launch); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); + HIP_CHECK(hipFree(d_temp_storage)); + } +}; template struct Benchmark; -template -struct Benchmark +template +struct sum_kernel { - static void run(benchmark::State& state, size_t size, const hipStream_t stream) - { - hipError_t (*ptr_to_sum)(void*, size_t&, T*, T*, int, hipStream_t) - = &hipcub::DeviceReduce::Sum; - run_benchmark(state, size, stream, ptr_to_sum); - } + static constexpr const char* name = "sum"; + + static constexpr hipError_t (*kernel)(void*, size_t&, T*, T*, int, hipStream_t) + = &hipcub::DeviceReduce::Sum; }; template -struct Benchmark +struct Benchmark { - static void run(benchmark::State& state, size_t size, const hipStream_t stream) - { - hipError_t (*ptr_to_min)(void*, size_t&, T*, T*, int, hipStream_t) - = &hipcub::DeviceReduce::Min; - run_benchmark(state, size, stream, ptr_to_min); - } + using type = reduce_benchmark>; +}; + +template +struct min_kernel +{ + static constexpr const char* name = "min"; + + static constexpr hipError_t (*kernel)(void*, size_t&, T*, T*, int, hipStream_t) + = &hipcub::DeviceReduce::Min; }; template -struct Benchmark +struct Benchmark +{ + using type = reduce_benchmark>; +}; + +template +struct argmin_kernel { + static constexpr const char* name = "argmin"; + using Difference = int; - using Iterator = typename hipcub::ArgIndexInputIterator; + using Iterator = hipcub::ArgIndexInputIterator; using KeyValue = typename Iterator::value_type; - static void run(benchmark::State& state, size_t size, const hipStream_t stream) - { - HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH - hipError_t (*ptr_to_argmin)(void*, size_t&, T*, KeyValue*, int, hipStream_t) + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + static constexpr hipError_t (*kernel)(void*, size_t&, T*, KeyValue*, int, hipStream_t) = &hipcub::DeviceReduce::ArgMin; - HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP - run_benchmark(state, size, stream, ptr_to_argmin); - } + HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP +}; +template +struct Benchmark +{ + using type = reduce_benchmark::KeyValue, argmin_kernel>; }; -#define CREATE_BENCHMARK(T, REDUCE_OP) \ - benchmark::RegisterBenchmark(std::string("device_reduce" \ - ".") \ - .c_str(), \ - &Benchmark::run, \ - size, \ - stream) +#define CREATE_BENCHMARK(T, REDUCE_OP) executor.queue::type>() -#define CREATE_BENCHMARKS(REDUCE_OP) \ - CREATE_BENCHMARK(int, REDUCE_OP), CREATE_BENCHMARK(long long, REDUCE_OP), \ - CREATE_BENCHMARK(float, REDUCE_OP), CREATE_BENCHMARK(double, REDUCE_OP), \ - CREATE_BENCHMARK(int8_t, REDUCE_OP) +#define CREATE_BENCHMARKS(REDUCE_OP) \ + CREATE_BENCHMARK(int, REDUCE_OP); \ + CREATE_BENCHMARK(int64_t, REDUCE_OP); \ + CREATE_BENCHMARK(float, REDUCE_OP); \ + CREATE_BENCHMARK(double, REDUCE_OP); \ + CREATE_BENCHMARK(int8_t, REDUCE_OP) int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_device_reduce" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - using custom_double2 = benchmark_utils::custom_type; - - // Add benchmarks - std::vector benchmarks = { - CREATE_BENCHMARKS(hipcub::Sum), - CREATE_BENCHMARK(custom_double2, hipcub::Sum), - CREATE_BENCHMARKS(hipcub::Min), + primbench::settings settings; + settings.size = 128 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; + + primbench::executor executor(argc, argv, settings); + + CREATE_BENCHMARKS(benchmark_utils::plus); + CREATE_BENCHMARK(custom_double2, benchmark_utils::plus); + CREATE_BENCHMARKS(benchmark_utils::minimum); #ifdef HIPCUB_ROCPRIM_API - CREATE_BENCHMARK(custom_double2, hipcub::Min), + CREATE_BENCHMARK(custom_double2, benchmark_utils::minimum); #endif - CREATE_BENCHMARKS(hipcub::ArgMin), + CREATE_BENCHMARKS(hipcub::ArgMin); #ifdef HIPCUB_ROCPRIM_API - CREATE_BENCHMARK(custom_double2, hipcub::ArgMin), + CREATE_BENCHMARK(custom_double2, hipcub::ArgMin); #endif - }; - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } - - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } - - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_reduce_by_key.cpp b/projects/hipcub/benchmark/benchmark_device_reduce_by_key.cpp index 0d9160f4c9c7..0558f016ae77 100644 --- a/projects/hipcub/benchmark/benchmark_device_reduce_by_key.cpp +++ b/projects/hipcub/benchmark/benchmark_device_reduce_by_key.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -26,222 +26,146 @@ #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #endif -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif - -const unsigned int batch_size = 10; -const unsigned int warmup_size = 5; - -template -void run_benchmark(benchmark::State& state, - size_t max_length, - hipStream_t stream, - size_t size, - BinaryFunction reduce_op) +template +class reduce_by_key_benchmark : public primbench::benchmark_interface { - using key_type = Key; - using value_type = Value; + static constexpr bool is_sum = std::is_same_v; + static constexpr bool is_min = std::is_same_v; + static_assert(is_sum || is_min, "unknown binary function"); - // Generate data - std::vector keys_input(size); + primbench::json meta() const override + { + return primbench::json{} + .add("algo", "device_reduce_by_key") + .add("lvl", "device") + .add("key_data_type", primbench::name()) + .add("value_data_type", primbench::name()) + .add("random_number_range", "[1, " + std::to_string(MaxLength) + "]") + .add("reduce_op", is_sum ? "sum" : (is_min ? "min" : "unknown")); + } - unsigned int unique_count = 0; - std::vector key_counts - = benchmark_utils::get_random_data(100000, 1, max_length); - size_t offset = 0; - while(offset < size) + void run(primbench::state& state) override { - const size_t key_count = key_counts[unique_count % key_counts.size()]; - const size_t end = std::min(size, offset + key_count); - for(size_t i = offset; i < end; i++) + const size_t items = state.size; + const auto& stream = state.stream; + + // Generate data + std::vector keys_input(items); + + unsigned int unique_count = 0; + std::vector key_counts + = benchmark_utils::get_random_data(100000, 1, MaxLength); + size_t offset = 0; + while(offset < items) { - keys_input[i] = unique_count; + const size_t key_count = key_counts[unique_count % key_counts.size()]; + const size_t end = _HIPCUB_STD::min(items, offset + key_count); + for(size_t i = offset; i < end; i++) + { + keys_input[i] = unique_count; + } + + unique_count++; + offset += key_count; } - unique_count++; - offset += key_count; - } + std::vector values_input(items); + std::iota(values_input.begin(), values_input.end(), 0); - std::vector values_input(size); - std::iota(values_input.begin(), values_input.end(), 0); - - key_type* d_keys_input; - HIP_CHECK(hipMalloc(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK( - hipMemcpy(d_keys_input, keys_input.data(), size * sizeof(key_type), hipMemcpyHostToDevice)); - - value_type* d_values_input; - HIP_CHECK(hipMalloc(&d_values_input, size * sizeof(value_type))); - HIP_CHECK(hipMemcpy(d_values_input, - values_input.data(), - size * sizeof(value_type), - hipMemcpyHostToDevice)); - - key_type* d_unique_output; - value_type* d_aggregates_output; - unsigned int* d_unique_count_output; - HIP_CHECK(hipMalloc(&d_unique_output, unique_count * sizeof(key_type))); - HIP_CHECK(hipMalloc(&d_aggregates_output, unique_count * sizeof(value_type))); - HIP_CHECK(hipMalloc(&d_unique_count_output, sizeof(unsigned int))); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - - HIP_CHECK(hipcub::DeviceReduce::ReduceByKey(nullptr, - temporary_storage_bytes, - d_keys_input, - d_unique_output, - d_values_input, - d_aggregates_output, - d_unique_count_output, - reduce_op, - size, - stream)); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) - { - HIP_CHECK(hipcub::DeviceReduce::ReduceByKey(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_unique_output, - d_values_input, - d_aggregates_output, - d_unique_count_output, - reduce_op, - size, - stream)); - } - HIP_CHECK(hipDeviceSynchronize()); + Key* d_keys_input; + HIP_CHECK(hipMalloc(&d_keys_input, items * sizeof(Key))); + HIP_CHECK( + hipMemcpy(d_keys_input, keys_input.data(), items * sizeof(Key), hipMemcpyHostToDevice)); - for(auto _ : state) - { - auto start = std::chrono::high_resolution_clock::now(); + Value* d_values_input; + HIP_CHECK(hipMalloc(&d_values_input, items * sizeof(Value))); + HIP_CHECK(hipMemcpy(d_values_input, + values_input.data(), + items * sizeof(Value), + hipMemcpyHostToDevice)); + + Key* d_unique_output; + Value* d_aggregates_output; + unsigned int* d_unique_count_output; + HIP_CHECK(hipMalloc(&d_unique_output, unique_count * sizeof(Key))); + HIP_CHECK(hipMalloc(&d_aggregates_output, unique_count * sizeof(Value))); + HIP_CHECK(hipMalloc(&d_unique_count_output, sizeof(unsigned int))); - for(size_t i = 0; i < batch_size; i++) + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + BinaryFunction reduce_op{}; + + const auto launch = [&] { - HIP_CHECK(hipcub::DeviceReduce::ReduceByKey(d_temporary_storage, - temporary_storage_bytes, + HIP_CHECK(hipcub::DeviceReduce::ReduceByKey(d_temp_storage, + temp_storage_bytes, d_keys_input, d_unique_output, d_values_input, d_aggregates_output, d_unique_count_output, reduce_op, - size, + items, stream)); - } - HIP_CHECK(hipStreamSynchronize(stream)); + }; + + launch(); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); + + state.set_items(items); + state.add_writes(items); + state.add_writes(items); + + state.run(launch); + + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_keys_input)); + HIP_CHECK(hipFree(d_values_input)); + HIP_CHECK(hipFree(d_unique_output)); + HIP_CHECK(hipFree(d_aggregates_output)); + HIP_CHECK(hipFree(d_unique_count_output)); } - state.SetBytesProcessed(state.iterations() * batch_size * size - * (sizeof(key_type) + sizeof(value_type))); - state.SetItemsProcessed(state.iterations() * batch_size * size); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_unique_output)); - HIP_CHECK(hipFree(d_aggregates_output)); - HIP_CHECK(hipFree(d_unique_count_output)); -} +}; -#define CREATE_BENCHMARK(Key, Value, REDUCE_OP) \ - benchmark::RegisterBenchmark(std::string("device_reduce_by_key" \ - "." \ - "(random_number_range:[1, " \ - + std::to_string(max_length) + "])") \ - .c_str(), \ - &run_benchmark, \ - max_length, \ - stream, \ - size, \ - REDUCE_OP()) - -#define CREATE_BENCHMARKS(REDUCE_OP) \ - CREATE_BENCHMARK(int, float, REDUCE_OP), CREATE_BENCHMARK(int, double, REDUCE_OP), \ - CREATE_BENCHMARK(int, custom_double2, REDUCE_OP), \ - CREATE_BENCHMARK(int8_t, int8_t, REDUCE_OP), \ - CREATE_BENCHMARK(long long, float, REDUCE_OP), \ - CREATE_BENCHMARK(long long, double, REDUCE_OP) - -void add_benchmarks(size_t max_length, - std::vector& benchmarks, - hipStream_t stream, - size_t size) -{ - using custom_double2 = benchmark_utils::custom_type; +#define CREATE_BENCHMARK(Key, Value, REDUCE_OP) \ + executor.queue>() + +#define CREATE_BENCHMARKS(REDUCE_OP) \ + CREATE_BENCHMARK(int, float, REDUCE_OP); \ + CREATE_BENCHMARK(int, double, REDUCE_OP); \ + CREATE_BENCHMARK(int, custom_double2, REDUCE_OP); \ + CREATE_BENCHMARK(int8_t, int8_t, REDUCE_OP); \ + CREATE_BENCHMARK(int64_t, float, REDUCE_OP); \ + CREATE_BENCHMARK(int64_t, double, REDUCE_OP) - std::vector bs = { - CREATE_BENCHMARKS(hipcub::Sum), - CREATE_BENCHMARK(long long, custom_double2, hipcub::Sum), - CREATE_BENCHMARKS(hipcub::Min), +template +void add_benchmarks(primbench::executor& executor) +{ + CREATE_BENCHMARKS(benchmark_utils::plus); + CREATE_BENCHMARK(int64_t, custom_double2, benchmark_utils::plus); + CREATE_BENCHMARKS(benchmark_utils::minimum); #ifdef HIPCUB_ROCPRIM_API - CREATE_BENCHMARK(long long, custom_double2, hipcub::Min), + CREATE_BENCHMARK(int64_t, custom_double2, benchmark_utils::minimum); #endif - }; - - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_device_reduce_by_key" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - add_benchmarks(1000, benchmarks, stream, size); - add_benchmarks(10, benchmarks, stream, size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + primbench::executor executor(argc, argv, settings); + + add_benchmarks<1000>(executor); + add_benchmarks<10>(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_run_length_encode.cpp b/projects/hipcub/benchmark/benchmark_device_run_length_encode.cpp index f0c858528a53..fb97a8535550 100644 --- a/projects/hipcub/benchmark/benchmark_device_run_length_encode.cpp +++ b/projects/hipcub/benchmark/benchmark_device_run_length_encode.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -26,332 +26,225 @@ #pragma GCC diagnostic ignored "-Wunused-parameter" #endif -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif - -template -void run_encode_benchmark(benchmark::State& state, - size_t max_length, - hipStream_t stream, - size_t size) +template +class encode_benchmark : public primbench::benchmark_interface { - using key_type = T; - using count_type = unsigned int; - - // Generate data - std::vector input(size); + primbench::json meta() const override + { + return primbench::json{} + .add("algo", "device_run_length_encode") + .add("subalgo", "encode") + .add("lvl", "device") + .add("data_type", primbench::name()) + .add("random_number_range", "[1, " + std::to_string(MaxLength) + "]"); + } - unsigned int runs_count = 0; - std::vector key_counts - = benchmark_utils::get_random_data(100000, 1, max_length); - size_t offset = 0; - while(offset < size) + void run(primbench::state& state) override { - const size_t key_count = key_counts[runs_count % key_counts.size()]; - const size_t end = std::min(size, offset + key_count); - for(size_t i = offset; i < end; i++) + const size_t items = state.size; + const auto& stream = state.stream; + + using Key = T; + using count_type = unsigned int; + + // Generate data + std::vector input(items); + + unsigned int runs_count = 0; + std::vector key_counts + = benchmark_utils::get_random_data(100000, 1, MaxLength); + size_t offset = 0; + while(offset < items) { - input[i] = runs_count; + const size_t key_count = key_counts[runs_count % key_counts.size()]; + const size_t end = _HIPCUB_STD::min(items, offset + key_count); + for(size_t i = offset; i < end; i++) + { + input[i] = runs_count; + } + + runs_count++; + offset += key_count; } - runs_count++; - offset += key_count; - } + Key* d_input; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(Key))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(Key), hipMemcpyHostToDevice)); - key_type* d_input; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(key_type), hipMemcpyHostToDevice)); - - key_type* d_unique_output; - count_type* d_counts_output; - count_type* d_runs_count_output; - HIP_CHECK(hipMalloc(&d_unique_output, runs_count * sizeof(key_type))); - HIP_CHECK(hipMalloc(&d_counts_output, runs_count * sizeof(count_type))); - HIP_CHECK(hipMalloc(&d_runs_count_output, sizeof(count_type))); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - - HIP_CHECK(hipcub::DeviceRunLengthEncode::Encode(nullptr, - temporary_storage_bytes, - d_input, - d_unique_output, - d_counts_output, - d_runs_count_output, - size, - stream)); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < 10; i++) - { - HIP_CHECK(hipcub::DeviceRunLengthEncode::Encode(d_temporary_storage, - temporary_storage_bytes, - d_input, - d_unique_output, - d_counts_output, - d_runs_count_output, - size, - stream)); - } - HIP_CHECK(hipDeviceSynchronize()); + Key* d_unique_output; + count_type* d_counts_output; + count_type* d_runs_count_output; + HIP_CHECK(hipMalloc(&d_unique_output, runs_count * sizeof(Key))); + HIP_CHECK(hipMalloc(&d_counts_output, runs_count * sizeof(count_type))); + HIP_CHECK(hipMalloc(&d_runs_count_output, sizeof(count_type))); - const unsigned int batch_size = 10; - for(auto _ : state) - { - auto start = std::chrono::high_resolution_clock::now(); + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; - for(size_t i = 0; i < batch_size; i++) + const auto launch = [&] { - HIP_CHECK(hipcub::DeviceRunLengthEncode::Encode(d_temporary_storage, - temporary_storage_bytes, + HIP_CHECK(hipcub::DeviceRunLengthEncode::Encode(d_temp_storage, + temp_storage_bytes, d_input, d_unique_output, d_counts_output, d_runs_count_output, - size, + items, stream)); - } - HIP_CHECK(hipStreamSynchronize(stream)); + }; - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + launch(); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); + + state.set_items(items); + state.add_writes(items); + + state.run(launch); + + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_unique_output)); + HIP_CHECK(hipFree(d_counts_output)); + HIP_CHECK(hipFree(d_runs_count_output)); } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(key_type)); - state.SetItemsProcessed(state.iterations() * batch_size * size); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_unique_output)); - HIP_CHECK(hipFree(d_counts_output)); - HIP_CHECK(hipFree(d_runs_count_output)); -} +}; -template -void run_non_trivial_runs_benchmark(benchmark::State& state, - size_t max_length, - hipStream_t stream, - size_t size) +template +class non_trivial_runs_benchmark : public primbench::benchmark_interface { - using key_type = T; - using offset_type = unsigned int; - using count_type = unsigned int; - - // Generate data - std::vector input(size); - - unsigned int runs_count = 0; - std::vector key_counts - = benchmark_utils::get_random_data(100000, 1, max_length); - size_t offset = 0; - while(offset < size) + primbench::json meta() const override { - const size_t key_count = key_counts[runs_count % key_counts.size()]; - const size_t end = std::min(size, offset + key_count); - for(size_t i = offset; i < end; i++) - { - input[i] = runs_count; - } - - runs_count++; - offset += key_count; + return primbench::json{} + .add("algo", "device_run_length_encode") + .add("subalgo", "non_trivial_runs") + .add("lvl", "device") + .add("data_type", primbench::name()) + .add("random_number_range", "[1, " + std::to_string(MaxLength) + "]"); } - key_type* d_input; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(key_type))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(key_type), hipMemcpyHostToDevice)); + void run(primbench::state& state) override + { + const size_t items = state.size; + const auto& stream = state.stream; - offset_type* d_offsets_output; - count_type* d_counts_output; - count_type* d_runs_count_output; - HIP_CHECK(hipMalloc(&d_offsets_output, runs_count * sizeof(offset_type))); - HIP_CHECK(hipMalloc(&d_counts_output, runs_count * sizeof(count_type))); - HIP_CHECK(hipMalloc(&d_runs_count_output, sizeof(count_type))); + using Key = T; + using offset_type = unsigned int; + using count_type = unsigned int; - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; + // Generate data + std::vector input(items); - HIP_CHECK(hipcub::DeviceRunLengthEncode::NonTrivialRuns(nullptr, - temporary_storage_bytes, - d_input, - d_offsets_output, - d_counts_output, - d_runs_count_output, - size, - stream)); + unsigned int runs_count = 0; + std::vector key_counts + = benchmark_utils::get_random_data(100000, 1, MaxLength); - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + size_t offset = 0; + while(offset < items) + { + const size_t key_count = key_counts[runs_count % key_counts.size()]; + const size_t end = _HIPCUB_STD::min(items, offset + key_count); + for(size_t i = offset; i < end; i++) + { + input[i] = runs_count; + } + + runs_count++; + offset += key_count; + } - // Warm-up - for(size_t i = 0; i < 10; i++) - { - HIP_CHECK(hipcub::DeviceRunLengthEncode::NonTrivialRuns(d_temporary_storage, - temporary_storage_bytes, - d_input, - d_offsets_output, - d_counts_output, - d_runs_count_output, - size, - stream)); - } - HIP_CHECK(hipDeviceSynchronize()); + Key* d_input; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(Key))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(Key), hipMemcpyHostToDevice)); - const unsigned int batch_size = 10; - for(auto _ : state) - { - auto start = std::chrono::high_resolution_clock::now(); + offset_type* d_offsets_output; + count_type* d_counts_output; + count_type* d_runs_count_output; + HIP_CHECK(hipMalloc(&d_offsets_output, runs_count * sizeof(offset_type))); + HIP_CHECK(hipMalloc(&d_counts_output, runs_count * sizeof(count_type))); + HIP_CHECK(hipMalloc(&d_runs_count_output, sizeof(count_type))); - for(size_t i = 0; i < batch_size; i++) + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { - HIP_CHECK(hipcub::DeviceRunLengthEncode::NonTrivialRuns(d_temporary_storage, - temporary_storage_bytes, + HIP_CHECK(hipcub::DeviceRunLengthEncode::NonTrivialRuns(d_temp_storage, + temp_storage_bytes, d_input, d_offsets_output, d_counts_output, d_runs_count_output, - size, + items, stream)); - } - HIP_CHECK(hipStreamSynchronize(stream)); + }; - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(key_type)); - state.SetItemsProcessed(state.iterations() * batch_size * size); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_offsets_output)); - HIP_CHECK(hipFree(d_counts_output)); - HIP_CHECK(hipFree(d_runs_count_output)); -} + launch(); -#define CREATE_ENCODE_BENCHMARK(T) \ - benchmark::RegisterBenchmark(std::string("device_run_length_encode" \ - "." \ - "(random_number_range:[1, " \ - + std::to_string(max_length) + "])") \ - .c_str(), \ - &run_encode_benchmark, \ - max_length, \ - stream, \ - size) - -void add_encode_benchmarks(size_t max_length, - std::vector& benchmarks, - hipStream_t stream, - size_t size) -{ - using custom_float2 = benchmark_utils::custom_type; - using custom_double2 = benchmark_utils::custom_type; + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); - std::vector bs = { - CREATE_ENCODE_BENCHMARK(int), - CREATE_ENCODE_BENCHMARK(long long), + state.set_items(items); + state.add_writes(items); - CREATE_ENCODE_BENCHMARK(int8_t), - CREATE_ENCODE_BENCHMARK(uint8_t), + state.run(launch); - CREATE_ENCODE_BENCHMARK(custom_float2), - CREATE_ENCODE_BENCHMARK(custom_double2), - }; + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_offsets_output)); + HIP_CHECK(hipFree(d_counts_output)); + HIP_CHECK(hipFree(d_runs_count_output)); + } +}; - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); -} +#define CREATE_ENCODE_BENCHMARK(T) executor.queue>() -#define CREATE_NON_TRIVIAL_RUNS_BENCHMARK(T) \ - benchmark::RegisterBenchmark(std::string("run_length_encode_non_trivial_runs" \ - "" \ - "(random_number_range:[1, " \ - + std::to_string(max_length) + "])") \ - .c_str(), \ - &run_non_trivial_runs_benchmark, \ - max_length, \ - stream, \ - size) - -void add_non_trivial_runs_benchmarks(size_t max_length, - std::vector& benchmarks, - hipStream_t stream, - size_t size) +template +void add_encode_benchmarks(primbench::executor& executor) { - using custom_float2 = benchmark_utils::custom_type; - using custom_double2 = benchmark_utils::custom_type; + CREATE_ENCODE_BENCHMARK(int); + CREATE_ENCODE_BENCHMARK(int64_t); + + CREATE_ENCODE_BENCHMARK(int8_t); + CREATE_ENCODE_BENCHMARK(uint8_t); + + CREATE_ENCODE_BENCHMARK(custom_float2); + CREATE_ENCODE_BENCHMARK(custom_double2); +} - std::vector bs = { - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int), - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(long long), +#define CREATE_NON_TRIVIAL_RUNS_BENCHMARK(T) \ + executor.queue>() - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int8_t), - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(uint8_t), +template +void add_non_trivial_runs_benchmarks(primbench::executor& executor) +{ + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int64_t); - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(custom_float2), - CREATE_NON_TRIVIAL_RUNS_BENCHMARK(custom_double2), - }; + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(int8_t); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(uint8_t); - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(custom_float2); + CREATE_NON_TRIVIAL_RUNS_BENCHMARK(custom_double2); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_device_run_length_encode" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - add_encode_benchmarks(1000, benchmarks, stream, size); - add_encode_benchmarks(10, benchmarks, stream, size); - add_non_trivial_runs_benchmarks(1000, benchmarks, stream, size); - add_non_trivial_runs_benchmarks(10, benchmarks, stream, size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + primbench::executor executor(argc, argv, settings); + + add_encode_benchmarks<1000>(executor); + add_encode_benchmarks<10>(executor); + add_non_trivial_runs_benchmarks<1000>(executor); + add_non_trivial_runs_benchmarks<10>(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_scan.cpp b/projects/hipcub/benchmark/benchmark_device_scan.cpp index 5d38b9628b5c..1204f4951f47 100644 --- a/projects/hipcub/benchmark/benchmark_device_scan.cpp +++ b/projects/hipcub/benchmark/benchmark_device_scan.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -26,14 +26,12 @@ #pragma GCC diagnostic ignored "-Wmaybe-uninitialized" #endif -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API +#include #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif +#include _HIPCUB_STD_INCLUDE(functional) template auto run_device_scan(void* temporary_storage, @@ -97,7 +95,7 @@ auto run_device_scan_by_key(void* temporary_storage, scan_op, initial_value, static_cast(input_size), - hipcub::Equality(), + _HIPCUB_STD::equal_to<>(), stream); } @@ -120,227 +118,183 @@ auto run_device_scan_by_key(void* temporary_storage, output, scan_op, static_cast(input_size), - hipcub::Equality(), + _HIPCUB_STD::equal_to<>(), stream); } -template -void run_benchmark(benchmark::State& state, - size_t size, - const hipStream_t stream, - BinaryFunction scan_op) +template +class scan_benchmark : public primbench::benchmark_interface { - std::vector input = benchmark_utils::get_random_data(size, T(0), T(1000)); - T initial_value = T(123); - T* d_input; - T* d_output; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMalloc(&d_output, size * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - // Allocate temporary storage memory - size_t temp_storage_size_bytes = 0; - void* d_temp_storage = nullptr; - // Get size of d_temp_storage - HIP_CHECK((run_device_scan(d_temp_storage, - temp_storage_size_bytes, - d_input, - d_output, - initial_value, - size, - scan_op, - stream))); - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < 5; i++) + primbench::json meta() const override { - HIP_CHECK((run_device_scan(d_temp_storage, - temp_storage_size_bytes, - d_input, - d_output, - initial_value, - size, - scan_op, - stream))); + return primbench::json{} + .add("algo", "device_scan") + .add("subalgo", "scan") + .add("lvl", "device") + .add("exclusive", Exclusive) + .add("data_type", primbench::name()) + .add("op", Tag::name); } - HIP_CHECK(hipDeviceSynchronize()); - const unsigned int batch_size = 10; - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); - for(size_t i = 0; i < batch_size; i++) - { - HIP_CHECK((run_device_scan(d_temp_storage, - temp_storage_size_bytes, - d_input, - d_output, - initial_value, - size, - scan_op, - stream))); - } - HIP_CHECK(hipStreamSynchronize(stream)); - - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * batch_size * size); + const size_t items = state.size; + const auto& stream = state.stream; - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); -} + BinaryFunction scan_op{}; -template -void run_benchmark_by_key(benchmark::State& state, - size_t size, - const hipStream_t stream, - BinaryFunction scan_op) -{ - using key_type = int; - constexpr size_t max_segment_length = 100; - - const std::vector keys - = benchmark_utils::get_random_segments(size, - max_segment_length, - std::random_device{}()); - const std::vector input = benchmark_utils::get_random_data(size, T(0), T(1000)); - const T initial_value = T(123); - key_type* d_keys; - T* d_input; - T* d_output; - HIP_CHECK(hipMalloc(&d_keys, size * sizeof(key_type))); - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMalloc(&d_output, size * sizeof(T))); - HIP_CHECK(hipMemcpy(d_keys, keys.data(), size * sizeof(key_type), hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - // Allocate temporary storage memory - size_t temp_storage_size_bytes = 0; - void* d_temp_storage = nullptr; - // Get size of d_temp_storage - HIP_CHECK((run_device_scan_by_key(d_temp_storage, - temp_storage_size_bytes, - d_keys, + std::vector input = benchmark_utils::get_random_data(items, T(0), T(1000)); + T initial_value = T(123); + T* d_input; + T* d_output; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK(hipMalloc(&d_output, items * sizeof(T))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK(hipDeviceSynchronize()); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] + { + HIP_CHECK(run_device_scan(d_temp_storage, + temp_storage_bytes, d_input, d_output, initial_value, - size, + items, scan_op, - stream))); - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); + stream)); + }; + + launch(); - // Warm-up - for(size_t i = 0; i < 5; i++) + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); + + state.set_items(items); + state.add_writes(items); + + state.run(launch); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); + HIP_CHECK(hipFree(d_temp_storage)); + } +}; + +template +class scan_by_key_benchmark : public primbench::benchmark_interface +{ + primbench::json meta() const override { - HIP_CHECK((run_device_scan_by_key(d_temp_storage, - temp_storage_size_bytes, - d_keys, - d_input, - d_output, - initial_value, - size, - scan_op, - stream))); + return primbench::json{} + .add("algo", "device_scan") + .add("subalgo", "scan_by_key") + .add("lvl", "device") + .add("exclusive", Exclusive) + .add("data_type", primbench::name()) + .add("op", Tag::name); } - HIP_CHECK(hipDeviceSynchronize()); - const unsigned int batch_size = 10; - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); - for(size_t i = 0; i < batch_size; i++) + const size_t items = state.size; + const auto& stream = state.stream; + + BinaryFunction scan_op{}; + + using Key = int; + constexpr size_t max_segment_length = 100; + + const std::vector keys + = benchmark_utils::get_random_segments(items, + max_segment_length, + std::random_device{}()); + const std::vector input = benchmark_utils::get_random_data(items, T(0), T(1000)); + const T initial_value = T(123); + Key* d_keys; + T* d_input; + T* d_output; + HIP_CHECK(hipMalloc(&d_keys, items * sizeof(Key))); + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK(hipMalloc(&d_output, items * sizeof(T))); + HIP_CHECK(hipMemcpy(d_keys, keys.data(), items * sizeof(Key), hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK(hipDeviceSynchronize()); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { - HIP_CHECK((run_device_scan_by_key(d_temp_storage, - temp_storage_size_bytes, - d_keys, - d_input, - d_output, - initial_value, - size, - scan_op, - stream))); - } - HIP_CHECK(hipStreamSynchronize(stream)); - - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + HIP_CHECK(run_device_scan_by_key(d_temp_storage, + temp_storage_bytes, + d_keys, + d_input, + d_output, + initial_value, + items, + scan_op, + stream)); + }; + + launch(); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); + + state.set_items(items); + state.add_writes(items); + + state.run(launch); + + HIP_CHECK(hipFree(d_keys)); + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); + HIP_CHECK(hipFree(d_temp_storage)); } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * batch_size * size); +}; - HIP_CHECK(hipFree(d_keys)); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_temp_storage)); -} +struct sum_tag +{ + static constexpr const char* name = "sum"; +}; -#define CREATE_BENCHMARK(EXCL, T, SCAN_OP) \ - benchmark::RegisterBenchmark( \ - std::string(std::string(EXCL ? "device_exclusive_scan" : "device_inclusive_scan") \ - + ".") \ - .c_str(), \ - &run_benchmark, \ - size, \ - stream, \ - SCAN_OP()), \ - benchmark::RegisterBenchmark( \ - std::string(std::string(EXCL ? "device_exclusive_scan_by_key" \ - : "device_inclusive_scan_by_key") \ - + ".") \ - .c_str(), \ - &run_benchmark_by_key, \ - size, \ - stream, \ - SCAN_OP()) - -#define CREATE_BENCHMARKS(SCAN_OP) \ - CREATE_BENCHMARK(false, int, SCAN_OP), CREATE_BENCHMARK(true, int, SCAN_OP), \ - CREATE_BENCHMARK(false, float, SCAN_OP), CREATE_BENCHMARK(true, float, SCAN_OP), \ - CREATE_BENCHMARK(false, double, SCAN_OP), CREATE_BENCHMARK(true, double, SCAN_OP), \ - CREATE_BENCHMARK(false, long long, SCAN_OP), CREATE_BENCHMARK(true, long long, SCAN_OP), \ - CREATE_BENCHMARK(false, custom_float2, SCAN_OP), \ - CREATE_BENCHMARK(true, custom_float2, SCAN_OP), \ - CREATE_BENCHMARK(false, custom_double2, SCAN_OP), \ - CREATE_BENCHMARK(true, custom_double2, SCAN_OP), CREATE_BENCHMARK(false, int8_t, SCAN_OP), \ - CREATE_BENCHMARK(true, int8_t, SCAN_OP), CREATE_BENCHMARK(false, uint8_t, SCAN_OP), \ - CREATE_BENCHMARK(true, uint8_t, SCAN_OP) +struct min_tag +{ + static constexpr const char* name = "min"; +}; + +#define CREATE_BENCHMARK(EXCL, T, SCAN_OP, TAG) \ + executor.queue>(); \ + executor.queue>() + +#define CREATE_BENCHMARKS(SCAN_OP, TAG) \ + CREATE_BENCHMARK(false, int, SCAN_OP, TAG); \ + CREATE_BENCHMARK(true, int, SCAN_OP, TAG); \ + CREATE_BENCHMARK(false, float, SCAN_OP, TAG); \ + CREATE_BENCHMARK(true, float, SCAN_OP, TAG); \ + CREATE_BENCHMARK(false, double, SCAN_OP, TAG); \ + CREATE_BENCHMARK(true, double, SCAN_OP, TAG); \ + CREATE_BENCHMARK(false, int64_t, SCAN_OP, TAG); \ + CREATE_BENCHMARK(true, int64_t, SCAN_OP, TAG); \ + CREATE_BENCHMARK(false, custom_float2, SCAN_OP, TAG); \ + CREATE_BENCHMARK(true, custom_float2, SCAN_OP, TAG); \ + CREATE_BENCHMARK(false, custom_double2, SCAN_OP, TAG); \ + CREATE_BENCHMARK(true, custom_double2, SCAN_OP, TAG); \ + CREATE_BENCHMARK(false, int8_t, SCAN_OP, TAG); \ + CREATE_BENCHMARK(true, int8_t, SCAN_OP, TAG); \ + CREATE_BENCHMARK(false, uint8_t, SCAN_OP, TAG); \ + CREATE_BENCHMARK(true, uint8_t, SCAN_OP, TAG) int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_device_scan" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; - using custom_double2 = benchmark_utils::custom_type; - using custom_float2 = benchmark_utils::custom_type; + primbench::executor executor(argc, argv, settings); // Compilation may never finish, if the compiler needs to compile too many // kernels, it is recommended to compile benchmarks only for 1-2 types when @@ -348,29 +302,8 @@ int main(int argc, char* argv[]) // commented/removed). // Add benchmarks - std::vector benchmarks = { - CREATE_BENCHMARKS(hipcub::Sum), - CREATE_BENCHMARKS(hipcub::Min), - }; - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } - - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } - - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); + CREATE_BENCHMARKS(benchmark_utils::plus, sum_tag); + CREATE_BENCHMARKS(benchmark_utils::minimum, min_tag); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_segmented_radix_sort.cpp b/projects/hipcub/benchmark/benchmark_device_segmented_radix_sort.cpp index 252e8ff34062..c1ad013d4371 100644 --- a/projects/hipcub/benchmark/benchmark_device_segmented_radix_sort.cpp +++ b/projects/hipcub/benchmark/benchmark_device_segmented_radix_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,452 +20,324 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif - -const unsigned int batch_size = 4; -const unsigned int warmup_size = 2; - -constexpr bool Ascending = false; -constexpr bool Descending = true; - -template -void run_sort_keys_benchmark(benchmark::State& state, - size_t desired_segments, - hipStream_t stream, - size_t size, - bool descending = false) +template +class sort_keys_benchmark : public primbench::benchmark_interface { - using offset_type = int; - using key_type = Key; - using sort_func = hipError_t (*)(void*, - size_t&, - const key_type*, - key_type*, - int, - int, - offset_type*, - offset_type*, - int, - int, - hipStream_t); - - sort_func func_ascending = &hipcub::DeviceSegmentedRadixSort::SortKeys; - sort_func func_descending - = &hipcub::DeviceSegmentedRadixSort::SortKeysDescending; - - sort_func sorting = descending ? func_descending : func_ascending; - - // Generate data - std::vector offsets; - - const double avg_segment_length = static_cast(size) / desired_segments; - - const unsigned int seed = 123; - std::default_random_engine gen(seed); - - std::uniform_real_distribution segment_length_dis(0, avg_segment_length * 2); - - unsigned int segments_count = 0; - size_t offset = 0; - while(offset < size) - { - const size_t segment_length = std::round(segment_length_dis(gen)); - offsets.push_back(offset); - segments_count++; - offset += segment_length; - } - offsets.push_back(size); - - std::vector keys_input = benchmark_utils::get_random_data( - size, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); - - offset_type* d_offsets; - HIP_CHECK(hipMalloc(&d_offsets, (segments_count + 1) * sizeof(offset_type))); - HIP_CHECK(hipMemcpy(d_offsets, - offsets.data(), - (segments_count + 1) * sizeof(offset_type), - hipMemcpyHostToDevice)); - - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(hipMalloc(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK(hipMalloc(&d_keys_output, size * sizeof(key_type))); - HIP_CHECK( - hipMemcpy(d_keys_input, keys_input.data(), size * sizeof(key_type), hipMemcpyHostToDevice)); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - HIP_CHECK(sorting(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - size, - segments_count, - d_offsets, - d_offsets + 1, - 0, - sizeof(key_type) * 8, - stream)); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) + primbench::json meta() const override { - HIP_CHECK(sorting(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - size, - segments_count, - d_offsets, - d_offsets + 1, - 0, - sizeof(key_type) * 8, - stream)); + return primbench::json{} + .add("algo", "device_segmented_radix_sort") + .add("subalgo", "sort_keys") + .add("lvl", "device") + .add("key_data_type", primbench::name()) + .add("ascending", !Descending) + .add("desired_segments", DesiredSegments); } - HIP_CHECK(hipDeviceSynchronize()); - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); + const size_t items = state.size; + const auto& stream = state.stream; + + using offset_type = int; + using sort_func = hipError_t (*)(void*, + size_t&, + const Key*, + Key*, + int, + int, + offset_type*, + offset_type*, + int, + int, + hipStream_t); + + sort_func func_ascending = &hipcub::DeviceSegmentedRadixSort::SortKeys; + sort_func func_descending + = &hipcub::DeviceSegmentedRadixSort::SortKeysDescending; + + sort_func sorting = Descending ? func_descending : func_ascending; + + // Generate data + std::vector offsets; + + const double avg_segment_length = static_cast(items) / DesiredSegments; - for(size_t i = 0; i < batch_size; i++) + const unsigned int seed = 123; + std::default_random_engine gen(seed); + + std::uniform_real_distribution segment_length_dis(0, avg_segment_length * 2); + + unsigned int segments_count = 0; + size_t offset = 0; + while(offset < items) { - HIP_CHECK(sorting(d_temporary_storage, - temporary_storage_bytes, + const size_t segment_length = std::round(segment_length_dis(gen)); + offsets.push_back(offset); + segments_count++; + offset += segment_length; + } + offsets.push_back(items); + + std::vector keys_input + = benchmark_utils::get_random_data(items, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); + + offset_type* d_offsets; + HIP_CHECK(hipMalloc(&d_offsets, (segments_count + 1) * sizeof(offset_type))); + HIP_CHECK(hipMemcpy(d_offsets, + offsets.data(), + (segments_count + 1) * sizeof(offset_type), + hipMemcpyHostToDevice)); + + Key* d_keys_input; + Key* d_keys_output; + HIP_CHECK(hipMalloc(&d_keys_input, items * sizeof(Key))); + HIP_CHECK(hipMalloc(&d_keys_output, items * sizeof(Key))); + HIP_CHECK( + hipMemcpy(d_keys_input, keys_input.data(), items * sizeof(Key), hipMemcpyHostToDevice)); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] + { + HIP_CHECK(sorting(d_temp_storage, + temp_storage_bytes, d_keys_input, d_keys_output, - size, + items, segments_count, d_offsets, d_offsets + 1, 0, - sizeof(key_type) * 8, + sizeof(Key) * 8, stream)); - } + }; + + launch(); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(key_type)); - state.SetItemsProcessed(state.iterations() * batch_size * size); + state.set_items(items); + state.add_writes(items); - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_offsets)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_keys_output)); -} + state.run(launch); -template -void run_sort_pairs_benchmark(benchmark::State& state, - size_t desired_segments, - hipStream_t stream, - size_t size, - bool descending = false) -{ - using offset_type = int; - using key_type = Key; - using value_type = Value; - using sort_func = hipError_t (*)(void*, - size_t&, - const key_type*, - key_type*, - const value_type*, - value_type*, - int, - int, - offset_type*, - offset_type*, - int, - int, - hipStream_t); - - sort_func func_ascending - = &hipcub::DeviceSegmentedRadixSort::SortPairs; - sort_func func_descending = &hipcub::DeviceSegmentedRadixSort:: - SortPairsDescending; - - sort_func sorting = descending ? func_descending : func_ascending; - - // Generate data - std::vector offsets; - - const double avg_segment_length = static_cast(size) / desired_segments; - - const unsigned int seed = 123; - std::default_random_engine gen(seed); - - std::uniform_real_distribution segment_length_dis(0, avg_segment_length * 2); - - unsigned int segments_count = 0; - size_t offset = 0; - while(offset < size) - { - const size_t segment_length = std::round(segment_length_dis(gen)); - offsets.push_back(offset); - segments_count++; - offset += segment_length; + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_offsets)); + HIP_CHECK(hipFree(d_keys_input)); + HIP_CHECK(hipFree(d_keys_output)); } - offsets.push_back(size); - - std::vector keys_input = benchmark_utils::get_random_data( - size, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); - - std::vector values_input(size); - std::iota(values_input.begin(), values_input.end(), 0); - - offset_type* d_offsets; - HIP_CHECK(hipMalloc(&d_offsets, (segments_count + 1) * sizeof(offset_type))); - HIP_CHECK(hipMemcpy(d_offsets, - offsets.data(), - (segments_count + 1) * sizeof(offset_type), - hipMemcpyHostToDevice)); - - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(hipMalloc(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK(hipMalloc(&d_keys_output, size * sizeof(key_type))); - HIP_CHECK( - hipMemcpy(d_keys_input, keys_input.data(), size * sizeof(key_type), hipMemcpyHostToDevice)); - - value_type* d_values_input; - value_type* d_values_output; - HIP_CHECK(hipMalloc(&d_values_input, size * sizeof(value_type))); - HIP_CHECK(hipMalloc(&d_values_output, size * sizeof(value_type))); - HIP_CHECK(hipMemcpy(d_values_input, - values_input.data(), - size * sizeof(value_type), - hipMemcpyHostToDevice)); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - HIP_CHECK(sorting(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, - size, - segments_count, - d_offsets, - d_offsets + 1, - 0, - sizeof(key_type) * 8, - stream)); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) +}; + +template +class sort_pairs_benchmark : public primbench::benchmark_interface +{ + primbench::json meta() const override { - HIP_CHECK(sorting(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, - size, - segments_count, - d_offsets, - d_offsets + 1, - 0, - sizeof(key_type) * 8, - stream)); + return primbench::json{} + .add("algo", "device_segmented_radix_sort") + .add("subalgo", "sort_pairs") + .add("lvl", "device") + .add("key_data_type", primbench::name()) + .add("value_data_type", primbench::name()) + .add("ascending", !Descending) + .add("desired_segments", DesiredSegments); } - HIP_CHECK(hipDeviceSynchronize()); - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); - - for(size_t i = 0; i < batch_size; i++) + const size_t items = state.size; + const auto& stream = state.stream; + + using offset_type = int; + using sort_func = hipError_t (*)(void*, + size_t&, + const Key*, + Key*, + const Value*, + Value*, + int, + int, + offset_type*, + offset_type*, + int, + int, + hipStream_t); + + sort_func func_ascending + = &hipcub::DeviceSegmentedRadixSort::SortPairs; + sort_func func_descending + = &hipcub::DeviceSegmentedRadixSort::SortPairsDescending; + + sort_func sorting = Descending ? func_descending : func_ascending; + + // Generate data + std::vector offsets; + + const double avg_segment_length = static_cast(items) / DesiredSegments; + + const unsigned int seed = 123; + std::default_random_engine gen(seed); + + std::uniform_real_distribution segment_length_dis(0, avg_segment_length * 2); + + unsigned int segments_count = 0; + size_t offset = 0; + while(offset < items) + { + const size_t segment_length = std::round(segment_length_dis(gen)); + offsets.push_back(offset); + segments_count++; + offset += segment_length; + } + offsets.push_back(items); + + std::vector keys_input + = benchmark_utils::get_random_data(items, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); + + std::vector values_input(items); + std::iota(values_input.begin(), values_input.end(), 0); + + offset_type* d_offsets; + HIP_CHECK(hipMalloc(&d_offsets, (segments_count + 1) * sizeof(offset_type))); + HIP_CHECK(hipMemcpy(d_offsets, + offsets.data(), + (segments_count + 1) * sizeof(offset_type), + hipMemcpyHostToDevice)); + + Key* d_keys_input; + Key* d_keys_output; + HIP_CHECK(hipMalloc(&d_keys_input, items * sizeof(Key))); + HIP_CHECK(hipMalloc(&d_keys_output, items * sizeof(Key))); + HIP_CHECK( + hipMemcpy(d_keys_input, keys_input.data(), items * sizeof(Key), hipMemcpyHostToDevice)); + + Value* d_values_input; + Value* d_values_output; + HIP_CHECK(hipMalloc(&d_values_input, items * sizeof(Value))); + HIP_CHECK(hipMalloc(&d_values_output, items * sizeof(Value))); + HIP_CHECK(hipMemcpy(d_values_input, + values_input.data(), + items * sizeof(Value), + hipMemcpyHostToDevice)); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { - HIP_CHECK(sorting(d_temporary_storage, - temporary_storage_bytes, + HIP_CHECK(sorting(d_temp_storage, + temp_storage_bytes, d_keys_input, d_keys_output, d_values_input, d_values_output, - size, + items, segments_count, d_offsets, d_offsets + 1, 0, - sizeof(key_type) * 8, + sizeof(Key) * 8, stream)); - } + }; + + launch(); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * size - * (sizeof(key_type) + sizeof(value_type))); - state.SetItemsProcessed(state.iterations() * batch_size * size); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_offsets)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_values_output)); -} + state.set_items(items); + state.add_writes(items); + state.add_writes(items); + + state.run(launch); -#define CREATE_SORT_KEYS_BENCHMARK(Key, SEGMENTS) \ - benchmark::RegisterBenchmark( \ - std::string("device_segmented_radix_sort_keys" \ - "." \ - "(segments:~" \ - + std::to_string(SEGMENTS) + " segments)") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_sort_keys_benchmark(state, SEGMENTS, stream, size, Ascending); }) - -#define CREATE_SORT_KEYS_DESCENDING_BENCHMARK(Key, SEGMENTS) \ - benchmark::RegisterBenchmark( \ - std::string("device_segmented_radix_sort_keys" \ - "." \ - "(segments:~" \ - + std::to_string(SEGMENTS) + " segments)") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_sort_keys_benchmark(state, SEGMENTS, stream, size, Descending); }) - -#define BENCHMARK_KEY_TYPE(type) \ - CREATE_SORT_KEYS_BENCHMARK(type, 1), CREATE_SORT_KEYS_BENCHMARK(type, 10), \ - CREATE_SORT_KEYS_BENCHMARK(type, 100), CREATE_SORT_KEYS_BENCHMARK(type, 1000), \ - CREATE_SORT_KEYS_BENCHMARK(type, 10000), CREATE_SORT_KEYS_DESCENDING_BENCHMARK(type, 1), \ - CREATE_SORT_KEYS_DESCENDING_BENCHMARK(type, 10), \ - CREATE_SORT_KEYS_DESCENDING_BENCHMARK(type, 100), \ - CREATE_SORT_KEYS_DESCENDING_BENCHMARK(type, 1000), \ - CREATE_SORT_KEYS_DESCENDING_BENCHMARK(type, 10000) - -void add_sort_keys_benchmarks(std::vector& benchmarks, - hipStream_t stream, - size_t size) + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_offsets)); + HIP_CHECK(hipFree(d_keys_input)); + HIP_CHECK(hipFree(d_keys_output)); + HIP_CHECK(hipFree(d_values_input)); + HIP_CHECK(hipFree(d_values_output)); + } +}; + +#define CREATE_SORT_KEYS_BENCHMARK(Key, SEGMENTS) \ + executor.queue>() + +#define CREATE_SORT_KEYS_DESCENDING_BENCHMARK(Key, SEGMENTS) \ + executor.queue>() + +#define BENCHMARK_KEY_TYPE(type) \ + CREATE_SORT_KEYS_BENCHMARK(type, 1); \ + CREATE_SORT_KEYS_BENCHMARK(type, 10); \ + CREATE_SORT_KEYS_BENCHMARK(type, 100); \ + CREATE_SORT_KEYS_BENCHMARK(type, 1000); \ + CREATE_SORT_KEYS_BENCHMARK(type, 10000); \ + CREATE_SORT_KEYS_DESCENDING_BENCHMARK(type, 1); \ + CREATE_SORT_KEYS_DESCENDING_BENCHMARK(type, 10); \ + CREATE_SORT_KEYS_DESCENDING_BENCHMARK(type, 100); \ + CREATE_SORT_KEYS_DESCENDING_BENCHMARK(type, 1000); \ + CREATE_SORT_KEYS_DESCENDING_BENCHMARK(type, 10000) + +void add_sort_keys_benchmarks(primbench::executor& executor) { - std::vector bs = { - BENCHMARK_KEY_TYPE(float), - BENCHMARK_KEY_TYPE(double), - BENCHMARK_KEY_TYPE(int8_t), - BENCHMARK_KEY_TYPE(uint8_t), - BENCHMARK_KEY_TYPE(int), - }; - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + BENCHMARK_KEY_TYPE(float); + BENCHMARK_KEY_TYPE(double); + BENCHMARK_KEY_TYPE(int8_t); + BENCHMARK_KEY_TYPE(uint8_t); + BENCHMARK_KEY_TYPE(int); } -#define CREATE_SORT_PAIRS_BENCHMARK(Key, Value, SEGMENTS) \ - benchmark::RegisterBenchmark( \ - std::string("device_segmented_radix_sort_pairs" \ - "." \ - "(segments:~" \ - + std::to_string(SEGMENTS) + " segments)") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_sort_pairs_benchmark(state, SEGMENTS, stream, size, Ascending); }) - -#define CREATE_SORT_PAIRS_DESCENDING_BENCHMARK(Key, Value, SEGMENTS) \ - benchmark::RegisterBenchmark( \ - std::string("device_segmented_radix_sort_pairs" \ - "." \ - "(segments:~" \ - + std::to_string(SEGMENTS) + " segments)") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_sort_pairs_benchmark(state, SEGMENTS, stream, size, Descending); }) - -#define BENCHMARK_PAIR_TYPE(type, value) \ - CREATE_SORT_PAIRS_BENCHMARK(type, value, 1), CREATE_SORT_PAIRS_BENCHMARK(type, value, 10), \ - CREATE_SORT_PAIRS_BENCHMARK(type, value, 100), \ - CREATE_SORT_PAIRS_BENCHMARK(type, value, 1000), \ - CREATE_SORT_PAIRS_BENCHMARK(type, value, 10000), \ - CREATE_SORT_PAIRS_DESCENDING_BENCHMARK(type, value, 1), \ - CREATE_SORT_PAIRS_DESCENDING_BENCHMARK(type, value, 10), \ - CREATE_SORT_PAIRS_DESCENDING_BENCHMARK(type, value, 100), \ - CREATE_SORT_PAIRS_DESCENDING_BENCHMARK(type, value, 1000), \ - CREATE_SORT_PAIRS_DESCENDING_BENCHMARK(type, value, 10000) - -void add_sort_pairs_benchmarks(std::vector& benchmarks, - hipStream_t stream, - size_t size) +#define CREATE_SORT_PAIRS_BENCHMARK(Key, Value, SEGMENTS) \ + executor.queue>() + +#define CREATE_SORT_PAIRS_DESCENDING_BENCHMARK(Key, Value, SEGMENTS) \ + executor.queue>() + +#define BENCHMARK_PAIR_TYPE(type, value) \ + CREATE_SORT_PAIRS_BENCHMARK(type, value, 1); \ + CREATE_SORT_PAIRS_BENCHMARK(type, value, 10); \ + CREATE_SORT_PAIRS_BENCHMARK(type, value, 100); \ + CREATE_SORT_PAIRS_BENCHMARK(type, value, 1000); \ + CREATE_SORT_PAIRS_BENCHMARK(type, value, 10000); \ + CREATE_SORT_PAIRS_DESCENDING_BENCHMARK(type, value, 1); \ + CREATE_SORT_PAIRS_DESCENDING_BENCHMARK(type, value, 10); \ + CREATE_SORT_PAIRS_DESCENDING_BENCHMARK(type, value, 100); \ + CREATE_SORT_PAIRS_DESCENDING_BENCHMARK(type, value, 1000); \ + CREATE_SORT_PAIRS_DESCENDING_BENCHMARK(type, value, 10000) + +void add_sort_pairs_benchmarks(primbench::executor& executor) { - using custom_float2 = benchmark_utils::custom_type; - using custom_double2 = benchmark_utils::custom_type; - - std::vector bs = { - BENCHMARK_PAIR_TYPE(int, float), - BENCHMARK_PAIR_TYPE(long long, double), - BENCHMARK_PAIR_TYPE(int8_t, int8_t), - BENCHMARK_PAIR_TYPE(uint8_t, uint8_t), - BENCHMARK_PAIR_TYPE(int, custom_float2), - BENCHMARK_PAIR_TYPE(long long, custom_double2), - }; - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + BENCHMARK_PAIR_TYPE(int, float); + BENCHMARK_PAIR_TYPE(int64_t, double); + BENCHMARK_PAIR_TYPE(int8_t, int8_t); + BENCHMARK_PAIR_TYPE(uint8_t, uint8_t); + BENCHMARK_PAIR_TYPE(int, custom_float2); + BENCHMARK_PAIR_TYPE(int64_t, custom_double2); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_device_segmented_radix_sort" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - add_sort_keys_benchmarks(benchmarks, stream, size); - add_sort_pairs_benchmarks(benchmarks, stream, size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 1000; + settings.batch_window_size = 3; + settings.noise_tolerance_percent = 3; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + primbench::executor executor(argc, argv, settings, primbench::flags::sync); + + add_sort_keys_benchmarks(executor); + add_sort_pairs_benchmarks(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_segmented_reduce.cpp b/projects/hipcub/benchmark/benchmark_device_segmented_reduce.cpp index 5ba4a94284fb..b2bdaca85245 100644 --- a/projects/hipcub/benchmark/benchmark_device_segmented_reduce.cpp +++ b/projects/hipcub/benchmark/benchmark_device_segmented_reduce.cpp @@ -20,263 +20,201 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif - -const unsigned int batch_size = 10; -const unsigned int warmup_size = 5; - using OffsetType = int; -template -void run_benchmark(benchmark::State& state, - size_t desired_segments, - hipStream_t stream, - size_t size, - SegmentedReduceKernel segmented_reduce) +template +class reduce_benchmark : public primbench::benchmark_interface { - using value_type = T; - - // Generate data - const unsigned int seed = 123; - std::default_random_engine gen(seed); - - // The minimal average length should at least be 1 to prevent infinite loop. - const double avg_segment_length = std::max(1.0, static_cast(size) / desired_segments); - std::uniform_real_distribution segment_length_dis(0, avg_segment_length * 2); - - std::vector offsets; - unsigned int segments_count = 0; - size_t offset = 0; - while(offset < size) - { - const size_t segment_length = std::round(segment_length_dis(gen)); - offsets.push_back(offset); - segments_count++; - offset += segment_length; - } - offsets.push_back(size); - - std::vector values_input(size); - std::iota(values_input.begin(), values_input.end(), 0); - - OffsetType* d_offsets; - HIP_CHECK(hipMalloc(&d_offsets, (segments_count + 1) * sizeof(OffsetType))); - HIP_CHECK(hipMemcpy(d_offsets, - offsets.data(), - (segments_count + 1) * sizeof(OffsetType), - hipMemcpyHostToDevice)); - - value_type* d_values_input; - HIP_CHECK(hipMalloc(&d_values_input, size * sizeof(value_type))); - HIP_CHECK(hipMemcpy(d_values_input, - values_input.data(), - size * sizeof(value_type), - hipMemcpyHostToDevice)); - - OutputT* d_aggregates_output; - HIP_CHECK(hipMalloc(&d_aggregates_output, segments_count * sizeof(OutputT))); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - - HIP_CHECK(segmented_reduce(d_temporary_storage, - temporary_storage_bytes, - d_values_input, - d_aggregates_output, - segments_count, - d_offsets, - d_offsets + 1, - stream)); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) + primbench::json meta() const override { - HIP_CHECK(segmented_reduce(d_temporary_storage, - temporary_storage_bytes, - d_values_input, - d_aggregates_output, - segments_count, - d_offsets, - d_offsets + 1, - stream)); + return primbench::json{} + .add("algo", "device_segmented_reduce") + .add("lvl", "device") + .add("reduce_op", SegmentedReduceKernel::name) + .add("data_type", primbench::name()) + .add("desired_segments", DesiredSegments); } - HIP_CHECK(hipDeviceSynchronize()); - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); + const size_t items = state.size; + const auto& stream = state.stream; + + const auto segmented_reduce = SegmentedReduceKernel::kernel; + + using Value = T; + + // Generate data + const unsigned int seed = 123; + std::default_random_engine gen(seed); + + const double avg_segment_length + = std::max(1.0, static_cast(items) / DesiredSegments); + std::uniform_real_distribution segment_length_dis(0, avg_segment_length * 2); + + std::vector offsets; + unsigned int segments_count = 0; + size_t offset = 0; + while(offset < items) + { + const size_t segment_length = std::round(segment_length_dis(gen)); + offsets.push_back(offset); + segments_count++; + offset += segment_length; + } + offsets.push_back(items); + + std::vector values_input(items); + std::iota(values_input.begin(), values_input.end(), 0); + + OffsetType* d_offsets; + HIP_CHECK(hipMalloc(&d_offsets, (segments_count + 1) * sizeof(OffsetType))); + HIP_CHECK(hipMemcpy(d_offsets, + offsets.data(), + (segments_count + 1) * sizeof(OffsetType), + hipMemcpyHostToDevice)); + + Value* d_values_input; + HIP_CHECK(hipMalloc(&d_values_input, items * sizeof(Value))); + HIP_CHECK(hipMemcpy(d_values_input, + values_input.data(), + items * sizeof(Value), + hipMemcpyHostToDevice)); - for(size_t i = 0; i < batch_size; i++) + OutputT* d_aggregates_output; + HIP_CHECK(hipMalloc(&d_aggregates_output, segments_count * sizeof(OutputT))); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { - HIP_CHECK(segmented_reduce(d_temporary_storage, - temporary_storage_bytes, + HIP_CHECK(segmented_reduce(d_temp_storage, + temp_storage_bytes, d_values_input, d_aggregates_output, segments_count, d_offsets, d_offsets + 1, stream)); - } - HIP_CHECK(hipStreamSynchronize(stream)); + }; - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(value_type)); - state.SetItemsProcessed(state.iterations() * batch_size * size); + launch(); - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_offsets)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_aggregates_output)); -} + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); + + state.set_items(items); + state.add_writes(items); + + state.run(launch); + + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_offsets)); + HIP_CHECK(hipFree(d_values_input)); + HIP_CHECK(hipFree(d_aggregates_output)); + } +}; -template +template struct Benchmark; template -struct Benchmark +struct min_kernel { - static void - run(benchmark::State& state, size_t desired_segments, const hipStream_t stream, size_t size) - { - hipError_t (*ptr_to_sum)(void*, size_t&, T*, T*, int, OffsetType*, OffsetType*, hipStream_t) - = &hipcub::DeviceSegmentedReduce::Sum; - run_benchmark(state, desired_segments, stream, size, ptr_to_sum); - } + static constexpr const char* name = "min"; + + static constexpr hipError_t (*kernel)( + void*, size_t&, T*, T*, _HIPCUB_STD::int64_t, OffsetType*, OffsetType*, hipStream_t) + = &hipcub::DeviceSegmentedReduce::Min; }; template -struct Benchmark +struct sum_kernel { - static void - run(benchmark::State& state, size_t desired_segments, const hipStream_t stream, size_t size) - { - hipError_t (*ptr_to_min)(void*, size_t&, T*, T*, int, OffsetType*, OffsetType*, hipStream_t) - = &hipcub::DeviceSegmentedReduce::Min; - run_benchmark(state, desired_segments, stream, size, ptr_to_min); - } + static constexpr const char* name = "sum"; + + static constexpr hipError_t (*kernel)( + void*, size_t&, T*, T*, _HIPCUB_STD::int64_t, OffsetType*, OffsetType*, hipStream_t) + = &hipcub::DeviceSegmentedReduce::Sum; }; template -struct Benchmark +struct argmin_kernel { using Difference = OffsetType; using Iterator = typename hipcub::ArgIndexInputIterator; using KeyValue = typename Iterator::value_type; - static void - run(benchmark::State& state, size_t desired_segments, const hipStream_t stream, size_t size) - { - hipError_t (*ptr_to_argmin)(void*, - size_t&, - T*, - KeyValue*, - int, - OffsetType*, - OffsetType*, - hipStream_t) - = &hipcub::DeviceSegmentedReduce::ArgMin; - run_benchmark(state, desired_segments, stream, size, ptr_to_argmin); - } + static constexpr const char* name = "argmin"; + + static constexpr hipError_t (*kernel)( + void*, size_t&, T*, KeyValue*, _HIPCUB_STD::int64_t, OffsetType*, OffsetType*, hipStream_t) + = &hipcub::DeviceSegmentedReduce::ArgMin; +}; + +template +struct Benchmark +{ + using type = reduce_benchmark, DesiredSegments>; +}; + +template +struct Benchmark +{ + using type = reduce_benchmark, DesiredSegments>; }; -#define CREATE_BENCHMARK(T, SEGMENTS, REDUCE_OP) \ - benchmark::RegisterBenchmark(std::string("device_segmented_reduce" \ - "." \ - "(number_of_segments:~" \ - + std::to_string(SEGMENTS) + " segments)") \ - .c_str(), \ - &Benchmark::run, \ - SEGMENTS, \ - stream, \ - size) - -#define BENCHMARK_TYPE(type, REDUCE_OP) \ - CREATE_BENCHMARK(type, 1, REDUCE_OP), CREATE_BENCHMARK(type, 100, REDUCE_OP), \ - CREATE_BENCHMARK(type, 10000, REDUCE_OP) - -#define CREATE_BENCHMARKS(REDUCE_OP) \ - BENCHMARK_TYPE(float, REDUCE_OP), BENCHMARK_TYPE(double, REDUCE_OP), \ - BENCHMARK_TYPE(int8_t, REDUCE_OP), BENCHMARK_TYPE(int, REDUCE_OP) - -void add_benchmarks(std::vector& benchmarks, - hipStream_t stream, - size_t size) +template +struct Benchmark { - using custom_double2 = benchmark_utils::custom_type; + using type = reduce_benchmark::KeyValue, + argmin_kernel, + DesiredSegments>; +}; + +#define CREATE_BENCHMARK(T, SEGMENTS, REDUCE_OP) \ + executor.queue::type>() + +#define BENCHMARK_TYPE(type, REDUCE_OP) \ + CREATE_BENCHMARK(type, 1, REDUCE_OP); \ + CREATE_BENCHMARK(type, 100, REDUCE_OP); \ + CREATE_BENCHMARK(type, 10000, REDUCE_OP) - std::vector bs = { - CREATE_BENCHMARKS(hipcub::Sum), - BENCHMARK_TYPE(custom_double2, hipcub::Sum), - CREATE_BENCHMARKS(hipcub::Min), +#define CREATE_BENCHMARKS(REDUCE_OP) \ + BENCHMARK_TYPE(float, REDUCE_OP); \ + BENCHMARK_TYPE(double, REDUCE_OP); \ + BENCHMARK_TYPE(int8_t, REDUCE_OP); \ + BENCHMARK_TYPE(int, REDUCE_OP) + +void add_benchmarks(primbench::executor& executor) +{ + CREATE_BENCHMARKS(benchmark_utils::plus); + BENCHMARK_TYPE(custom_double2, benchmark_utils::plus); + CREATE_BENCHMARKS(benchmark_utils::minimum); #ifdef HIPCUB_ROCPRIM_API - BENCHMARK_TYPE(custom_double2, hipcub::Min), + BENCHMARK_TYPE(custom_double2, benchmark_utils::minimum); #endif - CREATE_BENCHMARKS(hipcub::ArgMin), + CREATE_BENCHMARKS(hipcub::ArgMin); #ifdef HIPCUB_ROCPRIM_API - BENCHMARK_TYPE(custom_double2, hipcub::ArgMin), + BENCHMARK_TYPE(custom_double2, hipcub::ArgMin); #endif - }; - - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_device_segmented_reduce" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - add_benchmarks(benchmarks, stream, size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + primbench::executor executor(argc, argv, settings); + + add_benchmarks(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_segmented_sort.cpp b/projects/hipcub/benchmark/benchmark_device_segmented_sort.cpp index db69075e7cef..b33fc1526040 100644 --- a/projects/hipcub/benchmark/benchmark_device_segmented_sort.cpp +++ b/projects/hipcub/benchmark/benchmark_device_segmented_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,468 +20,313 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif - -const unsigned int batch_size = 4; -const unsigned int warmup_size = 2; - -template -void run_sort_keys_benchmark(benchmark::State& state, - size_t desired_segments, - hipStream_t stream, - size_t size, - bool Descending = false, - bool Stable = false) +template +class sort_keys_benchmark : public primbench::benchmark_interface { - using offset_type = int; - using key_type = Key; - using sort_func = hipError_t (*)(void*, - size_t&, - const key_type*, - key_type*, - int, - int, - offset_type*, - offset_type*, - hipStream_t); - - sort_func func_ascending = &hipcub::DeviceSegmentedSort::SortKeys; - sort_func func_descending - = &hipcub::DeviceSegmentedSort::SortKeysDescending; - sort_func func_ascending_stable - = &hipcub::DeviceSegmentedSort::StableSortKeys; - sort_func func_descending_stable - = &hipcub::DeviceSegmentedSort::StableSortKeysDescending; - - sort_func sorting = Descending ? (Stable ? func_descending_stable : func_descending) - : (Stable ? func_ascending_stable : func_ascending); - - std::vector offsets; - - const double avg_segment_length = static_cast(size) / desired_segments; - - std::random_device rd; - std::default_random_engine gen(rd()); - - std::uniform_real_distribution segment_length_dis(0, avg_segment_length * 2); - - unsigned int segments_count = 0; - size_t offset = 0; - while(offset < size) + primbench::json meta() const override { - const size_t segment_length = std::round(segment_length_dis(gen)); - offsets.push_back(offset); - ++segments_count; - offset += segment_length; + return primbench::json{} + .add("algo", "device_segmented_sort") + .add("subalgo", "sort_keys") + .add("lvl", "device") + .add("key_data_type", primbench::name()) + .add("desired_segments", DesiredSegments) + .add("ascending", !Descending) + .add("stable", Stable); } - offsets.push_back(size); - - std::vector keys_input = benchmark_utils::get_random_data( - size, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); - - offset_type* d_offsets; - HIP_CHECK(hipMalloc(&d_offsets, (segments_count + 1) * sizeof(offset_type))); - HIP_CHECK(hipMemcpy(d_offsets, - offsets.data(), - (segments_count + 1) * sizeof(offset_type), - hipMemcpyHostToDevice)); - - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(hipMalloc(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK(hipMalloc(&d_keys_output, size * sizeof(key_type))); - HIP_CHECK( - hipMemcpy(d_keys_input, keys_input.data(), size * sizeof(key_type), hipMemcpyHostToDevice)); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - HIP_CHECK(sorting(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - size, - segments_count, - d_offsets, - d_offsets + 1, - stream)); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; ++i) - { - HIP_CHECK(sorting(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - size, - segments_count, - d_offsets, - d_offsets + 1, - stream)); - } - HIP_CHECK(hipDeviceSynchronize()); - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); - - for(size_t i = 0; i < batch_size; ++i) + const size_t items = state.size; + const auto& stream = state.stream; + + using offset_type = int; + using sort_func = hipError_t (*)(void*, + size_t&, + const Key*, + Key*, + int64_t, + int64_t, + offset_type*, + offset_type*, + hipStream_t); + + sort_func func_ascending = &hipcub::DeviceSegmentedSort::SortKeys; + sort_func func_descending + = &hipcub::DeviceSegmentedSort::SortKeysDescending; + sort_func func_ascending_stable + = &hipcub::DeviceSegmentedSort::StableSortKeys; + sort_func func_descending_stable + = &hipcub::DeviceSegmentedSort::StableSortKeysDescending; + + sort_func sorting = Descending ? (Stable ? func_descending_stable : func_descending) + : (Stable ? func_ascending_stable : func_ascending); + + std::vector offsets; + + const double avg_segment_length = static_cast(items) / DesiredSegments; + + std::random_device rd; + std::default_random_engine gen(rd()); + + std::uniform_real_distribution segment_length_dis(0, avg_segment_length * 2); + + unsigned int segments_count = 0; + size_t offset = 0; + while(offset < items) { - HIP_CHECK(sorting(d_temporary_storage, - temporary_storage_bytes, + const size_t segment_length = std::round(segment_length_dis(gen)); + offsets.push_back(offset); + ++segments_count; + offset += segment_length; + } + offsets.push_back(items); + + std::vector keys_input + = benchmark_utils::get_random_data(items, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); + + offset_type* d_offsets; + HIP_CHECK(hipMalloc(&d_offsets, (segments_count + 1) * sizeof(offset_type))); + HIP_CHECK(hipMemcpy(d_offsets, + offsets.data(), + (segments_count + 1) * sizeof(offset_type), + hipMemcpyHostToDevice)); + + Key* d_keys_input; + Key* d_keys_output; + HIP_CHECK(hipMalloc(&d_keys_input, items * sizeof(Key))); + HIP_CHECK(hipMalloc(&d_keys_output, items * sizeof(Key))); + HIP_CHECK( + hipMemcpy(d_keys_input, keys_input.data(), items * sizeof(Key), hipMemcpyHostToDevice)); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] + { + HIP_CHECK(sorting(d_temp_storage, + temp_storage_bytes, d_keys_input, d_keys_output, - size, + items, segments_count, d_offsets, d_offsets + 1, stream)); - } + }; + + launch(); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(key_type)); - state.SetItemsProcessed(state.iterations() * batch_size * size); + state.set_items(items); + state.add_writes(items); - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_offsets)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_keys_output)); -} + state.run(launch); -template -void run_sort_pairs_benchmark(benchmark::State& state, - size_t desired_segments, - hipStream_t stream, - size_t size, - bool Descending = false, - bool Stable = false) -{ - using offset_type = int; - using key_type = Key; - using value_type = Value; - using sort_func = hipError_t (*)(void*, - size_t&, - const key_type*, - key_type*, - const value_type*, - value_type*, - int, - int, - offset_type*, - offset_type*, - hipStream_t); - - sort_func func_ascending - = &hipcub::DeviceSegmentedSort::SortPairs; - sort_func func_descending - = &hipcub::DeviceSegmentedSort::SortPairsDescending; - sort_func func_ascending_stable - = &hipcub::DeviceSegmentedSort::StableSortPairs; - sort_func func_descending_stable - = &hipcub::DeviceSegmentedSort:: - StableSortPairsDescending; - - sort_func sorting = Descending ? (Stable ? func_descending_stable : func_descending) - : (Stable ? func_ascending_stable : func_ascending); - - std::vector offsets; - - const double avg_segment_length = static_cast(size) / desired_segments; - - std::random_device rd; - std::default_random_engine gen(rd()); - - std::uniform_real_distribution segment_length_dis(0, avg_segment_length * 2); - - unsigned int segments_count = 0; - size_t offset = 0; - while(offset < size) - { - const size_t segment_length = std::round(segment_length_dis(gen)); - offsets.push_back(offset); - ++segments_count; - offset += segment_length; + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_offsets)); + HIP_CHECK(hipFree(d_keys_input)); + HIP_CHECK(hipFree(d_keys_output)); } - offsets.push_back(size); - - std::vector keys_input = benchmark_utils::get_random_data( - size, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); - - std::vector values_input(size); - std::iota(values_input.begin(), values_input.end(), 0); - - offset_type* d_offsets; - HIP_CHECK(hipMalloc(&d_offsets, (segments_count + 1) * sizeof(offset_type))); - HIP_CHECK(hipMemcpy(d_offsets, - offsets.data(), - (segments_count + 1) * sizeof(offset_type), - hipMemcpyHostToDevice)); - - key_type* d_keys_input; - key_type* d_keys_output; - HIP_CHECK(hipMalloc(&d_keys_input, size * sizeof(key_type))); - HIP_CHECK(hipMalloc(&d_keys_output, size * sizeof(key_type))); - HIP_CHECK( - hipMemcpy(d_keys_input, keys_input.data(), size * sizeof(key_type), hipMemcpyHostToDevice)); - - value_type* d_values_input; - value_type* d_values_output; - HIP_CHECK(hipMalloc(&d_values_input, size * sizeof(value_type))); - HIP_CHECK(hipMalloc(&d_values_output, size * sizeof(value_type))); - HIP_CHECK(hipMemcpy(d_values_input, - values_input.data(), - size * sizeof(value_type), - hipMemcpyHostToDevice)); - - void* d_temporary_storage = nullptr; - size_t temporary_storage_bytes = 0; - HIP_CHECK(sorting(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, - size, - segments_count, - d_offsets, - d_offsets + 1, - stream)); - - HIP_CHECK(hipMalloc(&d_temporary_storage, temporary_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) +}; + +template +class sort_pairs_benchmark : public primbench::benchmark_interface +{ + primbench::json meta() const override { - HIP_CHECK(sorting(d_temporary_storage, - temporary_storage_bytes, - d_keys_input, - d_keys_output, - d_values_input, - d_values_output, - size, - segments_count, - d_offsets, - d_offsets + 1, - stream)); + return primbench::json{} + .add("algo", "device_segmented_sort") + .add("subalgo", "sort_pairs") + .add("lvl", "device") + .add("key_data_type", primbench::name()) + .add("value_data_type", primbench::name()) + .add("desired_segments", DesiredSegments) + .add("ascending", !Descending) + .add("stable", Stable); } - HIP_CHECK(hipDeviceSynchronize()); - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); - - for(size_t i = 0; i < batch_size; i++) + const size_t items = state.size; + const auto& stream = state.stream; + + using offset_type = int; + using sort_func = hipError_t (*)(void*, + size_t&, + const Key*, + Key*, + const Value*, + Value*, + int64_t, + int64_t, + offset_type*, + offset_type*, + hipStream_t); + + sort_func func_ascending + = &hipcub::DeviceSegmentedSort::SortPairs; + sort_func func_descending + = &hipcub::DeviceSegmentedSort::SortPairsDescending; + sort_func func_ascending_stable + = &hipcub::DeviceSegmentedSort::StableSortPairs; + sort_func func_descending_stable + = &hipcub::DeviceSegmentedSort::StableSortPairsDescending; + + sort_func sorting = Descending ? (Stable ? func_descending_stable : func_descending) + : (Stable ? func_ascending_stable : func_ascending); + + std::vector offsets; + + const double avg_segment_length = static_cast(items) / DesiredSegments; + + std::random_device rd; + std::default_random_engine gen(rd()); + + std::uniform_real_distribution segment_length_dis(0, avg_segment_length * 2); + + unsigned int segments_count = 0; + size_t offset = 0; + while(offset < items) + { + const size_t segment_length = std::round(segment_length_dis(gen)); + offsets.push_back(offset); + ++segments_count; + offset += segment_length; + } + offsets.push_back(items); + + std::vector keys_input + = benchmark_utils::get_random_data(items, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); + + std::vector values_input(items); + std::iota(values_input.begin(), values_input.end(), 0); + + offset_type* d_offsets; + HIP_CHECK(hipMalloc(&d_offsets, (segments_count + 1) * sizeof(offset_type))); + HIP_CHECK(hipMemcpy(d_offsets, + offsets.data(), + (segments_count + 1) * sizeof(offset_type), + hipMemcpyHostToDevice)); + + Key* d_keys_input; + Key* d_keys_output; + HIP_CHECK(hipMalloc(&d_keys_input, items * sizeof(Key))); + HIP_CHECK(hipMalloc(&d_keys_output, items * sizeof(Key))); + HIP_CHECK( + hipMemcpy(d_keys_input, keys_input.data(), items * sizeof(Key), hipMemcpyHostToDevice)); + + Value* d_values_input; + Value* d_values_output; + HIP_CHECK(hipMalloc(&d_values_input, items * sizeof(Value))); + HIP_CHECK(hipMalloc(&d_values_output, items * sizeof(Value))); + HIP_CHECK(hipMemcpy(d_values_input, + values_input.data(), + items * sizeof(Value), + hipMemcpyHostToDevice)); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { - HIP_CHECK(sorting(d_temporary_storage, - temporary_storage_bytes, + HIP_CHECK(sorting(d_temp_storage, + temp_storage_bytes, d_keys_input, d_keys_output, d_values_input, d_values_output, - size, + items, segments_count, d_offsets, d_offsets + 1, stream)); - } + }; + + launch(); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + state.set_items(items); + state.add_writes(items); + state.add_writes(items); + + state.run(launch); + + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipFree(d_offsets)); + HIP_CHECK(hipFree(d_keys_input)); + HIP_CHECK(hipFree(d_keys_output)); + HIP_CHECK(hipFree(d_values_input)); + HIP_CHECK(hipFree(d_values_output)); } - state.SetBytesProcessed(state.iterations() * batch_size * size - * (sizeof(key_type) + sizeof(value_type))); - state.SetItemsProcessed(state.iterations() * batch_size * size); - - HIP_CHECK(hipFree(d_temporary_storage)); - HIP_CHECK(hipFree(d_offsets)); - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_values_output)); -} +}; + +#define CREATE_SORT_KEYS_BENCHMARK(Key, SEGMENTS) \ + executor.queue>(); \ + executor.queue>(); \ + executor.queue>(); \ + executor.queue>(); + +#define BENCHMARK_KEY_TYPE(type) \ + CREATE_SORT_KEYS_BENCHMARK(type, 10); \ + CREATE_SORT_KEYS_BENCHMARK(type, 100); \ + CREATE_SORT_KEYS_BENCHMARK(type, 1000); \ + CREATE_SORT_KEYS_BENCHMARK(type, 10000) -#define CREATE_SORT_KEYS_BENCHMARK(Key, SEGMENTS) \ - benchmark::RegisterBenchmark(std::string("device_segmented_sort_keys" \ - "." \ - "(number_of_segments:~" \ - + std::to_string(SEGMENTS) + " segments)") \ - .c_str(), \ - [=](benchmark::State& state) { \ - run_sort_keys_benchmark(state, SEGMENTS, stream, size); \ - }), \ - benchmark::RegisterBenchmark( \ - std::string("device_segmented_sort_keys" \ - "." \ - "(number_of_segments:~" \ - + std::to_string(SEGMENTS) + " segments)") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_sort_keys_benchmark(state, SEGMENTS, stream, size, true); }), \ - benchmark::RegisterBenchmark( \ - std::string("device_segmented_sort_keys" \ - "." \ - "(number_of_segments:~" \ - + std::to_string(SEGMENTS) + " segments)") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_sort_keys_benchmark(state, SEGMENTS, stream, size, false, true); }), \ - benchmark::RegisterBenchmark( \ - std::string("device_segmented_sort_keys" \ - "." \ - "(number_of_segments:~" \ - + std::to_string(SEGMENTS) + " segments)") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_sort_keys_benchmark(state, SEGMENTS, stream, size, true, true); }) - -#define BENCHMARK_KEY_TYPE(type) \ - CREATE_SORT_KEYS_BENCHMARK(type, 10), CREATE_SORT_KEYS_BENCHMARK(type, 100), \ - CREATE_SORT_KEYS_BENCHMARK(type, 1000), CREATE_SORT_KEYS_BENCHMARK(type, 10000) - -void add_sort_keys_benchmarks(std::vector& benchmarks, - hipStream_t stream, - size_t size) +void add_sort_keys_benchmarks(primbench::executor& executor) { - std::vector bs = { - BENCHMARK_KEY_TYPE(float), - BENCHMARK_KEY_TYPE(double), - BENCHMARK_KEY_TYPE(int8_t), - BENCHMARK_KEY_TYPE(uint8_t), - BENCHMARK_KEY_TYPE(int), - }; - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + BENCHMARK_KEY_TYPE(float); + BENCHMARK_KEY_TYPE(double); + BENCHMARK_KEY_TYPE(int8_t); + BENCHMARK_KEY_TYPE(uint8_t); + BENCHMARK_KEY_TYPE(int); } -#define CREATE_SORT_PAIRS_BENCHMARK(Key, Value, SEGMENTS) \ - benchmark::RegisterBenchmark( \ - std::string("device_segmented_sort_pairs" \ - "." \ - "(number_of_segments:~" \ - + std::to_string(SEGMENTS) + " segments)") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_sort_pairs_benchmark(state, SEGMENTS, stream, size); }), \ - benchmark::RegisterBenchmark( \ - std::string("device_segmented_sort_pairs" \ - "." \ - "(number_of_segments:~" \ - + std::to_string(SEGMENTS) + " segments)") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_sort_pairs_benchmark(state, SEGMENTS, stream, size, true); }), \ - benchmark::RegisterBenchmark( \ - std::string("device_segmented_sort_pairs" \ - "." \ - "(number_of_segments:~" \ - + std::to_string(SEGMENTS) + " segments)") \ - .c_str(), \ - [=](benchmark::State& state) { \ - run_sort_pairs_benchmark(state, SEGMENTS, stream, size, false, true); \ - }), \ - benchmark::RegisterBenchmark( \ - std::string("device_segmented_sort_pairs" \ - "." \ - "(number_of_segments:~" \ - + std::to_string(SEGMENTS) + " segments)") \ - .c_str(), \ - [=](benchmark::State& state) \ - { run_sort_pairs_benchmark(state, SEGMENTS, stream, size, true, true); }) -#define BENCHMARK_PAIR_TYPE(type, value) \ - CREATE_SORT_PAIRS_BENCHMARK(type, value, 10), CREATE_SORT_PAIRS_BENCHMARK(type, value, 100), \ - CREATE_SORT_PAIRS_BENCHMARK(type, value, 10000) - -void add_sort_pairs_benchmarks(std::vector& benchmarks, - hipStream_t stream, - size_t size) +#define CREATE_SORT_PAIRS_BENCHMARK(Key, Value, SEGMENTS) \ + executor.queue>(); \ + executor.queue>(); \ + executor.queue>(); \ + executor.queue>() + +#define BENCHMARK_PAIR_TYPE(type, value) \ + CREATE_SORT_PAIRS_BENCHMARK(type, value, 10); \ + CREATE_SORT_PAIRS_BENCHMARK(type, value, 100); \ + CREATE_SORT_PAIRS_BENCHMARK(type, value, 10000) + +void add_sort_pairs_benchmarks(primbench::executor& executor) { - using custom_float2 = benchmark_utils::custom_type; - using custom_double2 = benchmark_utils::custom_type; - - std::vector bs = { - BENCHMARK_PAIR_TYPE(int, float), - BENCHMARK_PAIR_TYPE(long long, double), - BENCHMARK_PAIR_TYPE(int8_t, int8_t), - BENCHMARK_PAIR_TYPE(uint8_t, uint8_t), - BENCHMARK_PAIR_TYPE(int, custom_float2), - BENCHMARK_PAIR_TYPE(long long, custom_double2), - }; - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); + BENCHMARK_PAIR_TYPE(int, float); + BENCHMARK_PAIR_TYPE(int64_t, double); + BENCHMARK_PAIR_TYPE(int8_t, int8_t); + BENCHMARK_PAIR_TYPE(uint8_t, uint8_t); + BENCHMARK_PAIR_TYPE(int, custom_float2); + BENCHMARK_PAIR_TYPE(int64_t, custom_double2); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_device_segmented_sort" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - add_sort_keys_benchmarks(benchmarks, stream, size); - add_sort_pairs_benchmarks(benchmarks, stream, size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 1000; + settings.batch_window_size = 3; + settings.noise_tolerance_percent = 3; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + primbench::executor executor(argc, argv, settings, primbench::flags::sync); + + add_sort_keys_benchmarks(executor); + add_sort_pairs_benchmarks(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_select.cpp b/projects/hipcub/benchmark/benchmark_device_select.cpp index 04097eca9be6..59484cb5a23d 100644 --- a/projects/hipcub/benchmark/benchmark_device_select.cpp +++ b/projects/hipcub/benchmark/benchmark_device_select.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,106 +20,95 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif - template -void run_flagged_benchmark(benchmark::State& state, - size_t size, - const hipStream_t stream, - float true_probability) +class flagged_benchmark : public primbench::benchmark_interface { - std::vector input - = benchmark_utils::get_random_data(size, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); - - std::vector flags - = benchmark_utils::get_random_data01(size, true_probability); - - T* d_input; - FlagType* d_flags; - T* d_output; - unsigned int* d_selected_count_output; - HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_flags, flags.size() * sizeof(FlagType))); - HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_selected_count_output, sizeof(unsigned int))); - HIP_CHECK(hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK( - hipMemcpy(d_flags, flags.data(), flags.size() * sizeof(FlagType), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - // Allocate temporary storage memory - size_t temp_storage_size_bytes = 0; - - // Get size of d_temp_storage - HIP_CHECK(hipcub::DeviceSelect::Flagged(nullptr, - temp_storage_size_bytes, - d_input, - d_flags, - d_output, - d_selected_count_output, - input.size(), - stream)); - HIP_CHECK(hipDeviceSynchronize()); - - // allocate temporary storage - void* d_temp_storage = nullptr; - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < 10; i++) +public: + flagged_benchmark(float true_probability) : m_true_probability(true_probability) {} + +private: + primbench::json meta() const override { - HIP_CHECK(hipcub::DeviceSelect::Flagged(d_temp_storage, - temp_storage_size_bytes, - d_input, - d_flags, - d_output, - d_selected_count_output, - input.size(), - stream)); + return primbench::json{} + .add("algo", "device_select") + .add("subalgo", "flagged") + .add("lvl", "device") + .add("data_type", primbench::name()) + .add("output_data_type", primbench::name()) + .add("flag_type", primbench::name()) + .add("selected_output_data_type", "u32") + .add("probability", m_true_probability); } - HIP_CHECK(hipDeviceSynchronize()); - const unsigned int batch_size = 10; - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); - for(size_t i = 0; i < batch_size; i++) + const size_t items = state.size; + const auto& stream = state.stream; + + std::vector input + = benchmark_utils::get_random_data(items, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); + + std::vector flags + = benchmark_utils::get_random_data01(items, m_true_probability); + + T* d_input; + FlagType* d_flags; + T* d_output; + unsigned int* d_selected_count_output; + HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_flags, flags.size() * sizeof(FlagType))); + HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_selected_count_output, sizeof(unsigned int))); + HIP_CHECK( + hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(d_flags, + flags.data(), + flags.size() * sizeof(FlagType), + hipMemcpyHostToDevice)); + HIP_CHECK(hipDeviceSynchronize()); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { HIP_CHECK(hipcub::DeviceSelect::Flagged(d_temp_storage, - temp_storage_size_bytes, + temp_storage_bytes, d_input, d_flags, d_output, d_selected_count_output, input.size(), stream)); - } + }; + + launch(); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); + + state.set_items(items); + state.add_writes(items); + + state.run(launch); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_flags)); + HIP_CHECK(hipFree(d_output)); + HIP_CHECK(hipFree(d_selected_count_output)); + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipDeviceSynchronize()); } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * batch_size * size); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_flags)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); - HIP_CHECK(hipDeviceSynchronize()); -} + + float m_true_probability; +}; template struct SelectOperator @@ -135,160 +124,136 @@ struct SelectOperator }; template -void run_selectop_benchmark(benchmark::State& state, - size_t size, - const hipStream_t stream, - float true_probability) +class selectop_benchmark : public primbench::benchmark_interface { - std::vector input = benchmark_utils::get_random_data(size, T(0), T(1000)); - - SelectOperator select_op(true_probability); - - T* d_input; - T* d_output; - unsigned int* d_selected_count_output; - HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_selected_count_output, sizeof(unsigned int))); - HIP_CHECK(hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - // Allocate temporary storage memory - size_t temp_storage_size_bytes; - - // Get size of d_temp_storage - HIP_CHECK(hipcub::DeviceSelect::If(nullptr, - temp_storage_size_bytes, - d_input, - d_output, - d_selected_count_output, - input.size(), - select_op, - stream)); - HIP_CHECK(hipDeviceSynchronize()); - - // allocate temporary storage - void* d_temp_storage = nullptr; - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < 10; i++) +public: + selectop_benchmark(float true_probability) : m_true_probability(true_probability) {} + + primbench::json meta() const override { - HIP_CHECK(hipcub::DeviceSelect::If(d_temp_storage, - temp_storage_size_bytes, - d_input, - d_output, - d_selected_count_output, - input.size(), - select_op, - stream)); + return primbench::json{} + .add("algo", "device_select") + .add("subalgo", "if") + .add("lvl", "device") + .add("data_type", primbench::name()) + .add("output_data_type", primbench::name()) + .add("selected_output_data_type", "u32") + .add("probability", m_true_probability); } - HIP_CHECK(hipDeviceSynchronize()); - const unsigned int batch_size = 10; - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); - for(size_t i = 0; i < batch_size; i++) + const size_t items = state.size; + const auto& stream = state.stream; + + std::vector input = benchmark_utils::get_random_data(items, T(0), T(1000)); + + SelectOperator select_op(m_true_probability); + + T* d_input; + T* d_output; + unsigned int* d_selected_count_output; + HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_selected_count_output, sizeof(unsigned int))); + HIP_CHECK( + hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK(hipDeviceSynchronize()); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { HIP_CHECK(hipcub::DeviceSelect::If(d_temp_storage, - temp_storage_size_bytes, + temp_storage_bytes, d_input, d_output, d_selected_count_output, input.size(), select_op, stream)); - } + }; + + launch(); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); + + state.set_items(items); + state.add_writes(items); + + state.run(launch); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); + HIP_CHECK(hipFree(d_selected_count_output)); + HIP_CHECK(hipFree(d_temp_storage)); + HIP_CHECK(hipDeviceSynchronize()); } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * batch_size * size); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); - HIP_CHECK(hipDeviceSynchronize()); -} + +private: + float m_true_probability; +}; template -void run_flagged_if_benchmark(benchmark::State& state, - size_t size, - const hipStream_t stream, - float true_probability) +class flagged_if_benchmark : public primbench::benchmark_interface { - std::vector input - = benchmark_utils::get_random_data(size, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); - - std::vector flags - = benchmark_utils::get_random_data01(size, true_probability); - - SelectOperator select_flag_op(true_probability); - - T* d_input; - FlagType* d_flags; - T* d_output; - unsigned int* d_selected_count_output; - HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_flags, flags.size() * sizeof(FlagType))); - HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_selected_count_output, sizeof(unsigned int))); - HIP_CHECK(hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK( - hipMemcpy(d_flags, flags.data(), flags.size() * sizeof(FlagType), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - // Allocate temporary storage memory - size_t temp_storage_size_bytes = 0; - - // Get size of d_temp_storage - HIP_CHECK(hipcub::DeviceSelect::FlaggedIf(nullptr, - temp_storage_size_bytes, - d_input, - d_flags, - d_output, - d_selected_count_output, - input.size(), - select_flag_op, - stream)); - HIP_CHECK(hipDeviceSynchronize()); - - // allocate temporary storage - void* d_temp_storage = nullptr; - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < 10; i++) +public: + flagged_if_benchmark(float true_probability) : m_true_probability(true_probability) {} + + primbench::json meta() const override { - HIP_CHECK(hipcub::DeviceSelect::FlaggedIf(d_temp_storage, - temp_storage_size_bytes, - d_input, - d_flags, - d_output, - d_selected_count_output, - input.size(), - select_flag_op, - stream)); + return primbench::json{} + .add("algo", "device_select") + .add("subalgo", "flagged_if") + .add("lvl", "device") + .add("data_type", primbench::name()) + .add("flag_type", primbench::name()) + .add("output_data_type", primbench::name()) + .add("selected_output_data_type", "u32") + .add("probability", m_true_probability); } - HIP_CHECK(hipDeviceSynchronize()); - const unsigned int batch_size = 10; - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); - for(size_t i = 0; i < batch_size; i++) + const size_t items = state.size; + const auto& stream = state.stream; + + std::vector input + = benchmark_utils::get_random_data(items, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); + + std::vector flags + = benchmark_utils::get_random_data01(items, m_true_probability); + + SelectOperator select_flag_op(m_true_probability); + + T* d_input; + FlagType* d_flags; + T* d_output; + unsigned int* d_selected_count_output; + HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_flags, flags.size() * sizeof(FlagType))); + HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_selected_count_output, sizeof(unsigned int))); + HIP_CHECK( + hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(d_flags, + flags.data(), + flags.size() * sizeof(FlagType), + hipMemcpyHostToDevice)); + HIP_CHECK(hipDeviceSynchronize()); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { HIP_CHECK(hipcub::DeviceSelect::FlaggedIf(d_temp_storage, - temp_storage_size_bytes, + temp_storage_bytes, d_input, d_flags, d_output, @@ -296,201 +261,186 @@ void run_flagged_if_benchmark(benchmark::State& state, input.size(), select_flag_op, stream)); - } + }; + + launch(); + HIP_CHECK(hipDeviceSynchronize()); + + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + state.set_items(items); + state.add_writes(items); + + state.run(launch); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_flags)); + HIP_CHECK(hipFree(d_output)); + HIP_CHECK(hipFree(d_selected_count_output)); + HIP_CHECK(hipFree(d_temp_storage)); } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * batch_size * size); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_flags)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); -} + +private: + float m_true_probability; +}; template -void run_unique_benchmark(benchmark::State& state, - size_t size, - const hipStream_t stream, - float discontinuity_probability) +class unique_benchmark : public primbench::benchmark_interface { - hipcub::Sum op; +public: + unique_benchmark(float discontinuity_probability) + : m_discontinuity_probability(discontinuity_probability) + {} - std::vector input(size); +private: + primbench::json meta() const override { - auto input01 = benchmark_utils::get_random_data01(size, discontinuity_probability); - auto acc = input01[0]; - input[0] = acc; - for(size_t i = 1; i < input01.size(); i++) - { - input[i] = op(acc, input01[i]); - } + return primbench::json{} + .add("algo", "device_select") + .add("subalgo", "unique") + .add("lvl", "device") + .add("data_type", primbench::name()) + .add("output_data_type", primbench::name()) + .add("selected_output_data_type", "u32") + .add("probability", m_discontinuity_probability); } - T* d_input; - T* d_output; - unsigned int* d_selected_count_output; - HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_selected_count_output, sizeof(unsigned int))); - HIP_CHECK(hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - // Allocate temporary storage memory - size_t temp_storage_size_bytes; - - // Get size of d_temp_storage - HIP_CHECK(hipcub::DeviceSelect::Unique(nullptr, - temp_storage_size_bytes, - d_input, - d_output, - d_selected_count_output, - input.size(), - stream)); - HIP_CHECK(hipDeviceSynchronize()); - - // allocate temporary storage - void* d_temp_storage = nullptr; - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < 10; i++) + void run(primbench::state& state) override { - HIP_CHECK(hipcub::DeviceSelect::Unique(d_temp_storage, - temp_storage_size_bytes, - d_input, - d_output, - d_selected_count_output, - input.size(), - stream)); - } - HIP_CHECK(hipDeviceSynchronize()); + const size_t items = state.size; + const auto& stream = state.stream; - const unsigned int batch_size = 10; - for(auto _ : state) - { - auto start = std::chrono::high_resolution_clock::now(); - for(size_t i = 0; i < batch_size; i++) + benchmark_utils::plus op{}; + + std::vector input(items); + { + auto input01 + = benchmark_utils::get_random_data01(items, m_discontinuity_probability); + auto acc = input01[0]; + input[0] = acc; + for(size_t i = 1; i < input01.size(); i++) + { + input[i] = op(acc, input01[i]); + } + } + + T* d_input; + T* d_output; + unsigned int* d_selected_count_output; + HIP_CHECK(hipMalloc(&d_input, input.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_output, input.size() * sizeof(T))); + HIP_CHECK(hipMalloc(&d_selected_count_output, sizeof(unsigned int))); + HIP_CHECK( + hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK(hipDeviceSynchronize()); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { HIP_CHECK(hipcub::DeviceSelect::Unique(d_temp_storage, - temp_storage_size_bytes, + temp_storage_bytes, d_input, d_output, d_selected_count_output, input.size(), stream)); - } + }; + + launch(); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + + state.set_items(items); + state.add_writes(items); + + state.run(launch); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); + HIP_CHECK(hipFree(d_selected_count_output)); + HIP_CHECK(hipFree(d_temp_storage)); } - state.SetBytesProcessed(state.iterations() * batch_size * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * batch_size * size); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); -} + float m_discontinuity_probability; +}; -template -void run_unique_by_key_benchmark(benchmark::State& state, - size_t size, - const hipStream_t stream, - float discontinuity_probability) +template +class unique_by_key_benchmark : public primbench::benchmark_interface { - hipcub::Sum op; +public: + unique_by_key_benchmark(float discontinuity_probability) + : m_discontinuity_probability(discontinuity_probability) + {} - std::vector input_keys(size); +private: + primbench::json meta() const override { - auto input01 = benchmark_utils::get_random_data01(size, discontinuity_probability); - auto acc = input01[0]; + return primbench::json{} + .add("algo", "device_select") + .add("subalgo", "unique_by_key") + .add("lvl", "device") + .add("key_data_type", primbench::name()) + .add("value_data_type", primbench::name()) + .add("selected_output_data_type", "u32") + .add("probability", m_discontinuity_probability); + } - input_keys[0] = acc; + void run(primbench::state& state) override + { + const size_t items = state.size; + const auto& stream = state.stream; - for(size_t i = 1; i < input01.size(); i++) + benchmark_utils::plus op{}; + + std::vector input_keys(items); { - input_keys[i] = op(acc, input01[i]); - } - } + auto input01 + = benchmark_utils::get_random_data01(items, m_discontinuity_probability); + auto acc = input01[0]; - const auto input_values - = benchmark_utils::get_random_data(size, ValueT(-1000), ValueT(1000)); - - KeyT* d_keys_input; - ValueT* d_values_input; - KeyT* d_keys_output; - ValueT* d_values_output; - unsigned int* d_selected_count_output; - - HIP_CHECK(hipMalloc(&d_keys_input, input_keys.size() * sizeof(input_keys[0]))); - HIP_CHECK(hipMalloc(&d_values_input, input_values.size() * sizeof(input_values[0]))); - HIP_CHECK(hipMalloc(&d_keys_output, input_keys.size() * sizeof(input_keys[0]))); - HIP_CHECK(hipMalloc(&d_values_output, input_values.size() * sizeof(input_values[0]))); - HIP_CHECK(hipMalloc(&d_selected_count_output, sizeof(*d_selected_count_output))); - - HIP_CHECK(hipMemcpy(d_keys_input, - input_keys.data(), - input_keys.size() * sizeof(input_keys[0]), - hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_values_input, - input_values.data(), - input_values.size() * sizeof(input_values[0]), - hipMemcpyHostToDevice)); - - // Allocate temporary storage memory - size_t temp_storage_size_bytes; - - // Get size of d_temp_storage - HIP_CHECK(hipcub::DeviceSelect::UniqueByKey(nullptr, - temp_storage_size_bytes, - d_keys_input, - d_values_input, - d_keys_output, - d_values_output, - d_selected_count_output, - input_keys.size(), - stream)); - HIP_CHECK(hipDeviceSynchronize()); - - // allocate temporary storage - void* d_temp_storage = nullptr; - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < 10; i++) - { - HIP_CHECK(hipcub::DeviceSelect::UniqueByKey(d_temp_storage, - temp_storage_size_bytes, - d_keys_input, - d_values_input, - d_keys_output, - d_values_output, - d_selected_count_output, - input_keys.size(), - stream)); - } - HIP_CHECK(hipDeviceSynchronize()); + input_keys[0] = acc; - const unsigned int batch_size = 10; - for(auto _ : state) - { - auto start = std::chrono::high_resolution_clock::now(); - for(size_t i = 0; i < batch_size; i++) + for(size_t i = 1; i < input01.size(); i++) + { + input_keys[i] = op(acc, input01[i]); + } + } + + const auto input_values + = benchmark_utils::get_random_data(items, Value(-1000), Value(1000)); + + Key* d_keys_input; + Value* d_values_input; + Key* d_keys_output; + Value* d_values_output; + unsigned int* d_selected_count_output; + + HIP_CHECK(hipMalloc(&d_keys_input, input_keys.size() * sizeof(input_keys[0]))); + HIP_CHECK(hipMalloc(&d_values_input, input_values.size() * sizeof(input_values[0]))); + HIP_CHECK(hipMalloc(&d_keys_output, input_keys.size() * sizeof(input_keys[0]))); + HIP_CHECK(hipMalloc(&d_values_output, input_values.size() * sizeof(input_values[0]))); + HIP_CHECK(hipMalloc(&d_selected_count_output, sizeof(*d_selected_count_output))); + + HIP_CHECK(hipMemcpy(d_keys_input, + input_keys.data(), + input_keys.size() * sizeof(input_keys[0]), + hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(d_values_input, + input_values.data(), + input_values.size() * sizeof(input_values[0]), + hipMemcpyHostToDevice)); + + void* d_temp_storage = nullptr; + size_t temp_storage_bytes; + + const auto launch = [&] { HIP_CHECK(hipcub::DeviceSelect::UniqueByKey(d_temp_storage, - temp_storage_size_bytes, + temp_storage_bytes, d_keys_input, d_values_input, d_keys_output, @@ -498,183 +448,113 @@ void run_unique_by_key_benchmark(benchmark::State& state, d_selected_count_output, input_keys.size(), stream)); - } + }; + + launch(); HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * size - * (sizeof(KeyT) + sizeof(ValueT))); - state.SetItemsProcessed(state.iterations() * batch_size * size); - - HIP_CHECK(hipFree(d_keys_input)); - HIP_CHECK(hipFree(d_values_input)); - HIP_CHECK(hipFree(d_keys_output)); - HIP_CHECK(hipFree(d_values_output)); - HIP_CHECK(hipFree(d_selected_count_output)); - HIP_CHECK(hipFree(d_temp_storage)); -} + HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes)); + HIP_CHECK(hipDeviceSynchronize()); -#define CREATE_SELECT_FLAGGED_BENCHMARK(T, F, p) \ - benchmark::RegisterBenchmark( \ - std::string("device_select_flagged.(probability:" #p ")") \ - .c_str(), \ - &run_flagged_benchmark, \ - size, \ - stream, \ - p) - -#define CREATE_SELECT_IF_BENCHMARK(T, p) \ - benchmark::RegisterBenchmark( \ - std::string("device_select_if.(probability:" #p ")") \ - .c_str(), \ - &run_selectop_benchmark, \ - size, \ - stream, \ - p) - -#define CREATE_SELECT_FLAGGED_IF_BENCHMARK(T, F, p) \ - benchmark::RegisterBenchmark( \ - std::string("device_select_flagged_if.(probability:" #p ")") \ - .c_str(), \ - &run_flagged_if_benchmark, \ - size, \ - stream, \ - p) - -#define CREATE_UNIQUE_BENCHMARK(T, p) \ - benchmark::RegisterBenchmark( \ - std::string("device_select_unique.(probability:" #p ")") \ - .c_str(), \ - &run_unique_benchmark, \ - size, \ - stream, \ - p) - -#define CREATE_UNIQUE_BY_KEY_BENCHMARK(K, V, p) \ - benchmark::RegisterBenchmark( \ - std::string("device_select_unique_by_key.(probability:" #p ")") \ - .c_str(), \ - &run_unique_by_key_benchmark, \ - size, \ - stream, \ - p) - -#define BENCHMARK_FLAGGED_TYPE(type, value) \ - CREATE_SELECT_FLAGGED_BENCHMARK(type, value, 0.05f), \ - CREATE_SELECT_FLAGGED_BENCHMARK(type, value, 0.25f), \ - CREATE_SELECT_FLAGGED_BENCHMARK(type, value, 0.5f), \ - CREATE_SELECT_FLAGGED_BENCHMARK(type, value, 0.75f) - -#define BENCHMARK_IF_TYPE(type) \ - CREATE_SELECT_IF_BENCHMARK(type, 0.05f), CREATE_SELECT_IF_BENCHMARK(type, 0.25f), \ - CREATE_SELECT_IF_BENCHMARK(type, 0.5f), CREATE_SELECT_IF_BENCHMARK(type, 0.75f) - -#define BENCHMARK_FLAGGED_IF_TYPE(type, value) \ - CREATE_SELECT_FLAGGED_IF_BENCHMARK(type, value, 0.05f), \ - CREATE_SELECT_FLAGGED_IF_BENCHMARK(type, value, 0.25f), \ - CREATE_SELECT_FLAGGED_IF_BENCHMARK(type, value, 0.5f), \ - CREATE_SELECT_FLAGGED_IF_BENCHMARK(type, value, 0.75f) - -#define BENCHMARK_UNIQUE_TYPE(type) \ - CREATE_UNIQUE_BENCHMARK(type, 0.05f), CREATE_UNIQUE_BENCHMARK(type, 0.25f), \ - CREATE_UNIQUE_BENCHMARK(type, 0.5f), CREATE_UNIQUE_BENCHMARK(type, 0.75f) - -#define BENCHMARK_UNIQUE_BY_KEY_TYPE(key_type, value_type) \ - CREATE_UNIQUE_BY_KEY_BENCHMARK(key_type, value_type, 0.05f), \ - CREATE_UNIQUE_BY_KEY_BENCHMARK(key_type, value_type, 0.25f), \ - CREATE_UNIQUE_BY_KEY_BENCHMARK(key_type, value_type, 0.5f), \ - CREATE_UNIQUE_BY_KEY_BENCHMARK(key_type, value_type, 0.75f) + state.set_items(items); + state.add_writes(items); + state.add_writes(items); -int main(int argc, char* argv[]) -{ - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_device_select" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - using custom_double2 = benchmark_utils::custom_type; - using custom_int_double = benchmark_utils::custom_type; - - // Add benchmarks - std::vector benchmarks - = {BENCHMARK_FLAGGED_TYPE(int, unsigned char), - BENCHMARK_FLAGGED_TYPE(float, unsigned char), - BENCHMARK_FLAGGED_TYPE(double, unsigned char), - BENCHMARK_FLAGGED_TYPE(uint8_t, uint8_t), - BENCHMARK_FLAGGED_TYPE(int8_t, int8_t), - BENCHMARK_FLAGGED_TYPE(custom_double2, unsigned char), - - BENCHMARK_IF_TYPE(int), - BENCHMARK_IF_TYPE(float), - BENCHMARK_IF_TYPE(double), - BENCHMARK_IF_TYPE(uint8_t), - BENCHMARK_IF_TYPE(int8_t), - BENCHMARK_IF_TYPE(custom_int_double), - - BENCHMARK_FLAGGED_IF_TYPE(int, unsigned char), - BENCHMARK_FLAGGED_IF_TYPE(float, unsigned char), - BENCHMARK_FLAGGED_IF_TYPE(double, unsigned char), - BENCHMARK_FLAGGED_IF_TYPE(uint8_t, uint8_t), - BENCHMARK_FLAGGED_IF_TYPE(int8_t, int8_t), - BENCHMARK_FLAGGED_IF_TYPE(custom_double2, unsigned char), - - BENCHMARK_UNIQUE_TYPE(int), - BENCHMARK_UNIQUE_TYPE(float), - BENCHMARK_UNIQUE_TYPE(double), - BENCHMARK_UNIQUE_TYPE(uint8_t), - BENCHMARK_UNIQUE_TYPE(int8_t), - BENCHMARK_UNIQUE_TYPE(custom_int_double), - - BENCHMARK_UNIQUE_BY_KEY_TYPE(int, int), - BENCHMARK_UNIQUE_BY_KEY_TYPE(float, double), - BENCHMARK_UNIQUE_BY_KEY_TYPE(double, custom_double2), - BENCHMARK_UNIQUE_BY_KEY_TYPE(uint8_t, uint8_t), - BENCHMARK_UNIQUE_BY_KEY_TYPE(int8_t, double), - BENCHMARK_UNIQUE_BY_KEY_TYPE(custom_int_double, custom_int_double)}; - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + state.run(launch); - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } + HIP_CHECK(hipFree(d_keys_input)); + HIP_CHECK(hipFree(d_values_input)); + HIP_CHECK(hipFree(d_keys_output)); + HIP_CHECK(hipFree(d_values_output)); + HIP_CHECK(hipFree(d_selected_count_output)); + HIP_CHECK(hipFree(d_temp_storage)); } - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); + float m_discontinuity_probability; +}; + +#define CREATE_SELECT_FLAGGED_BENCHMARK(T, F, p) executor.queue>(p) + +#define CREATE_SELECT_IF_BENCHMARK(T, p) executor.queue>(p) + +#define CREATE_SELECT_FLAGGED_IF_BENCHMARK(T, F, p) executor.queue>(p) + +#define CREATE_UNIQUE_BENCHMARK(T, p) executor.queue>(p) + +#define CREATE_UNIQUE_BY_KEY_BENCHMARK(K, V, p) executor.queue>(p) + +#define BENCHMARK_FLAGGED_TYPE(type, value) \ + CREATE_SELECT_FLAGGED_BENCHMARK(type, value, 0.05f); \ + CREATE_SELECT_FLAGGED_BENCHMARK(type, value, 0.25f); \ + CREATE_SELECT_FLAGGED_BENCHMARK(type, value, 0.5f); \ + CREATE_SELECT_FLAGGED_BENCHMARK(type, value, 0.75f) - return 0; +#define BENCHMARK_IF_TYPE(type) \ + CREATE_SELECT_IF_BENCHMARK(type, 0.05f); \ + CREATE_SELECT_IF_BENCHMARK(type, 0.25f); \ + CREATE_SELECT_IF_BENCHMARK(type, 0.5f); \ + CREATE_SELECT_IF_BENCHMARK(type, 0.75f) + +#define BENCHMARK_FLAGGED_IF_TYPE(type, value) \ + CREATE_SELECT_FLAGGED_IF_BENCHMARK(type, value, 0.05f); \ + CREATE_SELECT_FLAGGED_IF_BENCHMARK(type, value, 0.25f); \ + CREATE_SELECT_FLAGGED_IF_BENCHMARK(type, value, 0.5f); \ + CREATE_SELECT_FLAGGED_IF_BENCHMARK(type, value, 0.75f) + +#define BENCHMARK_UNIQUE_TYPE(type) \ + CREATE_UNIQUE_BENCHMARK(type, 0.05f); \ + CREATE_UNIQUE_BENCHMARK(type, 0.25f); \ + CREATE_UNIQUE_BENCHMARK(type, 0.5f); \ + CREATE_UNIQUE_BENCHMARK(type, 0.75f) + +#define BENCHMARK_UNIQUE_BY_KEY_TYPE(Key, Value) \ + CREATE_UNIQUE_BY_KEY_BENCHMARK(Key, Value, 0.05f); \ + CREATE_UNIQUE_BY_KEY_BENCHMARK(Key, Value, 0.25f); \ + CREATE_UNIQUE_BY_KEY_BENCHMARK(Key, Value, 0.5f); \ + CREATE_UNIQUE_BY_KEY_BENCHMARK(Key, Value, 0.75f) + +int main(int argc, char* argv[]) +{ + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; + + primbench::executor executor(argc, argv, settings); + + BENCHMARK_FLAGGED_TYPE(int, unsigned char); + BENCHMARK_FLAGGED_TYPE(float, unsigned char); + BENCHMARK_FLAGGED_TYPE(double, unsigned char); + BENCHMARK_FLAGGED_TYPE(uint8_t, uint8_t); + BENCHMARK_FLAGGED_TYPE(int8_t, int8_t); + BENCHMARK_FLAGGED_TYPE(custom_double2, unsigned char); + + BENCHMARK_IF_TYPE(int); + BENCHMARK_IF_TYPE(float); + BENCHMARK_IF_TYPE(double); + BENCHMARK_IF_TYPE(uint8_t); + BENCHMARK_IF_TYPE(int8_t); + BENCHMARK_IF_TYPE(custom_int_double); + + BENCHMARK_FLAGGED_IF_TYPE(int, unsigned char); + BENCHMARK_FLAGGED_IF_TYPE(float, unsigned char); + BENCHMARK_FLAGGED_IF_TYPE(double, unsigned char); + BENCHMARK_FLAGGED_IF_TYPE(uint8_t, uint8_t); + BENCHMARK_FLAGGED_IF_TYPE(int8_t, int8_t); + BENCHMARK_FLAGGED_IF_TYPE(custom_double2, unsigned char); + + BENCHMARK_UNIQUE_TYPE(int); + BENCHMARK_UNIQUE_TYPE(float); + BENCHMARK_UNIQUE_TYPE(double); + BENCHMARK_UNIQUE_TYPE(uint8_t); + BENCHMARK_UNIQUE_TYPE(int8_t); + BENCHMARK_UNIQUE_TYPE(custom_int_double); + + BENCHMARK_UNIQUE_BY_KEY_TYPE(int, int); + BENCHMARK_UNIQUE_BY_KEY_TYPE(float, double); + BENCHMARK_UNIQUE_BY_KEY_TYPE(double, custom_double2); + BENCHMARK_UNIQUE_BY_KEY_TYPE(uint8_t, uint8_t); + BENCHMARK_UNIQUE_BY_KEY_TYPE(int8_t, double); + BENCHMARK_UNIQUE_BY_KEY_TYPE(custom_int_double, custom_int_double); + + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_device_spmv.cpp b/projects/hipcub/benchmark/benchmark_device_spmv.cpp deleted file mode 100644 index fcdb1ab9be9c..000000000000 --- a/projects/hipcub/benchmark/benchmark_device_spmv.cpp +++ /dev/null @@ -1,269 +0,0 @@ -// MIT License -// -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -#include "common_benchmark_header.hpp" - -// HIP API -#include - -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 32; -#endif - -const unsigned int batch_size = 10; -const unsigned int warmup_size = 5; - -template -void run_benchmark(benchmark::State& state, - size_t size, - const hipStream_t stream, - float probability) -{ - const T rand_min = T(1); - const T rand_max = T(10); - - // generate a lexicograhically sorted list of (row, column) index tuples - // number of nonzeroes cannot be guaranteed as duplicates may exist - const int num_nonzeroes_attempt = static_cast( - std::min(static_cast(INT_MAX), - static_cast(probability * static_cast(size * size)))); - std::vector> indices(num_nonzeroes_attempt); - { - std::vector flat_indices - = benchmark_utils::get_random_data(2 * num_nonzeroes_attempt, - 0, - size - 1, - 2 * num_nonzeroes_attempt); - for(int i = 0; i < num_nonzeroes_attempt; i++) - { - indices[i] = std::make_pair(flat_indices[2 * i], flat_indices[2 * i + 1]); - } - std::sort(indices.begin(), indices.end()); - } - - // generate the compressed sparse rows matrix - std::pair prev_cell = std::make_pair(-1, -1); - int num_nonzeroes = 0; - std::vector row_offsets(size + 1); - // this vector might be too large, but doing the allocation now eliminates a - // scan - std::vector column_indices(num_nonzeroes_attempt); - row_offsets[0] = 0; - int last_row_written = 0; - for(int i = 0; i < num_nonzeroes_attempt; i++) - { - if(indices[i] != prev_cell) - { - // update the row offets if we go to the next row (or skip some) - if(indices[i].first != last_row_written) - { - for(int j = last_row_written + 1; j <= indices[i].first; j++) - { - row_offsets[j] = num_nonzeroes; - } - last_row_written = indices[i].first; - } - - column_indices[num_nonzeroes++] = indices[i].second; - - prev_cell = indices[i]; - } - } - // fill in the entries for any missing rows - for(int j = last_row_written + 1; j < static_cast(size) + 1; j++) - { - row_offsets[j] = num_nonzeroes; - } - - // generate the random data once the actual number of nonzeroes are known - std::vector values = benchmark_utils::get_random_data(num_nonzeroes, rand_min, rand_max); - - std::vector vector_x = benchmark_utils::get_random_data(size, rand_min, rand_max); - - T* d_values; - int* d_row_offsets; - int* d_column_indices; - T* d_vector_x; - T* d_vector_y; - HIP_CHECK(hipMalloc(&d_values, values.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_row_offsets, row_offsets.size() * sizeof(int))); - HIP_CHECK(hipMalloc(&d_column_indices, num_nonzeroes * sizeof(int))); - HIP_CHECK(hipMalloc(&d_vector_x, vector_x.size() * sizeof(T))); - HIP_CHECK(hipMalloc(&d_vector_y, size * sizeof(T))); - HIP_CHECK(hipMemcpy(d_values, values.data(), values.size() * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_row_offsets, - row_offsets.data(), - row_offsets.size() * sizeof(int), - hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_column_indices, - column_indices.data(), - num_nonzeroes * sizeof(int), - hipMemcpyHostToDevice)); - HIP_CHECK( - hipMemcpy(d_vector_x, vector_x.data(), vector_x.size() * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - // Allocate temporary storage memory - size_t temp_storage_size_bytes; - - // Get size of d_temp_storage - HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH - HIP_CHECK(hipcub::DeviceSpmv::CsrMV(nullptr, - temp_storage_size_bytes, - d_values, - d_row_offsets, - d_column_indices, - d_vector_x, - d_vector_y, - size, - size, - num_nonzeroes, - stream)); - HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP - HIP_CHECK(hipDeviceSynchronize()); - - // allocate temporary storage - void* d_temp_storage = nullptr; - HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_size_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - // Warm-up - for(size_t i = 0; i < warmup_size; i++) - { - HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH - HIP_CHECK(hipcub::DeviceSpmv::CsrMV(d_temp_storage, - temp_storage_size_bytes, - d_values, - d_row_offsets, - d_column_indices, - d_vector_x, - d_vector_y, - size, - size, - num_nonzeroes, - stream)); - HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH - } - HIP_CHECK(hipDeviceSynchronize()); - - for(auto _ : state) - { - auto start = std::chrono::high_resolution_clock::now(); - for(size_t i = 0; i < batch_size; i++) - { - HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH - HIP_CHECK(hipcub::DeviceSpmv::CsrMV(d_temp_storage, - temp_storage_size_bytes, - d_values, - d_row_offsets, - d_column_indices, - d_vector_x, - d_vector_y, - size, - size, - num_nonzeroes, - stream)); - HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP - } - HIP_CHECK(hipDeviceSynchronize()); - - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * batch_size * (num_nonzeroes + size) * sizeof(T)); - state.SetItemsProcessed(state.iterations() * batch_size * (num_nonzeroes + size)); - - HIP_CHECK(hipFree(d_temp_storage)); - HIP_CHECK(hipFree(d_vector_y)); - HIP_CHECK(hipFree(d_vector_x)); - HIP_CHECK(hipFree(d_column_indices)); - HIP_CHECK(hipFree(d_row_offsets)); - HIP_CHECK(hipFree(d_values)); - HIP_CHECK(hipDeviceSynchronize()); -} - -#define CREATE_BENCHMARK(T, p) \ - benchmark::RegisterBenchmark( \ - std::string("device_spmv_CsrMV.").c_str(), \ - &run_benchmark, \ - size, \ - stream, \ - p) - -#define BENCHMARK_TYPE(type) \ - CREATE_BENCHMARK(type, 1.0e-6f), CREATE_BENCHMARK(type, 1.0e-5f), \ - CREATE_BENCHMARK(type, 1.0e-4f), CREATE_BENCHMARK(type, 1.0e-3f), \ - CREATE_BENCHMARK(type, 1.0e-2f) - -int main(int argc, char* argv[]) -{ - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - - std::cout << "benchmark_device_spmv" << std::endl; - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks = { - BENCHMARK_TYPE(int), - BENCHMARK_TYPE(unsigned int), - BENCHMARK_TYPE(float), - BENCHMARK_TYPE(double), - }; - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } - - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } - - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - - return 0; -} diff --git a/projects/hipcub/benchmark/benchmark_utils.hpp b/projects/hipcub/benchmark/benchmark_utils.hpp index 489cf8de8525..3559df817dcc 100644 --- a/projects/hipcub/benchmark/benchmark_utils.hpp +++ b/projects/hipcub/benchmark/benchmark_utils.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,14 +20,29 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE. -#ifndef HIPCUB_BENCHMARK_UTILS_HPP_ -#define HIPCUB_BENCHMARK_UTILS_HPP_ +#pragma once -#ifndef BENCHMARK_UTILS_INCLUDE_GUARD - #error benchmark_utils.hpp must ONLY be included by common_benchmark_header.hpp. Please include common_benchmark_header.hpp instead. -#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "primbench.hpp" + +#include + +#include + +#include _HIPCUB_LIBCXX_INCLUDE(cmath) +#include _HIPCUB_STD_INCLUDE(limits) -// hipCUB API #ifdef __HIP_PLATFORM_AMD__ #include #elif defined(__HIP_PLATFORM_NVIDIA__) @@ -36,71 +51,76 @@ #endif #include +#include #ifndef HIPCUB_CUB_API #define HIPCUB_WARP_THREADS_MACRO warpSize #else - #define HIPCUB_WARP_THREADS_MACRO CUB_PTX_WARP_THREADS + #define HIPCUB_WARP_THREADS_MACRO warp_threads #endif +#include _HIPCUB_STD_INCLUDE(limits) + namespace benchmark_utils { + const size_t default_max_random_size = 1024 * 1024; + // get_random_data() generates only part of sequence and replicates it, // because benchmarks usually do not need "true" random sequence. template inline auto - get_random_data(size_t size, T min, T max, size_t max_random_size = default_max_random_size) -> + get_random_data(size_t items, T min, T max, size_t max_random_size = default_max_random_size) -> typename std::enable_if::value, std::vector>::type { std::random_device rd; std::default_random_engine gen(rd()); using distribution_type = typename std::conditional<(sizeof(T) == 1), short, T>::type; std::uniform_int_distribution distribution(min, max); - std::vector data(size); + std::vector data(items); std::generate(data.begin(), - data.begin() + std::min(size, max_random_size), + data.begin() + _HIPCUB_STD::min(items, max_random_size), [&]() { return distribution(gen); }); - for(size_t i = max_random_size; i < size; i += max_random_size) + for(size_t i = max_random_size; i < items; i += max_random_size) { - std::copy_n(data.begin(), std::min(size - i, max_random_size), data.begin() + i); + std::copy_n(data.begin(), _HIPCUB_STD::min(items - i, max_random_size), data.begin() + i); } return data; } template inline auto - get_random_data(size_t size, T min, T max, size_t max_random_size = default_max_random_size) -> + get_random_data(size_t items, T min, T max, size_t max_random_size = default_max_random_size) -> typename std::enable_if::value, std::vector>::type { std::random_device rd; std::default_random_engine gen(rd()); std::uniform_real_distribution distribution(min, max); - std::vector data(size); + std::vector data(items); std::generate(data.begin(), - data.begin() + std::min(size, max_random_size), + data.begin() + _HIPCUB_STD::min(items, max_random_size), [&]() { return distribution(gen); }); - for(size_t i = max_random_size; i < size; i += max_random_size) + for(size_t i = max_random_size; i < items; i += max_random_size) { - std::copy_n(data.begin(), std::min(size - i, max_random_size), data.begin() + i); + std::copy_n(data.begin(), _HIPCUB_STD::min(items - i, max_random_size), data.begin() + i); } return data; } template inline std::vector - get_random_data01(size_t size, float p, size_t max_random_size = default_max_random_size) + get_random_data01(size_t items, float p, size_t max_random_size = default_max_random_size) { std::random_device rd; std::default_random_engine gen(rd()); std::bernoulli_distribution distribution(p); - std::vector data(size); + std::vector data(items); std::generate(data.begin(), - data.begin() + std::min(size, max_random_size), + data.begin() + _HIPCUB_STD::min(items, max_random_size), [&]() { return distribution(gen); }); - for(size_t i = max_random_size; i < size; i += max_random_size) + for(size_t i = max_random_size; i < items; i += max_random_size) { - std::copy_n(data.begin(), std::min(size - i, max_random_size), data.begin() + i); + std::copy_n(data.begin(), _HIPCUB_STD::min(items - i, max_random_size), data.begin() + i); } return data; } @@ -114,12 +134,12 @@ inline T get_random_value(T min, T max) // Can't use std::prefix_sum for inclusive/exclusive scan, because // it does not handle short[] -> int(int a, int b) { a + b; } -> int[] // they way we expect. That's because sum in std::prefix_sum's implementation -// is of type typename std::iterator_traits::value_type (short) +// is of type ::hipcub::detail::it_value_t (short) template OutputIt host_inclusive_scan(InputIt first, InputIt last, OutputIt d_first, BinaryOperation op) { - using input_type = typename std::iterator_traits::value_type; - using output_type = typename std::iterator_traits::value_type; + using input_type = ::hipcub::detail::it_value_t; + using output_type = ::hipcub::detail::it_value_t; using result_type = typename std::conditional::value, input_type, output_type>::type; @@ -141,8 +161,8 @@ template OutputIt host_exclusive_scan( InputIt first, InputIt last, T initial_value, OutputIt d_first, BinaryOperation op) { - using input_type = typename std::iterator_traits::value_type; - using output_type = typename std::iterator_traits::value_type; + using input_type = ::hipcub::detail::it_value_t; + using output_type = ::hipcub::detail::it_value_t; using result_type = typename std::conditional::value, input_type, output_type>::type; @@ -175,8 +195,8 @@ OutputIt host_exclusive_scan_by_key(InputIt first, BinaryOperation op, KeyCompare key_compare_op) { - using input_type = typename std::iterator_traits::value_type; - using output_type = typename std::iterator_traits::value_type; + using input_type = ::hipcub::detail::it_value_t; + using output_type = ::hipcub::detail::it_value_t; using result_type = typename std::conditional::value, input_type, output_type>::type; @@ -191,7 +211,8 @@ OutputIt host_exclusive_scan_by_key(InputIt first, if(key_compare_op(*k_first, *++k_first)) { sum = op(sum, static_cast(*first)); - } else + } + else { sum = initial_value; } @@ -224,46 +245,54 @@ struct custom_type HIPCUB_HOST_DEVICE inline ~custom_type() = default; #endif - HIPCUB_HOST_DEVICE inline custom_type& operator=(const custom_type& other) + HIPCUB_HOST_DEVICE + inline custom_type& operator=(const custom_type& other) { x = other.x; y = other.y; return *this; } - HIPCUB_HOST_DEVICE inline custom_type operator+(const custom_type& rhs) const + HIPCUB_HOST_DEVICE + inline custom_type operator+(const custom_type& rhs) const { return custom_type(x + rhs.x, y + rhs.y); } - HIPCUB_HOST_DEVICE inline custom_type operator-(const custom_type& other) const + HIPCUB_HOST_DEVICE + inline custom_type operator-(const custom_type& other) const { return custom_type(x - other.x, y - other.y); } - HIPCUB_HOST_DEVICE inline bool operator<(const custom_type& rhs) const + HIPCUB_HOST_DEVICE + inline bool operator<(const custom_type& rhs) const { // intentionally suboptimal choice for short-circuting, // required to generate more performant device code return ((x == rhs.x && y < rhs.y) || x < rhs.x); } - HIPCUB_HOST_DEVICE inline bool operator>(const custom_type& other) const + HIPCUB_HOST_DEVICE + inline bool operator>(const custom_type& other) const { return (x > other.x || (x == other.x && y > other.y)); } - HIPCUB_HOST_DEVICE inline bool operator==(const custom_type& rhs) const + HIPCUB_HOST_DEVICE + inline bool operator==(const custom_type& rhs) const { return x == rhs.x && y == rhs.y; } - HIPCUB_HOST_DEVICE inline bool operator!=(const custom_type& other) const + HIPCUB_HOST_DEVICE + inline bool operator!=(const custom_type& other) const { return !(*this == other); } - HIPCUB_HOST_DEVICE custom_type& operator+=(const custom_type& rhs) + HIPCUB_HOST_DEVICE + custom_type& operator+=(const custom_type& rhs) { this->x += rhs.x; this->y += rhs.y; @@ -289,7 +318,8 @@ struct custom_type_decomposer using T = typename CustomType::first_type; using U = typename CustomType::second_type; - HIPCUB_HOST_DEVICE ::hipcub::tuple operator()(CustomType& key) const + HIPCUB_HOST_DEVICE + ::hipcub::tuple operator()(CustomType& key) const { return ::hipcub::tuple{key.x, key.y}; } @@ -303,11 +333,11 @@ struct generate_limits::value>> { static inline T min() { - return std::numeric_limits::min(); + return _HIPCUB_STD::numeric_limits::min(); } static inline T max() { - return std::numeric_limits::max(); + return _HIPCUB_STD::numeric_limits::max(); } }; @@ -340,15 +370,15 @@ struct generate_limits::value>> }; template -inline auto get_random_data(size_t size, T min, T max, size_t max_random_size = 1024 * 1024) -> +inline auto get_random_data(size_t items, T min, T max, size_t max_random_size = 1024 * 1024) -> typename std::enable_if::value, std::vector>::type { using first_type = typename T::first_type; using second_type = typename T::second_type; - std::vector data(size); - auto fdata = get_random_data(size, min.x, max.x, max_random_size); - auto sdata = get_random_data(size, min.y, max.y, max_random_size); - for(size_t i = 0; i < size; i++) + std::vector data(items); + auto fdata = get_random_data(items, min.x, max.x, max_random_size); + auto sdata = get_random_data(items, min.y, max.y, max_random_size); + for(size_t i = 0; i < items; i++) { data[i] = T(fdata[i], sdata[i]); } @@ -356,16 +386,15 @@ inline auto get_random_data(size_t size, T min, T max, size_t max_random_size = } template -inline auto get_random_data(size_t size, T min, T max, size_t max_random_size = 1024 * 1024) -> - typename std::enable_if::value - && !std::is_same::value, +inline auto get_random_data(size_t items, T min, T max, size_t max_random_size = 1024 * 1024) -> + typename std::enable_if::value && !std::is_same_v, std::vector>::type { using field_type = decltype(max.x); - std::vector data(size); - auto field_data = get_random_data(size, min.x, max.x, max_random_size); - for(size_t i = 0; i < size; i++) + std::vector data(items); + auto field_data = get_random_data(items, min.x, max.x, max_random_size); + for(size_t i = 0; i < items; i++) { data[i] = T(field_data[i]); } @@ -374,7 +403,7 @@ inline auto get_random_data(size_t size, T min, T max, size_t max_random_size = template std::vector - get_random_segments(const size_t size, const size_t max_segment_length, const int seed_value) + get_random_segments(const size_t items, const size_t max_segment_length, const int seed_value) { static_assert(std::is_arithmetic::value, "Key type must be arithmetic"); @@ -383,15 +412,16 @@ std::vector using key_distribution_type = std::conditional_t::value, std::uniform_int_distribution, std::uniform_real_distribution>; - key_distribution_type key_distribution(std::numeric_limits::max()); - std::vector keys(size); + key_distribution_type key_distribution(_HIPCUB_STD::numeric_limits::max()); + std::vector keys(items); size_t keys_start_index = 0; - while(keys_start_index < size) + while(keys_start_index < items) { const size_t new_segment_length = segment_length_distribution(prng); - const size_t new_segment_end = std::min(size, keys_start_index + new_segment_length); - const T key = key_distribution(prng); + const size_t new_segment_end + = _HIPCUB_STD::min(items, keys_start_index + new_segment_length); + const T key = key_distribution(prng); std::fill(std::next(keys.begin(), keys_start_index), std::next(keys.begin(), new_segment_end), key); @@ -417,51 +447,56 @@ inline constexpr bool is_power_of_two(const T x) return (x > 0) && ((x & (x - 1)) == 0); } -template -using it_value_t = typename std::iterator_traits::value_type; - -using engine_type = std::default_random_engine; - // generate_random_data_n() generates only part of sequence and replicates it, // because benchmarks usually do not need "true" random sequence. template -inline auto generate_random_data_n( - OutputIter it, size_t size, U min, V max, Generator& gen, size_t max_random_size = 1024 * 1024) - -> typename std::enable_if_t>::value, OutputIter> +inline auto generate_random_data_n(OutputIter it, + size_t items, + U min, + V max, + Generator& gen, + size_t max_random_size = 1024 * 1024) -> + typename std::enable_if_t>::value, + OutputIter> { - using T = it_value_t; + using T = ::hipcub::detail::it_value_t; using dis_type = typename std::conditional<(sizeof(T) == 1), short, T>::type; std::uniform_int_distribution distribution((T)min, (T)max); - std::generate_n(it, std::min(size, max_random_size), [&]() { return distribution(gen); }); - for(size_t i = max_random_size; i < size; i += max_random_size) + std::generate_n(it, + _HIPCUB_STD::min(items, max_random_size), + [&]() { return distribution(gen); }); + for(size_t i = max_random_size; i < items; i += max_random_size) { - std::copy_n(it, std::min(size - i, max_random_size), it + i); + std::copy_n(it, _HIPCUB_STD::min(items - i, max_random_size), it + i); } - return it + size; + return it + items; } template inline auto generate_random_data_n(OutputIterator it, - size_t size, + size_t items, U min, V max, Generator& gen, size_t max_random_size = 1024 * 1024) - -> std::enable_if_t>::value, OutputIterator> + -> std::enable_if_t>::value, + OutputIterator> { - using T = typename std::iterator_traits::value_type; + using T = ::hipcub::detail::it_value_t; std::uniform_real_distribution distribution((T)min, (T)max); - std::generate_n(it, std::min(size, max_random_size), [&]() { return distribution(gen); }); - for(size_t i = max_random_size; i < size; i += max_random_size) + std::generate_n(it, + _HIPCUB_STD::min(items, max_random_size), + [&]() { return distribution(gen); }); + for(size_t i = max_random_size; i < items; i += max_random_size) { - std::copy_n(it, std::min(size - i, max_random_size), it + i); + std::copy_n(it, _HIPCUB_STD::min(items - i, max_random_size), it + i); } - return it + size; + return it + items; } -template +template struct alignas(Alignment) custom_aligned_type { unsigned char data[Size]; @@ -475,6 +510,26 @@ inline constexpr auto ceiling_div(const T a, const U b) return a / b + (a % b > 0 ? 1 : 0); } +struct minimum +{ + template + HIPCUB_HOST_DEVICE + auto operator()(const T& a, const U& b) const + { + return a < b ? a : b; + } +}; + +struct plus +{ + template + HIPCUB_HOST_DEVICE + constexpr auto operator()(const A& a, const B& b) const -> decltype(a + b) + { + return a + b; + } +}; + } // namespace benchmark_utils // Need for hipcub::DeviceReduce::Min/Max etc. @@ -488,17 +543,17 @@ class numeric_limits> public: static constexpr inline T min() { - return std::numeric_limits::min(); + return _HIPCUB_STD::numeric_limits::min(); } static constexpr inline T max() { - return std::numeric_limits::max(); + return _HIPCUB_STD::numeric_limits::max(); } static constexpr inline T lowest() { - return std::numeric_limits::lowest(); + return _HIPCUB_STD::numeric_limits::lowest(); } }; @@ -510,19 +565,53 @@ class numeric_limits> public: static constexpr inline T min() { - return std::numeric_limits::min(); + return _HIPCUB_STD::numeric_limits::min(); } static constexpr inline T max() { - return std::numeric_limits::max(); + return _HIPCUB_STD::numeric_limits::max(); } static constexpr inline T lowest() { - return std::numeric_limits::lowest(); + return _HIPCUB_STD::numeric_limits::lowest(); } }; } // namespace std -#endif // HIPCUB_BENCHMARK_UTILS_HPP_ +#define HIP_CHECK(condition) \ + { \ + hipError_t error = condition; \ + if(error != hipSuccess) \ + { \ + std::cout << "HIP error: " << error << " line: " << __LINE__ << std::endl; \ + exit(error); \ + } \ + } + +PRIMBENCH_REGISTER_TYPE(int8_t, "i8") +PRIMBENCH_REGISTER_TYPE(int16_t, "i16") +PRIMBENCH_REGISTER_TYPE(int32_t, "i32") +PRIMBENCH_REGISTER_TYPE(int64_t, "i64") +PRIMBENCH_REGISTER_TYPE(uint8_t, "u8") +PRIMBENCH_REGISTER_TYPE(uint16_t, "u16") +PRIMBENCH_REGISTER_TYPE(uint32_t, "u32") +PRIMBENCH_REGISTER_TYPE(unsigned long long, "u64") +PRIMBENCH_REGISTER_TYPE(float, "f32") +PRIMBENCH_REGISTER_TYPE(double, "f64") +PRIMBENCH_REGISTER_TYPE(__half, "f16") + +using custom_int_t = benchmark_utils::custom_type; +using custom_float2 = benchmark_utils::custom_type; +using custom_double2 = benchmark_utils::custom_type; +using custom_char_double = benchmark_utils::custom_type; +using custom_double_char = benchmark_utils::custom_type; +using custom_int_double = benchmark_utils::custom_type; + +PRIMBENCH_REGISTER_TYPE(custom_int_t, "custom"); +PRIMBENCH_REGISTER_TYPE(custom_float2, "custom") +PRIMBENCH_REGISTER_TYPE(custom_double2, "custom") +PRIMBENCH_REGISTER_TYPE(custom_char_double, "custom") +PRIMBENCH_REGISTER_TYPE(custom_double_char, "custom") +PRIMBENCH_REGISTER_TYPE(custom_int_double, "custom") diff --git a/projects/hipcub/benchmark/benchmark_warp_exchange.cpp b/projects/hipcub/benchmark/benchmark_warp_exchange.cpp index 0c41be0588ad..2e67eb7e0481 100644 --- a/projects/hipcub/benchmark/benchmark_warp_exchange.cpp +++ b/projects/hipcub/benchmark/benchmark_warp_exchange.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,16 +20,21 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif +constexpr const char* get_algorithm_name(hipcub::WarpExchangeAlgorithm algorithm) +{ + switch(algorithm) + { + case hipcub::WarpExchangeAlgorithm::WARP_EXCHANGE_SMEM: return "warp_exchange_smem"; + case hipcub::WarpExchangeAlgorithm::WARP_EXCHANGE_SHUFFLE: return "warp_exchange_shuffle"; + } + return "unknown_algorithm"; +} template -__device__ auto warp_exchange_benchmark(T* d_output) +__device__ +auto warp_exchange_benchmark(T* d_output) -> std::enable_if_t> { T thread_data[ItemsPerThread]; -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(unsigned i = 0; i < ItemsPerThread; ++i) { thread_data[i] = static_cast(i); } - using WarpExchangeT = ::hipcub::WarpExchange; + using WarpExchangeT = ::hipcub::WarpExchange; constexpr unsigned warps_in_block = BlockSize / LogicalWarpSize; - __shared__ typename WarpExchangeT::TempStorage temp_storage[warps_in_block]; + __shared__ + typename WarpExchangeT::TempStorage temp_storage[warps_in_block]; const unsigned warp_id = threadIdx.x / LogicalWarpSize; WarpExchangeT warp_exchange(temp_storage[warp_id]); Op{}(warp_exchange, thread_data); -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(unsigned i = 0; i < ItemsPerThread; ++i) { const unsigned global_idx = (BlockSize * blockIdx.x + threadIdx.x) * ItemsPerThread + i; @@ -73,7 +76,8 @@ template -__device__ auto warp_exchange_benchmark(T* /*d_output*/) +__device__ +auto warp_exchange_benchmark(T* /*d_output*/) -> std::enable_if_t> {} @@ -83,7 +87,8 @@ template -__global__ __launch_bounds__(BlockSize) void warp_exchange_kernel(T* d_output) +__global__ __launch_bounds__(BlockSize) +void warp_exchange_kernel(T* d_output) { warp_exchange_benchmark(d_output); } @@ -93,13 +98,14 @@ template -__device__ auto warp_exchange_scatter_to_striped_benchmark(T* d_output) +__device__ +auto warp_exchange_scatter_to_striped_benchmark(T* d_output) -> std::enable_if_t> { const unsigned warp_id = threadIdx.x / LogicalWarpSize; T thread_data[ItemsPerThread]; OffsetT thread_ranks[ItemsPerThread]; -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(unsigned i = 0; i < ItemsPerThread; ++i) { thread_data[i] = static_cast(i); @@ -107,12 +113,13 @@ __device__ auto warp_exchange_scatter_to_striped_benchmark(T* d_output) } using WarpExchangeT = ::hipcub::WarpExchange; - constexpr unsigned warps_in_block = BlockSize / LogicalWarpSize; - __shared__ typename WarpExchangeT::TempStorage temp_storage[warps_in_block]; + constexpr unsigned warps_in_block = BlockSize / LogicalWarpSize; + __shared__ + typename WarpExchangeT::TempStorage temp_storage[warps_in_block]; WarpExchangeT(temp_storage[warp_id]).ScatterToStriped(thread_data, thread_ranks); -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(unsigned i = 0; i < ItemsPerThread; ++i) { const unsigned striped_global_idx @@ -126,7 +133,8 @@ template -__device__ auto warp_exchange_scatter_to_striped_benchmark(T* /*d_output*/) +__device__ +auto warp_exchange_scatter_to_striped_benchmark(T* /*d_output*/) -> std::enable_if_t> {} @@ -135,7 +143,8 @@ template -__global__ __launch_bounds__(BlockSize) void warp_exchange_scatter_to_striped_kernel(T* d_output) +__global__ __launch_bounds__(BlockSize) +void warp_exchange_scatter_to_striped_kernel(T* d_output) { warp_exchange_scatter_to_striped_benchmark( d_output); @@ -147,82 +156,103 @@ template -void run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) +class exchange_benchmark : public primbench::benchmark_interface { - constexpr unsigned trials = 100; - constexpr unsigned items_per_block = BlockSize * ItemsPerThread; - const unsigned size = items_per_block * ((N + items_per_block - 1) / items_per_block); - - T* d_output; - HIP_CHECK(hipMalloc(&d_output, size * sizeof(T))); - - for(auto _ : state) + primbench::json meta() const override { - auto start = std::chrono::high_resolution_clock::now(); - - for(size_t i = 0; i < trials; ++i) - { - warp_exchange_kernel - <<>>(d_output); - } - - HIP_CHECK(hipPeekAtLastError()) - HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + return primbench::json{} + .add("algo", "warp_exchange") + .add("subalgo", get_algorithm_name(Algorithm)) + .add("data_type", primbench::name()) + .add("block_size", BlockSize) + .add("items_per_thread", ItemsPerThread) + .add("warp_size", LogicalWarpSize) + .add("op", Op::name) + .add("lvl", "warp"); } - state.SetBytesProcessed(state.iterations() * trials * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * trials * size); - HIP_CHECK(hipFree(d_output)); -} + void run(primbench::state& state) override + { + const size_t& input_items = state.size; + const auto& stream = state.stream; + + constexpr unsigned items_per_block = BlockSize * ItemsPerThread; + const unsigned items + = items_per_block * ((input_items + items_per_block - 1) / items_per_block); + + T* d_output; + HIP_CHECK(hipMalloc(&d_output, items * sizeof(T))); + + state.set_items(items); + state.add_writes(items); + + state.run( + [&] + { + warp_exchange_kernel + <<>>(d_output); + }); + + HIP_CHECK(hipFree(d_output)); + } +}; template -void run_benchmark_scatter_to_striped(benchmark::State& state, hipStream_t stream, size_t N) +class scatter_to_striped_benchmark : public primbench::benchmark_interface { - constexpr unsigned trials = 100; - constexpr unsigned items_per_block = BlockSize * ItemsPerThread; - const unsigned size = items_per_block * ((N + items_per_block - 1) / items_per_block); - - T* d_output; - HIP_CHECK(hipMalloc(&d_output, size * sizeof(T))); - - for(auto _ : state) + primbench::json meta() const override { - auto start = std::chrono::high_resolution_clock::now(); - - for(size_t i = 0; i < trials; ++i) - { - warp_exchange_scatter_to_striped_kernel - <<>>(d_output); - } - - HIP_CHECK(hipPeekAtLastError()) - HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + return primbench::json{} + .add("algo", "warp_exchange") + .add("subalgo", "scatter_to_striped") + .add("data_type", primbench::name()) + .add("offset_type", primbench::name()) + .add("block_size", BlockSize) + .add("items_per_thread", ItemsPerThread) + .add("warp_size", LogicalWarpSize) + .add("lvl", "warp"); } - state.SetBytesProcessed(state.iterations() * trials * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * trials * size); - HIP_CHECK(hipFree(d_output)); -} + void run(primbench::state& state) override + { + const size_t& input_items = state.size; + const auto& stream = state.stream; + + constexpr unsigned items_per_block = BlockSize * ItemsPerThread; + const unsigned items + = items_per_block * ((input_items + items_per_block - 1) / items_per_block); + + T* d_output; + HIP_CHECK(hipMalloc(&d_output, items * sizeof(T))); + + state.set_items(items); + state.add_writes(items); + + state.run( + [&] + { + warp_exchange_scatter_to_striped_kernel + <<>>(d_output); + }); + + HIP_CHECK(hipFree(d_output)); + } +}; struct StripedToBlockedOp { + static constexpr const char* name = "striped_to_blocked_op"; + template - __device__ void operator()(WarpExchangeT& warp_exchange, T (&thread_data)[ItemsPerThread]) const + __device__ + void operator()(WarpExchangeT& warp_exchange, T (&thread_data)[ItemsPerThread]) const { warp_exchange.StripedToBlocked(thread_data, thread_data); } @@ -230,128 +260,75 @@ struct StripedToBlockedOp struct BlockedToStripedOp { + static constexpr const char* name = "blocked_to_striped_op"; + template - __device__ void operator()(WarpExchangeT& warp_exchange, T (&thread_data)[ItemsPerThread]) const + __device__ + void operator()(WarpExchangeT& warp_exchange, T (&thread_data)[ItemsPerThread]) const { warp_exchange.BlockedToStriped(thread_data, thread_data); } }; -#define CREATE_BENCHMARK_STRIPED_TO_BLOCKED(T, BS, IT, WS, ALG) \ - benchmark::RegisterBenchmark(std::string("warp_exchange_striped_to_blocked.") \ - .c_str(), \ - &run_benchmark, \ - stream, \ - size) - -#define CREATE_BENCHMARK_BLOCKED_TO_STRIPED(T, BS, IT, WS, ALG) \ - benchmark::RegisterBenchmark(std::string("warp_exchange_blocked_to_striped.") \ - .c_str(), \ - &run_benchmark, \ - stream, \ - size) - -#define CREATE_BENCHMARK_SCATTER_TO_STRIPED(T, OFFSET_T, BS, IT, WS) \ - benchmark::RegisterBenchmark(std::string("warp_exchange_scatter_to_striped.") \ - .c_str(), \ - &run_benchmark_scatter_to_striped, \ - stream, \ - size) +#define CREATE_BENCHMARK_STRIPED_TO_BLOCKED(T, BS, IT, WS, ALG) \ + executor.queue>(); + +#define CREATE_BENCHMARK_BLOCKED_TO_STRIPED(T, BS, IT, WS, ALG) \ + executor.queue>(); + +#define CREATE_BENCHMARK_SCATTER_TO_STRIPED(T, OFFSET_T, BS, IT, WS) \ + executor.queue>(); int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_warp_exchange" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; + + primbench::executor executor(argc, argv, settings); // Add benchmarks - std::vector benchmarks{ - CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 128, 4, 16, ::hipcub::WARP_EXCHANGE_SMEM), - CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 128, 4, 16, ::hipcub::WARP_EXCHANGE_SMEM), - CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 128, 16, 16, ::hipcub::WARP_EXCHANGE_SMEM), - CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 128, 16, 16, ::hipcub::WARP_EXCHANGE_SMEM), - CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 128, 4, 32, ::hipcub::WARP_EXCHANGE_SMEM), - CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 128, 4, 32, ::hipcub::WARP_EXCHANGE_SMEM), - CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 256, 4, 32, ::hipcub::WARP_EXCHANGE_SMEM), - CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 256, 4, 32, ::hipcub::WARP_EXCHANGE_SMEM), - CREATE_BENCHMARK_SCATTER_TO_STRIPED(int, int, 128, 4, 16), - CREATE_BENCHMARK_SCATTER_TO_STRIPED(int, int, 128, 4, 32), - CREATE_BENCHMARK_SCATTER_TO_STRIPED(int, int, 256, 4, 32), - - CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 128, 16, 16, ::hipcub::WARP_EXCHANGE_SHUFFLE), - CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 128, 16, 16, ::hipcub::WARP_EXCHANGE_SHUFFLE), + CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 128, 4, 16, ::hipcub::WARP_EXCHANGE_SMEM); + CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 128, 4, 16, ::hipcub::WARP_EXCHANGE_SMEM); + CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 128, 16, 16, ::hipcub::WARP_EXCHANGE_SMEM); + CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 128, 16, 16, ::hipcub::WARP_EXCHANGE_SMEM); + CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 128, 4, 32, ::hipcub::WARP_EXCHANGE_SMEM); + CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 128, 4, 32, ::hipcub::WARP_EXCHANGE_SMEM); + CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 256, 4, 32, ::hipcub::WARP_EXCHANGE_SMEM); + CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 256, 4, 32, ::hipcub::WARP_EXCHANGE_SMEM); + CREATE_BENCHMARK_SCATTER_TO_STRIPED(int, int, 128, 4, 16); + CREATE_BENCHMARK_SCATTER_TO_STRIPED(int, int, 128, 4, 32); + CREATE_BENCHMARK_SCATTER_TO_STRIPED(int, int, 256, 4, 32); + + CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 128, 16, 16, ::hipcub::WARP_EXCHANGE_SHUFFLE); + CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 128, 16, 16, ::hipcub::WARP_EXCHANGE_SHUFFLE); // CUB requires WS == IPT for WARP_EXCHANGE_SHUFFLE #ifdef HIPCUB_ROCPRIM_API - CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 128, 4, 16, ::hipcub::WARP_EXCHANGE_SHUFFLE), - CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 128, 4, 16, ::hipcub::WARP_EXCHANGE_SHUFFLE), - CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 128, 4, 32, ::hipcub::WARP_EXCHANGE_SHUFFLE), - CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 128, 4, 32, ::hipcub::WARP_EXCHANGE_SHUFFLE), - CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 256, 4, 32, ::hipcub::WARP_EXCHANGE_SHUFFLE), - CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 256, 4, 32, ::hipcub::WARP_EXCHANGE_SHUFFLE), + CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 128, 4, 16, ::hipcub::WARP_EXCHANGE_SHUFFLE); + CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 128, 4, 16, ::hipcub::WARP_EXCHANGE_SHUFFLE); + CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 128, 4, 32, ::hipcub::WARP_EXCHANGE_SHUFFLE); + CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 128, 4, 32, ::hipcub::WARP_EXCHANGE_SHUFFLE); + CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 256, 4, 32, ::hipcub::WARP_EXCHANGE_SHUFFLE); + CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 256, 4, 32, ::hipcub::WARP_EXCHANGE_SHUFFLE); #endif - }; #ifdef HIPCUB_ROCPRIM_API if(::benchmark_utils::is_warp_size_supported(64)) { - std::vector additional_benchmarks{ - CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 128, 4, 64, ::hipcub::WARP_EXCHANGE_SMEM), - CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 128, 4, 64, ::hipcub::WARP_EXCHANGE_SHUFFLE), - CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 128, 4, 64, ::hipcub::WARP_EXCHANGE_SMEM), - CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 128, 4, 64, ::hipcub::WARP_EXCHANGE_SHUFFLE), - CREATE_BENCHMARK_SCATTER_TO_STRIPED(int, int, 128, 4, 64), - - CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 256, 4, 64, ::hipcub::WARP_EXCHANGE_SMEM), - CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 256, 4, 64, ::hipcub::WARP_EXCHANGE_SHUFFLE), - CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 256, 4, 64, ::hipcub::WARP_EXCHANGE_SMEM), - CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 256, 4, 64, ::hipcub::WARP_EXCHANGE_SHUFFLE), - CREATE_BENCHMARK_SCATTER_TO_STRIPED(int, int, 256, 4, 64)}; - benchmarks.insert(benchmarks.end(), - additional_benchmarks.begin(), - additional_benchmarks.end()); + CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 128, 4, 64, ::hipcub::WARP_EXCHANGE_SMEM); + CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 128, 4, 64, ::hipcub::WARP_EXCHANGE_SHUFFLE); + CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 128, 4, 64, ::hipcub::WARP_EXCHANGE_SMEM); + CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 128, 4, 64, ::hipcub::WARP_EXCHANGE_SHUFFLE); + CREATE_BENCHMARK_SCATTER_TO_STRIPED(int, int, 128, 4, 64); + + CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 256, 4, 64, ::hipcub::WARP_EXCHANGE_SMEM); + CREATE_BENCHMARK_STRIPED_TO_BLOCKED(int, 256, 4, 64, ::hipcub::WARP_EXCHANGE_SHUFFLE); + CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 256, 4, 64, ::hipcub::WARP_EXCHANGE_SMEM); + CREATE_BENCHMARK_BLOCKED_TO_STRIPED(int, 256, 4, 64, ::hipcub::WARP_EXCHANGE_SHUFFLE); + CREATE_BENCHMARK_SCATTER_TO_STRIPED(int, int, 256, 4, 64); } #endif - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } - - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } - - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_warp_load.cpp b/projects/hipcub/benchmark/benchmark_warp_load.cpp index 2c74609dfc95..1c11c70fd084 100644 --- a/projects/hipcub/benchmark/benchmark_warp_load.cpp +++ b/projects/hipcub/benchmark/benchmark_warp_load.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,37 +20,46 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif +constexpr const char* getAlgorithmName(hipcub::WarpLoadAlgorithm algorithm) +{ + switch(algorithm) + { + case hipcub::WarpLoadAlgorithm::WARP_LOAD_DIRECT: return "warp_load_direct"; + case hipcub::WarpLoadAlgorithm::WARP_LOAD_STRIPED: return "warp_load_striped"; + case hipcub::WarpLoadAlgorithm::WARP_LOAD_VECTORIZE: return "warp_load_vectorize"; + case hipcub::WarpLoadAlgorithm::WARP_LOAD_TRANSPOSE: return "warp_load_transpose"; + } + + return nullptr; +} template -__device__ auto warp_load_benchmark(T* d_input, T* d_output) +__device__ +auto warp_load_device_fn(T* d_input, T* d_output) -> std::enable_if_t> { using WarpLoadT = ::hipcub::WarpLoad; constexpr unsigned warps_in_block = BlockSize / LogicalWarpSize; constexpr int tile_size = ItemsPerThread * LogicalWarpSize; - const unsigned warp_id = threadIdx.x / LogicalWarpSize; + const unsigned warp_id = threadIdx.x / LogicalWarpSize; const unsigned global_warp_id = blockIdx.x * warps_in_block + warp_id; __shared__ typename WarpLoadT::TempStorage temp_storage[warps_in_block]; T thread_data[ItemsPerThread]; WarpLoadT(temp_storage[warp_id]).Load(d_input + global_warp_id * tile_size, thread_data); -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(unsigned i = 0; i < ItemsPerThread; ++i) { const unsigned striped_global_idx @@ -64,7 +73,8 @@ template -__device__ auto warp_load_benchmark(T* /*d_input*/, T* /*d_output*/) +__device__ +auto warp_load_device_fn(T* /*d_input*/, T* /*d_output*/) -> std::enable_if_t> {} @@ -73,196 +83,173 @@ template -__global__ __launch_bounds__(BlockSize) void warp_load_kernel(T* d_input, T* d_output) +__global__ __launch_bounds__(BlockSize) +void warp_load_kernel(T* d_input, T* d_output) { - warp_load_benchmark(d_input, d_output); + warp_load_device_fn(d_input, d_output); } template -void run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) + ::hipcub::WarpLoadAlgorithm Algorithm> +struct warp_load_benchmark : public primbench::benchmark_interface { - constexpr unsigned items_per_block = BlockSize * ItemsPerThread; - const unsigned size = items_per_block * ((N + items_per_block - 1) / items_per_block); - - std::vector input = benchmark_utils::get_random_data(size, T(0), T(10)); - T* d_input; - T* d_output; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMalloc(&d_output, size * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - - for(auto _ : state) + primbench::json meta() const override { - auto start = std::chrono::high_resolution_clock::now(); - - for(size_t i = 0; i < Trials; i++) - { - warp_load_kernel - <<>>(d_input, d_output); - } - HIP_CHECK(hipPeekAtLastError()) - HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + auto json = primbench::json{} + .add("data_type", primbench::name()) + .add("block_size", BlockSize) + .add("items_per_thread", ItemsPerThread) + .add("warp_size", LogicalWarpSize) + .add("algo", "warp_load") + .add("lvl", "warp") + .add("subalgo", getAlgorithmName(Algorithm)); + + return json; } - state.SetBytesProcessed(state.iterations() * Trials * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * Trials * size); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); -} + void run(primbench::state& state) override + { + const auto& input_items = state.size; + const auto& stream = state.stream; + + constexpr auto items_per_block = BlockSize * ItemsPerThread; + const auto items + = items_per_block * ((input_items + items_per_block - 1) / items_per_block); + + std::vector input = benchmark_utils::get_random_data(items, T(0), T(10)); + T* d_input; + T* d_output; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK(hipMalloc(&d_output, items * sizeof(T))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); + + state.set_items(items); + state.add_writes(items); + + state.run( + [&] + { + warp_load_kernel + <<>>(d_input, + d_output); + }); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); + } +}; -#define CREATE_BENCHMARK(T, BS, IT, WS, ALG) \ - benchmark::RegisterBenchmark(std::string("warp_load.") \ - .c_str(), \ - &run_benchmark, \ - stream, \ - size) +#define CREATE_BENCHMARK(T, BS, IT, WS, ALG) \ + executor.queue>() int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - - std::cout << "benchmark_warp_load" << std::endl; - std::cout << "[HIP] Device name: " << devProp.name << std::endl; + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; + + primbench::executor executor(argc, argv, settings); // Add benchmarks - std::vector benchmarks{ - CREATE_BENCHMARK(int, 256, 4, 32, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(int, 256, 4, 32, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(int, 256, 4, 32, ::hipcub::WARP_LOAD_VECTORIZE), - CREATE_BENCHMARK(int, 256, 4, 32, ::hipcub::WARP_LOAD_TRANSPOSE), - CREATE_BENCHMARK(int, 256, 8, 32, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(int, 256, 8, 32, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(int, 256, 8, 32, ::hipcub::WARP_LOAD_VECTORIZE), - CREATE_BENCHMARK(int, 256, 8, 32, ::hipcub::WARP_LOAD_TRANSPOSE), - CREATE_BENCHMARK(int, 256, 16, 32, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(int, 256, 16, 32, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(int, 256, 16, 32, ::hipcub::WARP_LOAD_VECTORIZE), - CREATE_BENCHMARK(int, 256, 16, 32, ::hipcub::WARP_LOAD_TRANSPOSE), - CREATE_BENCHMARK(int, 256, 32, 32, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(int, 256, 32, 32, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(int, 256, 32, 32, ::hipcub::WARP_LOAD_VECTORIZE), - CREATE_BENCHMARK(int, 256, 32, 32, ::hipcub::WARP_LOAD_TRANSPOSE), - CREATE_BENCHMARK(int, 256, 64, 32, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(int, 256, 64, 32, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(int, 256, 64, 32, ::hipcub::WARP_LOAD_VECTORIZE), - CREATE_BENCHMARK(double, 256, 4, 32, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(double, 256, 4, 32, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(double, 256, 4, 32, ::hipcub::WARP_LOAD_VECTORIZE), - CREATE_BENCHMARK(double, 256, 4, 32, ::hipcub::WARP_LOAD_TRANSPOSE), - CREATE_BENCHMARK(double, 256, 8, 32, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(double, 256, 8, 32, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(double, 256, 8, 32, ::hipcub::WARP_LOAD_VECTORIZE), - CREATE_BENCHMARK(double, 256, 8, 32, ::hipcub::WARP_LOAD_TRANSPOSE), - CREATE_BENCHMARK(double, 256, 16, 32, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(double, 256, 16, 32, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(double, 256, 16, 32, ::hipcub::WARP_LOAD_VECTORIZE), - CREATE_BENCHMARK(double, 256, 16, 32, ::hipcub::WARP_LOAD_TRANSPOSE), - CREATE_BENCHMARK(double, 256, 32, 32, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(double, 256, 32, 32, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(double, 256, 32, 32, ::hipcub::WARP_LOAD_VECTORIZE), - // WARP_LOAD_TRANSPOSE removed because of shared memory limit - // CREATE_BENCHMARK(double, 256, 32, 32, ::hipcub::WARP_LOAD_TRANSPOSE), - CREATE_BENCHMARK(double, 256, 64, 32, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(double, 256, 64, 32, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(double, 256, 64, 32, ::hipcub::WARP_LOAD_VECTORIZE) - // WARP_LOAD_TRANSPOSE removed because of shared memory limit - // CREATE_BENCHMARK(double, 256, 64, 32, ::hipcub::WARP_LOAD_TRANSPOSE) - }; + CREATE_BENCHMARK(int, 256, 4, 32, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(int, 256, 4, 32, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(int, 256, 4, 32, ::hipcub::WARP_LOAD_VECTORIZE); + CREATE_BENCHMARK(int, 256, 4, 32, ::hipcub::WARP_LOAD_TRANSPOSE); + CREATE_BENCHMARK(int, 256, 8, 32, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(int, 256, 8, 32, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(int, 256, 8, 32, ::hipcub::WARP_LOAD_VECTORIZE); + CREATE_BENCHMARK(int, 256, 8, 32, ::hipcub::WARP_LOAD_TRANSPOSE); + CREATE_BENCHMARK(int, 256, 16, 32, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(int, 256, 16, 32, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(int, 256, 16, 32, ::hipcub::WARP_LOAD_VECTORIZE); + CREATE_BENCHMARK(int, 256, 16, 32, ::hipcub::WARP_LOAD_TRANSPOSE); + CREATE_BENCHMARK(int, 256, 32, 32, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(int, 256, 32, 32, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(int, 256, 32, 32, ::hipcub::WARP_LOAD_VECTORIZE); + CREATE_BENCHMARK(int, 256, 32, 32, ::hipcub::WARP_LOAD_TRANSPOSE); + CREATE_BENCHMARK(int, 256, 64, 32, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(int, 256, 64, 32, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(int, 256, 64, 32, ::hipcub::WARP_LOAD_VECTORIZE); + CREATE_BENCHMARK(double, 256, 4, 32, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(double, 256, 4, 32, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(double, 256, 4, 32, ::hipcub::WARP_LOAD_VECTORIZE); + CREATE_BENCHMARK(double, 256, 4, 32, ::hipcub::WARP_LOAD_TRANSPOSE); + CREATE_BENCHMARK(double, 256, 8, 32, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(double, 256, 8, 32, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(double, 256, 8, 32, ::hipcub::WARP_LOAD_VECTORIZE); + CREATE_BENCHMARK(double, 256, 8, 32, ::hipcub::WARP_LOAD_TRANSPOSE); + CREATE_BENCHMARK(double, 256, 16, 32, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(double, 256, 16, 32, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(double, 256, 16, 32, ::hipcub::WARP_LOAD_VECTORIZE); + CREATE_BENCHMARK(double, 256, 16, 32, ::hipcub::WARP_LOAD_TRANSPOSE); + CREATE_BENCHMARK(double, 256, 32, 32, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(double, 256, 32, 32, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(double, 256, 32, 32, ::hipcub::WARP_LOAD_VECTORIZE); + + // WARP_LOAD_TRANSPOSE removed because of shared memory limit + // CREATE_BENCHMARK(double, 256, 32, 32, ::hipcub::WARP_LOAD_TRANSPOSE); + + CREATE_BENCHMARK(double, 256, 64, 32, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(double, 256, 64, 32, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(double, 256, 64, 32, ::hipcub::WARP_LOAD_VECTORIZE); + + // WARP_LOAD_TRANSPOSE removed because of shared memory limit + // CREATE_BENCHMARK(double, 256, 64, 32, ::hipcub::WARP_LOAD_TRANSPOSE) if(::benchmark_utils::is_warp_size_supported(64)) { - std::vector additional_benchmarks{ - CREATE_BENCHMARK(int, 256, 4, 64, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(int, 256, 4, 64, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(int, 256, 4, 64, ::hipcub::WARP_LOAD_VECTORIZE), - CREATE_BENCHMARK(int, 256, 4, 64, ::hipcub::WARP_LOAD_TRANSPOSE), - CREATE_BENCHMARK(int, 256, 8, 64, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(int, 256, 8, 64, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(int, 256, 8, 64, ::hipcub::WARP_LOAD_VECTORIZE), - CREATE_BENCHMARK(int, 256, 8, 64, ::hipcub::WARP_LOAD_TRANSPOSE), - CREATE_BENCHMARK(int, 256, 16, 64, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(int, 256, 16, 64, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(int, 256, 16, 64, ::hipcub::WARP_LOAD_VECTORIZE), - CREATE_BENCHMARK(int, 256, 16, 64, ::hipcub::WARP_LOAD_TRANSPOSE), - CREATE_BENCHMARK(int, 256, 32, 64, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(int, 256, 32, 64, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(int, 256, 32, 64, ::hipcub::WARP_LOAD_VECTORIZE), - CREATE_BENCHMARK(int, 256, 32, 64, ::hipcub::WARP_LOAD_TRANSPOSE), - CREATE_BENCHMARK(int, 256, 64, 64, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(int, 256, 64, 64, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(int, 256, 64, 64, ::hipcub::WARP_LOAD_VECTORIZE), - CREATE_BENCHMARK(double, 256, 4, 64, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(double, 256, 4, 64, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(double, 256, 4, 64, ::hipcub::WARP_LOAD_VECTORIZE), - CREATE_BENCHMARK(double, 256, 4, 64, ::hipcub::WARP_LOAD_TRANSPOSE), - CREATE_BENCHMARK(double, 256, 8, 64, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(double, 256, 8, 64, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(double, 256, 8, 64, ::hipcub::WARP_LOAD_VECTORIZE), - CREATE_BENCHMARK(double, 256, 8, 64, ::hipcub::WARP_LOAD_TRANSPOSE), - CREATE_BENCHMARK(double, 256, 16, 64, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(double, 256, 16, 64, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(double, 256, 16, 64, ::hipcub::WARP_LOAD_VECTORIZE), - // WARP_LOAD_TRANSPOSE removed because of shared memory limit - // CREATE_BENCHMARK(double, 256, 16, 64, ::hipcub::WARP_LOAD_TRANSPOSE), - CREATE_BENCHMARK(double, 256, 32, 64, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(double, 256, 32, 64, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(double, 256, 32, 64, ::hipcub::WARP_LOAD_VECTORIZE), - // WARP_LOAD_TRANSPOSE removed because of shared memory limit - // CREATE_BENCHMARK(double, 256, 32, 64, ::hipcub::WARP_LOAD_TRANSPOSE), - CREATE_BENCHMARK(double, 256, 64, 64, ::hipcub::WARP_LOAD_DIRECT), - CREATE_BENCHMARK(double, 256, 64, 64, ::hipcub::WARP_LOAD_STRIPED), - CREATE_BENCHMARK(double, 256, 64, 64, ::hipcub::WARP_LOAD_VECTORIZE) - // WARP_LOAD_TRANSPOSE removed because of shared memory limit - // CREATE_BENCHMARK(double, 256, 64, 64, ::hipcub::WARP_LOAD_TRANSPOSE) - }; - benchmarks.insert(benchmarks.end(), - additional_benchmarks.begin(), - additional_benchmarks.end()); - } + CREATE_BENCHMARK(int, 256, 4, 64, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(int, 256, 4, 64, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(int, 256, 4, 64, ::hipcub::WARP_LOAD_VECTORIZE); + CREATE_BENCHMARK(int, 256, 4, 64, ::hipcub::WARP_LOAD_TRANSPOSE); + CREATE_BENCHMARK(int, 256, 8, 64, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(int, 256, 8, 64, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(int, 256, 8, 64, ::hipcub::WARP_LOAD_VECTORIZE); + CREATE_BENCHMARK(int, 256, 8, 64, ::hipcub::WARP_LOAD_TRANSPOSE); + CREATE_BENCHMARK(int, 256, 16, 64, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(int, 256, 16, 64, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(int, 256, 16, 64, ::hipcub::WARP_LOAD_VECTORIZE); + CREATE_BENCHMARK(int, 256, 16, 64, ::hipcub::WARP_LOAD_TRANSPOSE); + CREATE_BENCHMARK(int, 256, 32, 64, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(int, 256, 32, 64, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(int, 256, 32, 64, ::hipcub::WARP_LOAD_VECTORIZE); + CREATE_BENCHMARK(int, 256, 32, 64, ::hipcub::WARP_LOAD_TRANSPOSE); + CREATE_BENCHMARK(int, 256, 64, 64, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(int, 256, 64, 64, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(int, 256, 64, 64, ::hipcub::WARP_LOAD_VECTORIZE); + CREATE_BENCHMARK(double, 256, 4, 64, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(double, 256, 4, 64, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(double, 256, 4, 64, ::hipcub::WARP_LOAD_VECTORIZE); + CREATE_BENCHMARK(double, 256, 4, 64, ::hipcub::WARP_LOAD_TRANSPOSE); + CREATE_BENCHMARK(double, 256, 8, 64, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(double, 256, 8, 64, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(double, 256, 8, 64, ::hipcub::WARP_LOAD_VECTORIZE); + CREATE_BENCHMARK(double, 256, 8, 64, ::hipcub::WARP_LOAD_TRANSPOSE); + CREATE_BENCHMARK(double, 256, 16, 64, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(double, 256, 16, 64, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(double, 256, 16, 64, ::hipcub::WARP_LOAD_VECTORIZE); - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + // WARP_LOAD_TRANSPOSE removed because of shared memory limit + // CREATE_BENCHMARK(double, 256, 16, 64, ::hipcub::WARP_LOAD_TRANSPOSE); - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } + CREATE_BENCHMARK(double, 256, 32, 64, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(double, 256, 32, 64, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(double, 256, 32, 64, ::hipcub::WARP_LOAD_VECTORIZE); + + // WARP_LOAD_TRANSPOSE removed because of shared memory limit + // CREATE_BENCHMARK(double, 256, 32, 64, ::hipcub::WARP_LOAD_TRANSPOSE); + + CREATE_BENCHMARK(double, 256, 64, 64, ::hipcub::WARP_LOAD_DIRECT); + CREATE_BENCHMARK(double, 256, 64, 64, ::hipcub::WARP_LOAD_STRIPED); + CREATE_BENCHMARK(double, 256, 64, 64, ::hipcub::WARP_LOAD_VECTORIZE); + + // WARP_LOAD_TRANSPOSE removed because of shared memory limit + // CREATE_BENCHMARK(double, 256, 64, 64, ::hipcub::WARP_LOAD_TRANSPOSE); } // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_warp_merge_sort.cpp b/projects/hipcub/benchmark/benchmark_warp_merge_sort.cpp index f6d91fe7f9a2..0d3cff4b99b8 100644 --- a/projects/hipcub/benchmark/benchmark_warp_merge_sort.cpp +++ b/projects/hipcub/benchmark/benchmark_warp_merge_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,10 +20,10 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" #include "../test/hipcub/test_utils_sort_comparator.hpp" -// HIP API + #include #include #include @@ -31,10 +31,6 @@ #include -#ifndef DEFAULT_N -constexpr size_t DEFAULT_N = 1024 * 1024 * 128; -#endif - enum class benchmark_kinds { sort_keys, @@ -46,7 +42,8 @@ template -__device__ auto sort_keys_benchmark(const T* input, T* output, Compare compare_op) +__device__ +auto sort_keys_benchmark(const T* input, T* output, Compare compare_op) -> std::enable_if_t> { constexpr unsigned int items_per_block = BlockSize * ItemsPerThread; @@ -73,7 +70,8 @@ template -__device__ auto sort_keys_benchmark(const T* /*input*/, T* /*output*/, Compare /*compare_op*/) +__device__ +auto sort_keys_benchmark(const T* /*input*/, T* /*output*/, Compare /*compare_op*/) -> std::enable_if_t> {} @@ -82,8 +80,8 @@ template -__global__ - __launch_bounds__(BlockSize) void sort_keys(const T* input, T* output, Compare compare_op) +__global__ __launch_bounds__(BlockSize) +void sort_keys(const T* input, T* output, Compare compare_op) { sort_keys_benchmark(input, output, compare_op); } @@ -93,7 +91,8 @@ template -__device__ auto sort_pairs_benchmark(const T* input, T* output, Compare compare_op) +__device__ +auto sort_pairs_benchmark(const T* input, T* output, Compare compare_op) -> std::enable_if_t> { constexpr unsigned int items_per_block = BlockSize * ItemsPerThread; @@ -131,7 +130,8 @@ template -__device__ auto sort_pairs_benchmark(const T* /*input*/, T* /*output*/, Compare /*compare_op*/) +__device__ +auto sort_pairs_benchmark(const T* /*input*/, T* /*output*/, Compare /*compare_op*/) -> std::enable_if_t> {} @@ -140,8 +140,8 @@ template -__global__ - __launch_bounds__(BlockSize) void sort_pairs(const T* input, T* output, Compare compare_op) +__global__ __launch_bounds__(BlockSize) +void sort_pairs(const T* input, T* output, Compare compare_op) { sort_pairs_benchmark(input, output, compare_op); } @@ -149,7 +149,7 @@ __global__ template struct max_value { - static constexpr T value = std::numeric_limits::max(); + static constexpr T value = _HIPCUB_STD::numeric_limits::max(); }; template -__device__ auto sort_keys_segmented_benchmark(const T* input, - T* output, - const unsigned int* segment_sizes, - Compare compare) +__device__ +auto sort_keys_segmented_benchmark(const T* input, + T* output, + const unsigned int* segment_sizes, + Compare compare) -> std::enable_if_t> { constexpr unsigned int max_segment_size = LogicalWarpSize * ItemsPerThread; @@ -192,10 +193,11 @@ template -__device__ auto sort_keys_segmented_benchmark(const T* /*input*/, - T* /*output*/, - const unsigned int* /*segment_sizes*/, - Compare /*compare*/) +__device__ +auto sort_keys_segmented_benchmark(const T* /*input*/, + T* /*output*/, + const unsigned int* /*segment_sizes*/, + Compare /*compare*/) -> std::enable_if_t> {} @@ -204,10 +206,11 @@ template -__global__ __launch_bounds__(BlockSize) void sort_keys_segmented(const T* input, - T* output, - const unsigned int* segment_sizes, - Compare compare) +__global__ __launch_bounds__(BlockSize) +void sort_keys_segmented(const T* input, + T* output, + const unsigned int* segment_sizes, + Compare compare) { sort_keys_segmented_benchmark(input, output, @@ -220,10 +223,11 @@ template -__device__ auto sort_pairs_segmented_benchmark(const T* input, - T* output, - const unsigned int* segment_sizes, - Compare compare) +__device__ +auto sort_pairs_segmented_benchmark(const T* input, + T* output, + const unsigned int* segment_sizes, + Compare compare) -> std::enable_if_t> { constexpr unsigned int max_segment_size = LogicalWarpSize * ItemsPerThread; @@ -272,10 +276,11 @@ template -__device__ auto sort_pairs_segmented_benchmark(const T* /*input*/, - T* /*output*/, - const unsigned int* /*segment_sizes*/, - Compare /*compare*/) +__device__ +auto sort_pairs_segmented_benchmark(const T* /*input*/, + T* /*output*/, + const unsigned int* /*segment_sizes*/, + Compare /*compare*/) -> std::enable_if_t> {} @@ -284,10 +289,11 @@ template -__global__ __launch_bounds__(BlockSize) void sort_pairs_segmented(const T* input, - T* output, - const unsigned int* segment_sizes, - Compare compare) +__global__ __launch_bounds__(BlockSize) +void sort_pairs_segmented(const T* input, + T* output, + const unsigned int* segment_sizes, + Compare compare) { sort_pairs_segmented_benchmark(input, output, @@ -296,208 +302,209 @@ __global__ __launch_bounds__(BlockSize) void sort_pairs_segmented(const T* } template -void run_benchmark(benchmark::State& state, - const benchmark_kinds benchmark_kind, - const hipStream_t stream, - const size_t N) + unsigned int BlockSize, + unsigned int LogicalWarpSize, + unsigned int ItemsPerThread, + benchmark_kinds BenchmarkKind, + class CompareOp = test_utils::less> +struct sort_benchmark : public primbench::benchmark_interface { - constexpr auto items_per_block = BlockSize * ItemsPerThread; - const auto size = items_per_block * ((N + items_per_block - 1) / items_per_block); + primbench::json meta() const override + { + auto json = primbench::json{} + .add("algo", "warp_merge_sort") + .add("segmented", false) + .add("pairs", BenchmarkKind == benchmark_kinds::sort_pairs) + .add("data_type", primbench::name()) + .add("block_size", BlockSize) + .add("items_per_thread", ItemsPerThread) + .add("warp_size", LogicalWarpSize); + + return json; + } - const std::vector input - = benchmark_utils::get_random_data(size, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); + void run(primbench::state& state) override + { + const auto& input_items = state.size; + const auto& stream = state.stream; - T* d_input = nullptr; - T* d_output = nullptr; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(input[0]))); - HIP_CHECK(hipMalloc(&d_output, size * sizeof(input[0]))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); + constexpr auto items_per_block = BlockSize * ItemsPerThread; + const auto items + = items_per_block * ((input_items + items_per_block - 1) / items_per_block); - for(auto _ : state) - { - auto start = std::chrono::high_resolution_clock::now(); + const std::vector input + = benchmark_utils::get_random_data(items, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); - if(benchmark_kind == benchmark_kinds::sort_keys) - { - for(unsigned int i = 0; i < Trials; ++i) - { - sort_keys - <<>>(d_input, - d_output, - CompareOp{}); - } - } else if(benchmark_kind == benchmark_kinds::sort_pairs) - { - for(unsigned int i = 0; i < Trials; ++i) - { - sort_pairs - <<>>(d_input, - d_output, - CompareOp{}); - } - } - HIP_CHECK(hipPeekAtLastError()); - HIP_CHECK(hipDeviceSynchronize()); + T* d_input = nullptr; + T* d_output = nullptr; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(input[0]))); + HIP_CHECK(hipMalloc(&d_output, items * sizeof(input[0]))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * Trials * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * Trials * size); + state.set_items(items); + state.add_writes(items); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); -} + state.run( + [&] + { + if constexpr(BenchmarkKind == benchmark_kinds::sort_keys) + { + sort_keys + <<>>( + d_input, + d_output, + CompareOp{}); + } + else + { + static_assert(BenchmarkKind == benchmark_kinds::sort_pairs); + sort_pairs + <<>>( + d_input, + d_output, + CompareOp{}); + } + }); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); + } +}; template -void run_segmented_benchmark(benchmark::State& state, - const benchmark_kinds benchmark_kind, - const hipStream_t stream, - const size_t N) + unsigned int BlockSize, + unsigned int LogicalWarpSize, + unsigned int ItemsPerThread, + benchmark_kinds BenchmarkKind, + class CompareOp = test_utils::less> +struct segmented_sort_benchmark : public primbench::benchmark_interface { - constexpr auto max_segment_size = LogicalWarpSize * ItemsPerThread; - constexpr auto segments_per_block = BlockSize / LogicalWarpSize; - constexpr auto items_per_block = BlockSize * ItemsPerThread; - - const auto num_blocks = (N + items_per_block - 1) / items_per_block; - const auto num_segments = num_blocks * segments_per_block; - const auto size = num_blocks * items_per_block; - - const std::vector input - = benchmark_utils::get_random_data(size, - benchmark_utils::generate_limits::min(), - benchmark_utils::generate_limits::max()); - - const auto segment_sizes - = benchmark_utils::get_random_data(num_segments, 0, max_segment_size); - - T* d_input = nullptr; - T* d_output = nullptr; - unsigned int* d_segment_sizes = nullptr; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(input[0]))); - HIP_CHECK(hipMalloc(&d_output, size * sizeof(input[0]))); - HIP_CHECK(hipMalloc(&d_segment_sizes, num_segments * sizeof(segment_sizes[0]))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_segment_sizes, - segment_sizes.data(), - num_segments * sizeof(segment_sizes[0]), - hipMemcpyHostToDevice)); - - for(auto _ : state) + + primbench::json meta() const override { - auto start = std::chrono::high_resolution_clock::now(); + auto json = primbench::json{} + .add("algo", "warp_merge_sort") + .add("segmented", true) + .add("pairs", BenchmarkKind == benchmark_kinds::sort_pairs) + .add("data_type", primbench::name()) + .add("block_size", BlockSize) + .add("items_per_thread", ItemsPerThread) + .add("warp_size", LogicalWarpSize); + + return json; + } - if(benchmark_kind == benchmark_kinds::sort_keys) - { - for(unsigned int i = 0; i < Trials; ++i) - { - sort_keys_segmented - <<>>(d_input, - d_output, - d_segment_sizes, - CompareOp{}); - } - } else if(benchmark_kind == benchmark_kinds::sort_pairs) - { - for(unsigned int i = 0; i < Trials; ++i) + void run(primbench::state& state) override + { + const auto& input_items = state.size; + const auto& stream = state.stream; + + constexpr auto max_segment_size = LogicalWarpSize * ItemsPerThread; + constexpr auto segments_per_block = BlockSize / LogicalWarpSize; + constexpr auto items_per_block = BlockSize * ItemsPerThread; + + const auto num_blocks = (input_items + items_per_block - 1) / items_per_block; + const auto num_segments = num_blocks * segments_per_block; + const auto items = num_blocks * items_per_block; + + const std::vector input + = benchmark_utils::get_random_data(items, + benchmark_utils::generate_limits::min(), + benchmark_utils::generate_limits::max()); + + const auto segment_sizes + = benchmark_utils::get_random_data(num_segments, 0, max_segment_size); + + T* d_input = nullptr; + T* d_output = nullptr; + unsigned int* d_segment_sizes = nullptr; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(input[0]))); + HIP_CHECK(hipMalloc(&d_output, items * sizeof(input[0]))); + HIP_CHECK(hipMalloc(&d_segment_sizes, num_segments * sizeof(segment_sizes[0]))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(d_segment_sizes, + segment_sizes.data(), + num_segments * sizeof(segment_sizes[0]), + hipMemcpyHostToDevice)); + + state.set_items(items); + state.add_writes(items); + + state.run( + [&] { - sort_pairs_segmented - <<>>(d_input, - d_output, - d_segment_sizes, - CompareOp{}); - } - } - HIP_CHECK(hipPeekAtLastError()); - HIP_CHECK(hipDeviceSynchronize()); - - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + if constexpr(BenchmarkKind == benchmark_kinds::sort_keys) + { + sort_keys_segmented + <<>>(d_input, + d_output, + d_segment_sizes, + CompareOp{}); + } + else + { + static_assert(BenchmarkKind == benchmark_kinds::sort_pairs); + + sort_pairs_segmented + <<>>(d_input, + d_output, + d_segment_sizes, + CompareOp{}); + } + }); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); + HIP_CHECK(hipFree(d_segment_sizes)); } - state.SetBytesProcessed(state.iterations() * Trials * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * Trials * size); - - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_segment_sizes)); -} +}; -#define CREATE_BENCHMARK(T, BS, WS, IPT) \ - if(WS <= device_warp_size) \ - { \ - benchmarks.push_back(benchmark::RegisterBenchmark( \ - std::string("warp_merge_sort.sub_algorithm_name:" \ - + name) \ - .c_str(), \ - segmented ? &run_benchmark : &run_segmented_benchmark, \ - benchmark_kind, \ - stream, \ - size)); \ +#define CREATE_BENCHMARK(T, BS, WS, IPT, BK) \ + if(WS <= device_warp_size) \ + { \ + if(segmented) \ + { \ + executor.queue>(); \ + } \ + else \ + { \ + executor.queue>(); \ + } \ } -#define BENCHMARK_TYPE_WS(type, block, warp) \ - CREATE_BENCHMARK(type, block, warp, 1); \ - CREATE_BENCHMARK(type, block, warp, 4); \ - CREATE_BENCHMARK(type, block, warp, 8) - -#define BENCHMARK_TYPE(type, block) \ - BENCHMARK_TYPE_WS(type, block, 4); \ - BENCHMARK_TYPE_WS(type, block, 16); \ - BENCHMARK_TYPE_WS(type, block, 32); \ - BENCHMARK_TYPE_WS(type, block, 64) - -void add_benchmarks(const benchmark_kinds benchmark_kind, - const std::string& name, - std::vector& benchmarks, - const hipStream_t stream, - const size_t size, - const bool segmented, - const unsigned int device_warp_size) +#define BENCHMARK_TYPE_WS(type, block, warp, kind) \ + CREATE_BENCHMARK(type, block, warp, 1, kind); \ + CREATE_BENCHMARK(type, block, warp, 4, kind); \ + CREATE_BENCHMARK(type, block, warp, 8, kind) + +#define BENCHMARK_TYPE(type, block) \ + BENCHMARK_TYPE_WS(type, block, 4, BenchmarkKind); \ + BENCHMARK_TYPE_WS(type, block, 16, BenchmarkKind); \ + BENCHMARK_TYPE_WS(type, block, 32, BenchmarkKind); \ + BENCHMARK_TYPE_WS(type, block, 64, BenchmarkKind) + +template +void add_benchmarks(primbench::executor& executor, + const bool segmented, + const unsigned int device_warp_size) { BENCHMARK_TYPE(int, 256); BENCHMARK_TYPE(int8_t, 256); BENCHMARK_TYPE(uint8_t, 256); - BENCHMARK_TYPE(long long, 256); + BENCHMARK_TYPE(int64_t, 256); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_warp_merge_sort" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; + primbench::settings settings; + settings.size = 128 * primbench::MiB; + settings.min_gpu_ms_per_batch = 1000; + settings.batch_window_size = 3; + settings.noise_tolerance_percent = 3; + + primbench::executor executor(argc, argv, settings); const auto device_warp_size = [] { @@ -505,7 +512,8 @@ int main(int argc, char* argv[]) if(result > 0) { std::cout << "[HIP] Device warp size: " << result << std::endl; - } else + } + else { std::cerr << "Failed to get device warp size! Aborting.\n"; std::exit(1); @@ -513,54 +521,10 @@ int main(int argc, char* argv[]) return static_cast(result); }(); - // Add benchmarks - std::vector benchmarks; - add_benchmarks(benchmark_kinds::sort_keys, - "sort(keys)", - benchmarks, - stream, - size, - false, - device_warp_size); - add_benchmarks(benchmark_kinds::sort_pairs, - "sort(keys, values)", - benchmarks, - stream, - size, - false, - device_warp_size); - add_benchmarks(benchmark_kinds::sort_keys, - "segmented_sort(keys)", - benchmarks, - stream, - size, - true, - device_warp_size); - add_benchmarks(benchmark_kinds::sort_pairs, - "segmented_sort(keys, values)", - benchmarks, - stream, - size, - true, - device_warp_size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } - - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + add_benchmarks(executor, false, device_warp_size); + add_benchmarks(executor, false, device_warp_size); + add_benchmarks(executor, true, device_warp_size); + add_benchmarks(executor, true, device_warp_size); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_warp_reduce.cpp b/projects/hipcub/benchmark/benchmark_warp_reduce.cpp index 66e80917ab74..68ebc1f249b7 100644 --- a/projects/hipcub/benchmark/benchmark_warp_reduce.cpp +++ b/projects/hipcub/benchmark/benchmark_warp_reduce.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,17 +20,15 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif +constexpr unsigned int Trials = 100; -template -__device__ auto warp_reduce_benchmark(const T* d_input, T* d_output) +template +__device__ +auto warp_reduce_benchmark_fn(const T* d_input, T* d_output) -> std::enable_if_t> { const unsigned int i = hipBlockIdx_x * hipBlockDim_x + hipThreadIdx_x; @@ -39,8 +37,8 @@ __device__ auto warp_reduce_benchmark(const T* d_input, T* d_output) using wreduce_t = hipcub::WarpReduce; __shared__ typename wreduce_t::TempStorage storage; - auto reduce_op = hipcub::Sum(); -#pragma nounroll + auto reduce_op = benchmark_utils::plus{}; + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { value = wreduce_t(storage).Reduce(value, reduce_op); @@ -49,20 +47,22 @@ __device__ auto warp_reduce_benchmark(const T* d_input, T* d_output) d_output[i] = value; } -template -__device__ auto warp_reduce_benchmark(const T* /*d_input*/, T* /*d_output*/) +template +__device__ +auto warp_reduce_benchmark_fn(const T* /*d_input*/, T* /*d_output*/) -> std::enable_if_t> {} -template +template __global__ __launch_bounds__(64) void warp_reduce_kernel(const T* d_input, T* d_output) { - warp_reduce_benchmark(d_input, d_output); + warp_reduce_benchmark_fn(d_input, d_output); } -template -__device__ auto segmented_warp_reduce_benchmark(const T* d_input, Flag* d_flags, T* d_output) +template +__device__ +auto segmented_warp_reduce_benchmark_fn(const T* d_input, Flag* d_flags, T* d_output) -> std::enable_if_t> { const unsigned int i = hipBlockIdx_x * hipBlockDim_x + hipThreadIdx_x; @@ -72,7 +72,7 @@ __device__ auto segmented_warp_reduce_benchmark(const T* d_input, Flag* d_flags, using wreduce_t = hipcub::WarpReduce; __shared__ typename wreduce_t::TempStorage storage; -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { value = wreduce_t(storage).HeadSegmentedSum(value, flag); @@ -81,30 +81,26 @@ __device__ auto segmented_warp_reduce_benchmark(const T* d_input, Flag* d_flags, d_output[i] = value; } -template -__device__ auto segmented_warp_reduce_benchmark(const T* /*d_input*/, Flag* /*d_flags*/, T* /*d_output*/) +template +__device__ +auto segmented_warp_reduce_benchmark_fn(const T* /*d_input*/, Flag* /*d_flags*/, T* /*d_output*/) -> std::enable_if_t> {} -template +template __global__ __launch_bounds__(64) -void segmented_warp_reduce_kernel(const T* d_input, Flag* d_flags, T* d_output) +void segmented_warp_reduce_kernel_fn(const T* d_input, Flag* d_flags, T* d_output) { - segmented_warp_reduce_benchmark(d_input, d_flags, d_output); + segmented_warp_reduce_benchmark_fn(d_input, d_flags, d_output); } -template +template inline auto execute_warp_reduce_kernel( - T* input, T* output, Flag* /* flags */, size_t size, hipStream_t stream) -> + T* input, T* output, Flag* /* flags */, size_t items, hipStream_t stream) -> typename std::enable_if::type { - hipLaunchKernelGGL(HIP_KERNEL_NAME(warp_reduce_kernel), - dim3(size / BlockSize), + hipLaunchKernelGGL(HIP_KERNEL_NAME(warp_reduce_kernel), + dim3(items / BlockSize), dim3(BlockSize), 0, stream, @@ -113,18 +109,13 @@ inline auto execute_warp_reduce_kernel( HIP_CHECK(hipPeekAtLastError()); } -template -inline auto - execute_warp_reduce_kernel(T* input, T* output, Flag* flags, size_t size, hipStream_t stream) -> +template +inline auto execute_warp_reduce_kernel( + T* input, T* output, Flag* flags, size_t items, hipStream_t stream) -> typename std::enable_if::type { - hipLaunchKernelGGL(HIP_KERNEL_NAME(segmented_warp_reduce_kernel), - dim3(size / BlockSize), + hipLaunchKernelGGL(HIP_KERNEL_NAME(segmented_warp_reduce_kernel_fn), + dim3(items / BlockSize), dim3(BlockSize), 0, stream, @@ -134,148 +125,118 @@ inline auto HIP_CHECK(hipPeekAtLastError()); } -template -void run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) +template +class warp_reduce_benchmark : public primbench::benchmark_interface { - using flag_type = unsigned char; - - const auto size = BlockSize * ((N + BlockSize - 1) / BlockSize); - - std::vector input = benchmark_utils::get_random_data(size, T(0), T(10)); - std::vector flags = benchmark_utils::get_random_data(size, 0, 1); - T* d_input; - flag_type* d_flags; - T* d_output; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMalloc(&d_flags, size * sizeof(flag_type))); - HIP_CHECK(hipMalloc(&d_output, size * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(d_flags, flags.data(), size * sizeof(flag_type), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - for(auto _ : state) + primbench::json meta() const override + { + auto json = primbench::json{} + .add("algo", "warp_reduce") + .add("segmented", Segmented) + .add("warp_size", WarpSize) + .add("block_size", BlockSize) + .add("data_type", primbench::name()); + + return json; + } + + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); - execute_warp_reduce_kernel(d_input, + using flag_type = unsigned char; + + const auto& input_size = state.size; + const auto& stream = state.stream; + + const auto items = BlockSize * ((input_size + BlockSize - 1) / BlockSize); + + std::vector input = benchmark_utils::get_random_data(items, T(0), T(10)); + std::vector flags = benchmark_utils::get_random_data(items, 0, 1); + T* d_input; + flag_type* d_flags; + T* d_output; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK(hipMalloc(&d_flags, items * sizeof(flag_type))); + HIP_CHECK(hipMalloc(&d_output, items * sizeof(T))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK( + hipMemcpy(d_flags, flags.data(), items * sizeof(flag_type), hipMemcpyHostToDevice)); + HIP_CHECK(hipDeviceSynchronize()); + + state.set_items(Trials * items); + state.add_writes(Trials * items); + + state.run( + [&] + { + execute_warp_reduce_kernel(d_input, d_output, d_flags, - size, + items, stream); - HIP_CHECK(hipDeviceSynchronize()); + }); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); + HIP_CHECK(hipFree(d_flags)); } - state.SetBytesProcessed(state.iterations() * Trials * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * Trials * size); +}; - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); - HIP_CHECK(hipFree(d_flags)); -} - -#define CREATE_BENCHMARK(T, WS, BS) \ - benchmark::RegisterBenchmark(std::string("warp_reduce.sub_algorithm_name:" \ - + name) \ - .c_str(), \ - &run_benchmark, \ - stream, \ - size) +#define CREATE_BENCHMARK(T, WS, BS) executor.queue>() // If warp size limit is 16 -#define BENCHMARK_TYPE_WS16(type) CREATE_BENCHMARK(type, 15, 32), CREATE_BENCHMARK(type, 16, 32) +#define BENCHMARK_TYPE_WS16(type) \ + CREATE_BENCHMARK(type, 15, 32); \ + CREATE_BENCHMARK(type, 16, 32); // If warp size limit is 32 -#define BENCHMARK_TYPE_WS32(type) \ - BENCHMARK_TYPE_WS16(type), CREATE_BENCHMARK(type, 31, 32), CREATE_BENCHMARK(type, 32, 32), \ - CREATE_BENCHMARK(type, 32, 64) +#define BENCHMARK_TYPE_WS32(type) \ + BENCHMARK_TYPE_WS16(type); \ + CREATE_BENCHMARK(type, 31, 32); \ + CREATE_BENCHMARK(type, 32, 32); \ + CREATE_BENCHMARK(type, 32, 64); // If warp size limit is 64 -#define BENCHMARK_TYPE_WS64(type) \ - BENCHMARK_TYPE_WS32(type), CREATE_BENCHMARK(type, 37, 64), CREATE_BENCHMARK(type, 61, 64), \ - CREATE_BENCHMARK(type, 64, 64) +#define BENCHMARK_TYPE_WS64(type) \ + BENCHMARK_TYPE_WS32(type); \ + CREATE_BENCHMARK(type, 37, 64); \ + CREATE_BENCHMARK(type, 61, 64); \ + CREATE_BENCHMARK(type, 64, 64); template -void add_benchmarks(const std::string& name, - std::vector& benchmarks, - hipStream_t stream, - size_t size) +void add_benchmarks(primbench::executor& executor) { - std::vector bs = { #if HIPCUB_WARP_THREADS_MACRO == 16 - BENCHMARK_TYPE_WS16(int), - BENCHMARK_TYPE_WS16(float), - BENCHMARK_TYPE_WS16(double), - BENCHMARK_TYPE_WS16(int8_t), - BENCHMARK_TYPE_WS16(uint8_t) + BENCHMARK_TYPE_WS16(int); + BENCHMARK_TYPE_WS16(float); + BENCHMARK_TYPE_WS16(double); + BENCHMARK_TYPE_WS16(int8_t); + BENCHMARK_TYPE_WS16(uint8_t); #elif HIPCUB_WARP_THREADS_MACRO == 32 - BENCHMARK_TYPE_WS32(int), - BENCHMARK_TYPE_WS32(float), - BENCHMARK_TYPE_WS32(double), - BENCHMARK_TYPE_WS32(int8_t), - BENCHMARK_TYPE_WS32(uint8_t) + BENCHMARK_TYPE_WS32(int); + BENCHMARK_TYPE_WS32(float); + BENCHMARK_TYPE_WS32(double); + BENCHMARK_TYPE_WS32(int8_t); + BENCHMARK_TYPE_WS32(uint8_t); #else - BENCHMARK_TYPE_WS64(int), - BENCHMARK_TYPE_WS64(float), - BENCHMARK_TYPE_WS64(double), - BENCHMARK_TYPE_WS64(int8_t), - BENCHMARK_TYPE_WS64(uint8_t) + BENCHMARK_TYPE_WS64(int); + BENCHMARK_TYPE_WS64(float); + BENCHMARK_TYPE_WS64(double); + BENCHMARK_TYPE_WS64(int8_t); + BENCHMARK_TYPE_WS64(uint8_t); #endif - }; - benchmarks.insert(benchmarks.end(), bs.begin(), bs.end()); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_warp_reduce" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; - // Add benchmarks - std::vector benchmarks; - add_benchmarks("reduce", benchmarks, stream, size); - add_benchmarks("segmented_reduce", benchmarks, stream, size); + primbench::executor executor(argc, argv, settings); - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } - - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + // Add benchmarks + add_benchmarks(executor); + add_benchmarks(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_warp_scan.cpp b/projects/hipcub/benchmark/benchmark_warp_scan.cpp index db3fe941f6cb..fc26dc3aa95a 100644 --- a/projects/hipcub/benchmark/benchmark_warp_scan.cpp +++ b/projects/hipcub/benchmark/benchmark_warp_scan.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2020-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2020-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,14 +20,11 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif +constexpr unsigned int Trials = 100; enum class scan_type { @@ -36,15 +33,18 @@ enum class scan_type broadcast }; -template -__global__ __launch_bounds__(BlockSize) void kernel(const T* input, T* output, const T init) +template +__global__ __launch_bounds__(BlockSize) +void kernel(const T* input, T* output, const T init) { - Runner::template run(input, output, init); + Runner::template run(input, output, init); } struct inclusive_scan { - template + static constexpr const char* name = "inclusive_scan"; + + template __device__ static auto run(const T* input, T* output, const T init) -> std::enable_if_t> @@ -56,8 +56,8 @@ struct inclusive_scan using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage; - auto scan_op = hipcub::Sum(); -#pragma nounroll + auto scan_op = benchmark_utils::plus{}; + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { wscan_t(storage).InclusiveScan(value, value, scan_op); @@ -66,7 +66,7 @@ struct inclusive_scan output[i] = value; } - template + template __device__ static auto run(const T* /*input*/, T* /*output*/, const T /*init*/) -> std::enable_if_t> @@ -75,7 +75,9 @@ struct inclusive_scan struct exclusive_scan { - template + static constexpr const char* name = "exclusive_scan"; + + template __device__ static auto run(const T* input, T* output, const T init) -> std::enable_if_t> @@ -85,8 +87,8 @@ struct exclusive_scan using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage; - auto scan_op = hipcub::Sum(); -#pragma nounroll + auto scan_op = benchmark_utils::plus{}; + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { wscan_t(storage).ExclusiveScan(value, value, init, scan_op); @@ -94,7 +96,7 @@ struct exclusive_scan output[i] = value; } - template + template __device__ static auto run(const T* /*input*/, T* /*output*/, const T /*init*/) -> std::enable_if_t> @@ -103,7 +105,9 @@ struct exclusive_scan struct broadcast { - template + static constexpr const char* name = "broadcast"; + + template __device__ static auto run(const T* input, T* output, const T init) -> std::enable_if_t<(benchmark_utils::device_test_enabled_for_warp_size_v @@ -118,7 +122,7 @@ struct broadcast using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage; -#pragma nounroll + _CCCL_PRAGMA_NOUNROLL() for(unsigned int trial = 0; trial < Trials; trial++) { value = wscan_t(storage).Broadcast(value, src_lane); @@ -127,7 +131,7 @@ struct broadcast output[i] = value; } - template + template __device__ static auto run(const T* /*input*/, T* /*output*/, const T /*init*/) -> std::enable_if_t @@ -135,63 +139,61 @@ struct broadcast {} }; -template -void run_benchmark(benchmark::State& state, hipStream_t stream, size_t size) +template +class warp_scan_benchmark : public primbench::benchmark_interface { - // Make sure size is a multiple of BlockSize - size = BlockSize * ((size + BlockSize - 1) / BlockSize); - // Allocate and fill memory - std::vector input(size, 1.0f); - T* d_input; - T* d_output; - HIP_CHECK(hipMalloc(&d_input, size * sizeof(T))); - HIP_CHECK(hipMalloc(&d_output, size * sizeof(T))); - HIP_CHECK(hipMemcpy(d_input, input.data(), size * sizeof(T), hipMemcpyHostToDevice)); - HIP_CHECK(hipDeviceSynchronize()); - - for(auto _ : state) + primbench::json meta() const override { - auto start = std::chrono::high_resolution_clock::now(); - hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), - dim3(size / BlockSize), - dim3(BlockSize), - 0, - stream, - d_input, - d_output, - input[0]); - HIP_CHECK(hipPeekAtLastError()); - HIP_CHECK(hipDeviceSynchronize()); + return primbench::json{} + .add("algo", "warp_scan") + .add("subalgo", Benchmark::name) + .add("data_type", primbench::name()) + .add("block_size", BlockSize) + .add("warp_size", WarpSize) + .add("lvl", "warp"); + } - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); + void run(primbench::state& state) override + { + const size_t input_items = state.size; + const auto& stream = state.stream; - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * size * sizeof(T) * Trials); - state.SetItemsProcessed(state.iterations() * size * Trials); + const size_t items = BlockSize * ((input_items + BlockSize - 1) / BlockSize); - HIP_CHECK(hipFree(d_input)); - HIP_CHECK(hipFree(d_output)); -} + std::vector input(items, 1.0f); -#define CREATE_BENCHMARK_IMPL(T, BS, WS, OP) \ - benchmark::RegisterBenchmark(std::string("warp_scan.sub_algorithm_name:" \ - + method_name) \ - .c_str(), \ - &run_benchmark, \ - stream, \ - size) + T* d_input; + T* d_output; + HIP_CHECK(hipMalloc(&d_input, items * sizeof(T))); + HIP_CHECK(hipMalloc(&d_output, items * sizeof(T))); + HIP_CHECK(hipMemcpy(d_input, input.data(), items * sizeof(T), hipMemcpyHostToDevice)); + HIP_CHECK(hipDeviceSynchronize()); + + state.set_items(items * Trials); + state.add_writes(items * Trials); + + state.run( + [&] + { + hipLaunchKernelGGL(HIP_KERNEL_NAME(kernel), + dim3(items / BlockSize), + dim3(BlockSize), + 0, + stream, + d_input, + d_output, + input[0]); + }); + + HIP_CHECK(hipFree(d_input)); + HIP_CHECK(hipFree(d_output)); + } +}; + +#define CREATE_BENCHMARK_IMPL(T, BS, WS, OP) executor.queue>() #define CREATE_BENCHMARK(T, BS, WS) CREATE_BENCHMARK_IMPL(T, BS, WS, Benchmark) -// clang-format off #if HIPCUB_WARP_THREADS_MACRO == 32 #define BENCHMARK_TYPE(type) \ CREATE_BENCHMARK(type, 60, 15), \ @@ -222,93 +224,43 @@ void run_benchmark(benchmark::State& state, hipStream_t stream, size_t size) CREATE_BENCHMARK(type, 128, 64), \ CREATE_BENCHMARK(type, 256, 64) #endif -// clang-format on template -auto add_benchmarks(std::vector& benchmarks, - const std::string& method_name, - hipStream_t stream, - size_t size) - -> std::enable_if_t::value - || std::is_same::value> +auto add_benchmarks(primbench::executor& executor) + -> std::enable_if_t + || std::is_same_v> { - using custom_double2 = benchmark_utils::custom_type; - using custom_int_double = benchmark_utils::custom_type; - - std::vector new_benchmarks - = {BENCHMARK_TYPE(int), - BENCHMARK_TYPE(float), - BENCHMARK_TYPE(double), - BENCHMARK_TYPE(int8_t), - BENCHMARK_TYPE(custom_double2), - BENCHMARK_TYPE(custom_int_double)}; - benchmarks.insert(benchmarks.end(), new_benchmarks.begin(), new_benchmarks.end()); + BENCHMARK_TYPE(int); + BENCHMARK_TYPE(float); + BENCHMARK_TYPE(double); + BENCHMARK_TYPE(int8_t); + BENCHMARK_TYPE(custom_double2); + BENCHMARK_TYPE(custom_int_double); } template -auto add_benchmarks(std::vector& benchmarks, - const std::string& method_name, - hipStream_t stream, - size_t size) -> std::enable_if_t::value> +auto add_benchmarks(primbench::executor& executor) + -> std::enable_if_t> { - using custom_double2 = benchmark_utils::custom_type; - using custom_int_double = benchmark_utils::custom_type; - - std::vector new_benchmarks - = {BENCHMARK_TYPE_P2(int), - BENCHMARK_TYPE_P2(float), - BENCHMARK_TYPE_P2(double), - BENCHMARK_TYPE_P2(int8_t), - BENCHMARK_TYPE_P2(custom_double2), - BENCHMARK_TYPE_P2(custom_int_double)}; - benchmarks.insert(benchmarks.end(), new_benchmarks.begin(), new_benchmarks.end()); + BENCHMARK_TYPE_P2(int); + BENCHMARK_TYPE_P2(float); + BENCHMARK_TYPE_P2(double); + BENCHMARK_TYPE_P2(int8_t); + BENCHMARK_TYPE_P2(custom_double2); + BENCHMARK_TYPE_P2(custom_int_double); } int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); - - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); - - std::cout << "benchmark_warp_scan" << std::endl; - - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; - - // Add benchmarks - std::vector benchmarks; - add_benchmarks(benchmarks, "inclusive_scan", stream, size); - add_benchmarks(benchmarks, "exclusive_scan", stream, size); - add_benchmarks(benchmarks, "broadcast", stream, size); - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } - } + primbench::executor executor(argc, argv, settings); + + add_benchmarks(executor); + add_benchmarks(executor); + add_benchmarks(executor); - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/benchmark_warp_store.cpp b/projects/hipcub/benchmark/benchmark_warp_store.cpp index 6632faf178be..d9b72366316a 100644 --- a/projects/hipcub/benchmark/benchmark_warp_store.cpp +++ b/projects/hipcub/benchmark/benchmark_warp_store.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -20,27 +20,36 @@ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE // SOFTWARE. -#include "common_benchmark_header.hpp" +#include "benchmark_utils.hpp" -// HIP API #include #include -#ifndef DEFAULT_N -const size_t DEFAULT_N = 1024 * 1024 * 32; -#endif +constexpr const char* get_algorithm_name(hipcub::WarpStoreAlgorithm algorithm) +{ + switch(algorithm) + { + case hipcub::WarpStoreAlgorithm::WARP_STORE_DIRECT: return "warp_store_direct"; + case hipcub::WarpStoreAlgorithm::WARP_STORE_STRIPED: return "warp_store_striped"; + case hipcub::WarpStoreAlgorithm::WARP_STORE_VECTORIZE: return "warp_store_vectorize"; + case hipcub::WarpStoreAlgorithm::WARP_STORE_TRANSPOSE: return "warp_store_transpose"; + } + + return "unknown_algorithm"; +} template -__device__ auto warp_store_benchmark(T* d_output) +__device__ +auto warp_store_benchmark_device_fn(T* d_output) -> std::enable_if_t> { T thread_data[ItemsPerThread]; -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(unsigned i = 0; i < ItemsPerThread; ++i) { thread_data[i] = static_cast(i); @@ -49,7 +58,8 @@ __device__ auto warp_store_benchmark(T* d_output) using WarpStoreT = ::hipcub::WarpStore; constexpr unsigned warps_in_block = BlockSize / LogicalWarpSize; constexpr int tile_size = ItemsPerThread * LogicalWarpSize; - __shared__ typename WarpStoreT::TempStorage temp_storage[warps_in_block]; + __shared__ + typename WarpStoreT::TempStorage temp_storage[warps_in_block]; const unsigned warp_id = threadIdx.x / LogicalWarpSize; const unsigned global_warp_id = blockIdx.x * warps_in_block + warp_id; @@ -61,7 +71,8 @@ template -__device__ auto warp_store_benchmark(T* /*d_output*/) +__device__ +auto warp_store_benchmark_device_fn(T* /*d_output*/) -> std::enable_if_t> {} @@ -70,193 +81,163 @@ template -__global__ __launch_bounds__(BlockSize) void warp_store_kernel(T* d_output) +__global__ __launch_bounds__(BlockSize) +void warp_store_kernel(T* d_output) { - warp_store_benchmark(d_output); + warp_store_benchmark_device_fn(d_output); } template -void run_benchmark(benchmark::State& state, hipStream_t stream, size_t N) + ::hipcub::WarpStoreAlgorithm Algorithm> +class warp_store_benchmark : public primbench::benchmark_interface { - constexpr unsigned items_per_block = BlockSize * ItemsPerThread; - const unsigned size = items_per_block * ((N + items_per_block - 1) / items_per_block); - - T* d_output; - HIP_CHECK(hipMalloc(&d_output, size * sizeof(T))); + primbench::json meta() const override + { + return primbench::json{} + .add("algo", "warp_store") + .add("subalgo", get_algorithm_name(Algorithm)) + .add("data_type", primbench::name()) + .add("block_size", BlockSize) + .add("items_per_thread", ItemsPerThread) + .add("warp_size", LogicalWarpSize) + .add("lvl", "warp"); + } - for(auto _ : state) + void run(primbench::state& state) override { - auto start = std::chrono::high_resolution_clock::now(); + const size_t input_items = state.size; + const auto& stream = state.stream; - for(size_t i = 0; i < Trials; ++i) - { - warp_store_kernel - <<>>(d_output); - } - HIP_CHECK(hipPeekAtLastError()) - HIP_CHECK(hipDeviceSynchronize()); - auto end = std::chrono::high_resolution_clock::now(); - auto elapsed_seconds - = std::chrono::duration_cast>(end - start); - state.SetIterationTime(elapsed_seconds.count()); - } - state.SetBytesProcessed(state.iterations() * Trials * size * sizeof(T)); - state.SetItemsProcessed(state.iterations() * Trials * size); + constexpr unsigned items_per_block = BlockSize * ItemsPerThread; + const unsigned items + = items_per_block * ((input_items + items_per_block - 1) / items_per_block); - HIP_CHECK(hipFree(d_output)); -} + T* d_output; + HIP_CHECK(hipMalloc(&d_output, items * sizeof(T))); -#define CREATE_BENCHMARK(T, BS, IT, WS, ALG) \ - benchmark::RegisterBenchmark(std::string("warp_store.") \ - .c_str(), \ - &run_benchmark, \ - stream, \ - size) + state.set_items(items); + state.add_writes(items); + + state.run( + [&] + { + warp_store_kernel + <<>>(d_output); + }); + + HIP_CHECK(hipFree(d_output)); + } +}; + +#define CREATE_BENCHMARK(T, BS, IT, WS, ALG) \ + executor.queue>() int main(int argc, char* argv[]) { - cli::Parser parser(argc, argv); - parser.set_optional("size", "size", DEFAULT_N, "number of values"); - parser.set_optional("trials", "trials", -1, "number of iterations"); - parser.run_and_exit_if_error(); + primbench::settings settings; + settings.size = 32 * primbench::MiB; // In items + settings.min_gpu_ms_per_batch = 100; + + primbench::executor executor(argc, argv, settings); + + CREATE_BENCHMARK(int, 256, 4, 32, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(int, 256, 4, 32, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(int, 256, 4, 32, ::hipcub::WARP_STORE_VECTORIZE); + CREATE_BENCHMARK(int, 256, 4, 32, ::hipcub::WARP_STORE_TRANSPOSE); + CREATE_BENCHMARK(int, 256, 8, 32, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(int, 256, 8, 32, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(int, 256, 8, 32, ::hipcub::WARP_STORE_VECTORIZE); + CREATE_BENCHMARK(int, 256, 8, 32, ::hipcub::WARP_STORE_TRANSPOSE); + CREATE_BENCHMARK(int, 256, 16, 32, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(int, 256, 16, 32, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(int, 256, 16, 32, ::hipcub::WARP_STORE_VECTORIZE); + CREATE_BENCHMARK(int, 256, 16, 32, ::hipcub::WARP_STORE_TRANSPOSE); + CREATE_BENCHMARK(int, 256, 32, 32, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(int, 256, 32, 32, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(int, 256, 32, 32, ::hipcub::WARP_STORE_VECTORIZE); + CREATE_BENCHMARK(int, 256, 32, 32, ::hipcub::WARP_STORE_TRANSPOSE); + CREATE_BENCHMARK(int, 256, 64, 32, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(int, 256, 64, 32, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(int, 256, 64, 32, ::hipcub::WARP_STORE_VECTORIZE); + CREATE_BENCHMARK(double, 256, 4, 32, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(double, 256, 4, 32, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(double, 256, 4, 32, ::hipcub::WARP_STORE_VECTORIZE); + CREATE_BENCHMARK(double, 256, 4, 32, ::hipcub::WARP_STORE_TRANSPOSE); + CREATE_BENCHMARK(double, 256, 8, 32, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(double, 256, 8, 32, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(double, 256, 8, 32, ::hipcub::WARP_STORE_VECTORIZE); + CREATE_BENCHMARK(double, 256, 8, 32, ::hipcub::WARP_STORE_TRANSPOSE); + CREATE_BENCHMARK(double, 256, 16, 32, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(double, 256, 16, 32, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(double, 256, 16, 32, ::hipcub::WARP_STORE_VECTORIZE); + CREATE_BENCHMARK(double, 256, 16, 32, ::hipcub::WARP_STORE_TRANSPOSE); + CREATE_BENCHMARK(double, 256, 32, 32, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(double, 256, 32, 32, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(double, 256, 32, 32, ::hipcub::WARP_STORE_VECTORIZE); + + // WARP_STORE_TRANSPOSE removed because of shared memory limit + // CREATE_BENCHMARK(double, 256, 32, 32, ::hipcub::WARP_STORE_TRANSPOSE); + + CREATE_BENCHMARK(double, 256, 64, 32, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(double, 256, 64, 32, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(double, 256, 64, 32, ::hipcub::WARP_STORE_VECTORIZE); + + // WARP_STORE_TRANSPOSE removed because of shared memory limit + // CREATE_BENCHMARK(double, 256, 64, 32, ::hipcub::WARP_STORE_TRANSPOSE); - // Parse argv - benchmark::Initialize(&argc, argv); - const size_t size = parser.get("size"); - const int trials = parser.get("trials"); + if(::benchmark_utils::is_warp_size_supported(64)) + { + CREATE_BENCHMARK(int, 256, 4, 64, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(int, 256, 4, 64, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(int, 256, 4, 64, ::hipcub::WARP_STORE_VECTORIZE); + CREATE_BENCHMARK(int, 256, 4, 64, ::hipcub::WARP_STORE_TRANSPOSE); + CREATE_BENCHMARK(int, 256, 8, 64, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(int, 256, 8, 64, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(int, 256, 8, 64, ::hipcub::WARP_STORE_VECTORIZE); + CREATE_BENCHMARK(int, 256, 8, 64, ::hipcub::WARP_STORE_TRANSPOSE); + CREATE_BENCHMARK(int, 256, 16, 64, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(int, 256, 16, 64, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(int, 256, 16, 64, ::hipcub::WARP_STORE_VECTORIZE); + CREATE_BENCHMARK(int, 256, 16, 64, ::hipcub::WARP_STORE_TRANSPOSE); + CREATE_BENCHMARK(int, 256, 32, 64, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(int, 256, 32, 64, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(int, 256, 32, 64, ::hipcub::WARP_STORE_VECTORIZE); + CREATE_BENCHMARK(int, 256, 32, 64, ::hipcub::WARP_STORE_TRANSPOSE); + CREATE_BENCHMARK(int, 256, 64, 64, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(int, 256, 64, 64, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(int, 256, 64, 64, ::hipcub::WARP_STORE_VECTORIZE); + CREATE_BENCHMARK(double, 256, 4, 64, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(double, 256, 4, 64, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(double, 256, 4, 64, ::hipcub::WARP_STORE_VECTORIZE); + CREATE_BENCHMARK(double, 256, 4, 64, ::hipcub::WARP_STORE_TRANSPOSE); + CREATE_BENCHMARK(double, 256, 8, 64, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(double, 256, 8, 64, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(double, 256, 8, 64, ::hipcub::WARP_STORE_VECTORIZE); + CREATE_BENCHMARK(double, 256, 8, 64, ::hipcub::WARP_STORE_TRANSPOSE); + CREATE_BENCHMARK(double, 256, 16, 64, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(double, 256, 16, 64, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(double, 256, 16, 64, ::hipcub::WARP_STORE_VECTORIZE); - std::cout << "benchmark_warp_store" << std::endl; + // WARP_STORE_TRANSPOSE removed because of shared memory limit + // CREATE_BENCHMARK(double, 256, 16, 64, ::hipcub::WARP_STORE_TRANSPOSE); - // HIP - hipStream_t stream = 0; // default - hipDeviceProp_t devProp; - int device_id = 0; - HIP_CHECK(hipGetDevice(&device_id)); - HIP_CHECK(hipGetDeviceProperties(&devProp, device_id)); - std::cout << "[HIP] Device name: " << devProp.name << std::endl; + CREATE_BENCHMARK(double, 256, 32, 64, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(double, 256, 32, 64, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(double, 256, 32, 64, ::hipcub::WARP_STORE_VECTORIZE); - // Add benchmarks - std::vector benchmarks{ - CREATE_BENCHMARK(int, 256, 4, 32, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(int, 256, 4, 32, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(int, 256, 4, 32, ::hipcub::WARP_STORE_VECTORIZE), - CREATE_BENCHMARK(int, 256, 4, 32, ::hipcub::WARP_STORE_TRANSPOSE), - CREATE_BENCHMARK(int, 256, 8, 32, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(int, 256, 8, 32, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(int, 256, 8, 32, ::hipcub::WARP_STORE_VECTORIZE), - CREATE_BENCHMARK(int, 256, 8, 32, ::hipcub::WARP_STORE_TRANSPOSE), - CREATE_BENCHMARK(int, 256, 16, 32, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(int, 256, 16, 32, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(int, 256, 16, 32, ::hipcub::WARP_STORE_VECTORIZE), - CREATE_BENCHMARK(int, 256, 16, 32, ::hipcub::WARP_STORE_TRANSPOSE), - CREATE_BENCHMARK(int, 256, 32, 32, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(int, 256, 32, 32, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(int, 256, 32, 32, ::hipcub::WARP_STORE_VECTORIZE), - CREATE_BENCHMARK(int, 256, 32, 32, ::hipcub::WARP_STORE_TRANSPOSE), - CREATE_BENCHMARK(int, 256, 64, 32, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(int, 256, 64, 32, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(int, 256, 64, 32, ::hipcub::WARP_STORE_VECTORIZE), - CREATE_BENCHMARK(double, 256, 4, 32, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(double, 256, 4, 32, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(double, 256, 4, 32, ::hipcub::WARP_STORE_VECTORIZE), - CREATE_BENCHMARK(double, 256, 4, 32, ::hipcub::WARP_STORE_TRANSPOSE), - CREATE_BENCHMARK(double, 256, 8, 32, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(double, 256, 8, 32, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(double, 256, 8, 32, ::hipcub::WARP_STORE_VECTORIZE), - CREATE_BENCHMARK(double, 256, 8, 32, ::hipcub::WARP_STORE_TRANSPOSE), - CREATE_BENCHMARK(double, 256, 16, 32, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(double, 256, 16, 32, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(double, 256, 16, 32, ::hipcub::WARP_STORE_VECTORIZE), - CREATE_BENCHMARK(double, 256, 16, 32, ::hipcub::WARP_STORE_TRANSPOSE), - CREATE_BENCHMARK(double, 256, 32, 32, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(double, 256, 32, 32, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(double, 256, 32, 32, ::hipcub::WARP_STORE_VECTORIZE), // WARP_STORE_TRANSPOSE removed because of shared memory limit - // CREATE_BENCHMARK(double, 256, 32, 32, ::hipcub::WARP_STORE_TRANSPOSE), - CREATE_BENCHMARK(double, 256, 64, 32, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(double, 256, 64, 32, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(double, 256, 64, 32, ::hipcub::WARP_STORE_VECTORIZE) - // WARP_STORE_TRANSPOSE removed because of shared memory limit - // CREATE_BENCHMARK(double, 256, 64, 32, ::hipcub::WARP_STORE_TRANSPOSE) - }; + // CREATE_BENCHMARK(double, 256, 32, 64, ::hipcub::WARP_STORE_TRANSPOSE); - if(::benchmark_utils::is_warp_size_supported(64)) - { - std::vector additional_benchmarks{ - CREATE_BENCHMARK(int, 256, 4, 64, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(int, 256, 4, 64, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(int, 256, 4, 64, ::hipcub::WARP_STORE_VECTORIZE), - CREATE_BENCHMARK(int, 256, 4, 64, ::hipcub::WARP_STORE_TRANSPOSE), - CREATE_BENCHMARK(int, 256, 8, 64, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(int, 256, 8, 64, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(int, 256, 8, 64, ::hipcub::WARP_STORE_VECTORIZE), - CREATE_BENCHMARK(int, 256, 8, 64, ::hipcub::WARP_STORE_TRANSPOSE), - CREATE_BENCHMARK(int, 256, 16, 64, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(int, 256, 16, 64, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(int, 256, 16, 64, ::hipcub::WARP_STORE_VECTORIZE), - CREATE_BENCHMARK(int, 256, 16, 64, ::hipcub::WARP_STORE_TRANSPOSE), - CREATE_BENCHMARK(int, 256, 32, 64, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(int, 256, 32, 64, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(int, 256, 32, 64, ::hipcub::WARP_STORE_VECTORIZE), - CREATE_BENCHMARK(int, 256, 32, 64, ::hipcub::WARP_STORE_TRANSPOSE), - CREATE_BENCHMARK(int, 256, 64, 64, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(int, 256, 64, 64, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(int, 256, 64, 64, ::hipcub::WARP_STORE_VECTORIZE), - CREATE_BENCHMARK(double, 256, 4, 64, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(double, 256, 4, 64, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(double, 256, 4, 64, ::hipcub::WARP_STORE_VECTORIZE), - CREATE_BENCHMARK(double, 256, 4, 64, ::hipcub::WARP_STORE_TRANSPOSE), - CREATE_BENCHMARK(double, 256, 8, 64, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(double, 256, 8, 64, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(double, 256, 8, 64, ::hipcub::WARP_STORE_VECTORIZE), - CREATE_BENCHMARK(double, 256, 8, 64, ::hipcub::WARP_STORE_TRANSPOSE), - CREATE_BENCHMARK(double, 256, 16, 64, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(double, 256, 16, 64, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(double, 256, 16, 64, ::hipcub::WARP_STORE_VECTORIZE), - // WARP_STORE_TRANSPOSE removed because of shared memory limit - // CREATE_BENCHMARK(double, 256, 16, 64, - // ::hipcub::WARP_STORE_TRANSPOSE), - CREATE_BENCHMARK(double, 256, 32, 64, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(double, 256, 32, 64, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(double, 256, 32, 64, ::hipcub::WARP_STORE_VECTORIZE), - // WARP_STORE_TRANSPOSE removed because of shared memory limit - // CREATE_BENCHMARK(double, 256, 32, 64, - // ::hipcub::WARP_STORE_TRANSPOSE), - CREATE_BENCHMARK(double, 256, 64, 64, ::hipcub::WARP_STORE_DIRECT), - CREATE_BENCHMARK(double, 256, 64, 64, ::hipcub::WARP_STORE_STRIPED), - CREATE_BENCHMARK(double, 256, 64, 64, ::hipcub::WARP_STORE_VECTORIZE) - // WARP_STORE_TRANSPOSE removed because of shared memory limit - // CREATE_BENCHMARK(double, 256, 64, 64, ::hipcub::WARP_STORE_TRANSPOSE) - }; - benchmarks.insert(benchmarks.end(), - additional_benchmarks.begin(), - additional_benchmarks.end()); - } - - // Use manual timing - for(auto& b : benchmarks) - { - b->UseManualTime(); - b->Unit(benchmark::kMillisecond); - } + CREATE_BENCHMARK(double, 256, 64, 64, ::hipcub::WARP_STORE_DIRECT); + CREATE_BENCHMARK(double, 256, 64, 64, ::hipcub::WARP_STORE_STRIPED); + CREATE_BENCHMARK(double, 256, 64, 64, ::hipcub::WARP_STORE_VECTORIZE); - // Force number of iterations - if(trials > 0) - { - for(auto& b : benchmarks) - { - b->Iterations(trials); - } + // WARP_STORE_TRANSPOSE removed because of shared memory limit + // CREATE_BENCHMARK(double, 256, 64, 64, ::hipcub::WARP_STORE_TRANSPOSE); } - // Run benchmarks - benchmark::RunSpecifiedBenchmarks(); - return 0; + executor.run(); } diff --git a/projects/hipcub/benchmark/cmdparser.hpp b/projects/hipcub/benchmark/cmdparser.hpp deleted file mode 100644 index cdbbdabb13e5..000000000000 --- a/projects/hipcub/benchmark/cmdparser.hpp +++ /dev/null @@ -1,515 +0,0 @@ -// The MIT License (MIT) -// -// Copyright (c) 2015 - 2016 Florian Rappl -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -/* - This file is part of the C++ CmdParser utility. - Copyright (c) 2015 - 2016 Florian Rappl -*/ - -#pragma once -#include -#include -#include -#include -#include -#include - -namespace cli { - struct CallbackArgs { - const std::vector& arguments; - std::ostream& output; - std::ostream& error; - }; - class Parser { - private: - class CmdBase { - public: - explicit CmdBase(const std::string& name, const std::string& alternative, const std::string& description, bool required, bool dominant, bool variadic) : - name(name), - command(name.size() > 0 ? "-" + name : ""), - alternative(alternative.size() > 0 ? "--" + alternative : ""), - description(description), - required(required), - handled(false), - arguments({}), - dominant(dominant), - variadic(variadic) { - } - - virtual ~CmdBase() { - } - - std::string name; - std::string command; - std::string alternative; - std::string description; - bool required; - bool handled; - std::vector arguments; - bool const dominant; - bool const variadic; - - virtual std::string print_value() const = 0; - virtual bool parse(std::ostream& output, std::ostream& error) = 0; - - bool is(const std::string& given) const { - return given == command || given == alternative; - } - }; - - template - struct ArgumentCountChecker - { - static constexpr bool Variadic = false; - }; - - template - struct ArgumentCountChecker> - { - static constexpr bool Variadic = true; - }; - - template - class CmdFunction final : public CmdBase { - public: - explicit CmdFunction(const std::string& name, const std::string& alternative, const std::string& description, bool required, bool dominant) : - CmdBase(name, alternative, description, required, dominant, ArgumentCountChecker::Variadic) { - } - - virtual bool parse(std::ostream& output, std::ostream& error) override { - try { - CallbackArgs args { arguments, output, error }; - value = callback(args); - return true; - } catch (...) { - return false; - } - } - - virtual std::string print_value() const override { - return ""; - } - - std::function callback; - T value; - }; - - template - class CmdArgument final : public CmdBase { - public: - explicit CmdArgument(const std::string& name, const std::string& alternative, const std::string& description, bool required, bool dominant) : - CmdBase(name, alternative, description, required, dominant, ArgumentCountChecker::Variadic), value(T()) { - } - - virtual bool parse(std::ostream&, std::ostream&) override { - try { - value = Parser::parse(arguments, value); - return true; - } catch (...) { - return false; - } - } - - virtual std::string print_value() const override { - return stringify(value); - } - - T value; - }; - - static int parse(const std::vector& elements, const int&) { - if (elements.size() != 1) - throw std::bad_cast(); - - return std::stoi(elements[0]); - } - - static bool parse(const std::vector& elements, const bool& defval) { - if (elements.size() != 0) - throw std::runtime_error("A boolean command line parameter cannot have any arguments."); - - return !defval; - } - - static double parse(const std::vector& elements, const double&) { - if (elements.size() != 1) - throw std::bad_cast(); - - return std::stod(elements[0]); - } - - static float parse(const std::vector& elements, const float&) { - if (elements.size() != 1) - throw std::bad_cast(); - - return std::stof(elements[0]); - } - - static long double parse(const std::vector& elements, const long double&) { - if (elements.size() != 1) - throw std::bad_cast(); - - return std::stold(elements[0]); - } - - static unsigned int parse(const std::vector& elements, const unsigned int&) { - if (elements.size() != 1) - throw std::bad_cast(); - - return static_cast(std::stoul(elements[0])); - } - - static unsigned long parse(const std::vector& elements, const unsigned long&) { - if (elements.size() != 1) - throw std::bad_cast(); - - return std::stoul(elements[0]); - } - - static unsigned long long parse(const std::vector& elements, const unsigned long long&) { - if (elements.size() != 1) - throw std::bad_cast(); - - return std::stoull(elements[0]); - } - - static long parse(const std::vector& elements, const long&) { - if (elements.size() != 1) - throw std::bad_cast(); - - return std::stol(elements[0]); - } - - static std::string parse(const std::vector& elements, const std::string&) { - if (elements.size() != 1) - throw std::bad_cast(); - - return elements[0]; - } - - template - static std::vector parse(const std::vector& elements, const std::vector&) { - const T defval = T(); - std::vector values { }; - std::vector buffer(1); - - for (const auto& element : elements) { - buffer[0] = element; - values.push_back(parse(buffer, defval)); - } - - return values; - } - - template - static std::string stringify(const T& value) { - return std::to_string(value); - } - - template - static std::string stringify(const std::vector& values) { - std::stringstream ss { }; - ss << "[ "; - - for (const auto& value : values) { - ss << stringify(value) << " "; - } - - ss << "]"; - return ss.str(); - } - - static std::string stringify(const std::string& str) { - return str; - } - - public: - explicit Parser(int argc, const char** argv) : - _appname(argv[0]) { - for (int i = 1; i < argc; ++i) { - _arguments.push_back(argv[i]); - } - enable_help(); - } - - explicit Parser(int argc, char** argv) : - _appname(argv[0]) { - for (int i = 1; i < argc; ++i) { - _arguments.push_back(argv[i]); - } - enable_help(); - } - - ~Parser() { - for (int i = 0, n = _commands.size(); i < n; ++i) { - delete _commands[i]; - } - } - - bool has_help() const { - for (const auto command : _commands) { - if (command->name == "h" && command->alternative == "--help") { - return true; - } - } - - return false; - } - - void enable_help() { - set_callback("h", "help", std::function([this](CallbackArgs& args){ - args.output << this->usage(); - /*exit(0);*/ - return false; - }), "", true); - } - - void disable_help() { - for (auto command = _commands.begin(); command != _commands.end(); ++command) { - if ((*command)->name == "h" && (*command)->alternative == "--help") { - _commands.erase(command); - break; - } - } - } - - template - void set_default(bool is_required, const std::string& description = "") { - auto command = new CmdArgument { "", "", description, is_required, false }; - _commands.push_back(command); - } - - template - void set_required(const std::string& name, const std::string& alternative, const std::string& description = "", bool dominant = false) { - auto command = new CmdArgument { name, alternative, description, true, dominant }; - _commands.push_back(command); - } - - template - void set_optional(const std::string& name, const std::string& alternative, T defaultValue, const std::string& description = "", bool dominant = false) { - auto command = new CmdArgument { name, alternative, description, false, dominant }; - command->value = defaultValue; - _commands.push_back(command); - } - - template - void set_callback(const std::string& name, const std::string& alternative, std::function callback, const std::string& description = "", bool dominant = false) { - auto command = new CmdFunction { name, alternative, description, false, dominant }; - command->callback = callback; - _commands.push_back(command); - } - - inline void run_and_exit_if_error() { - if (run() == false) { - exit(1); - } - } - - inline bool run() { - return run(std::cout, std::cerr); - } - - inline bool run(std::ostream& output) { - return run(output, std::cerr); - } - - bool run(std::ostream& output, std::ostream& error) { - if (_arguments.size() > 0) { - auto current = find_default(); - - for (int i = 0, n = _arguments.size(); i < n; ++i) { - auto isarg = _arguments[i].size() > 0 && _arguments[i][0] == '-'; - auto associated = isarg ? find(_arguments[i]) : nullptr; - - if (associated != nullptr) { - current = associated; - associated->handled = true; - } else if (current == nullptr) { - current = find(_arguments[i]); - // Code was commented out so cmdparser can ignore unknown options - // error << no_default(); - // return false; - } else { - current->arguments.push_back(_arguments[i]); - current->handled = true; - if (!current->variadic) - { - // If the current command is not variadic, then no more arguments - // should be added to it. In this case, switch back to the default - // command. - current = find_default(); - } - } - } - } - - // First, parse dominant arguments since they succeed even if required - // arguments are missing. - for (auto command : _commands) { - if (command->handled && command->dominant && !command->parse(output, error)) { - error << howto_use(command); - return false; - } - } - - // Next, check for any missing arguments. - for (auto command : _commands) { - if (command->required && !command->handled) { - error << howto_required(command); - return false; - } - } - - // Finally, parse all remaining arguments. - for (auto command : _commands) { - if (command->handled && !command->dominant && !command->parse(output, error)) { - error << howto_use(command); - return false; - } - } - - return true; - } - - template - T get(const std::string& name) const { - for (const auto& command : _commands) { - if (command->name == name) { - auto cmd = dynamic_cast*>(command); - - if (cmd == nullptr) { - throw std::runtime_error("Invalid usage of the parameter " + name + " detected."); - } - - return cmd->value; - } - } - - throw std::runtime_error("The parameter " + name + " could not be found."); - } - - template - T get_if(const std::string& name, std::function callback) const { - auto value = get(name); - return callback(value); - } - - int requirements() const { - int count = 0; - - for (const auto& command : _commands) { - if (command->required) { - ++count; - } - } - - return count; - } - - int commands() const { - return static_cast(_commands.size()); - } - - inline const std::string& app_name() const { - return _appname; - } - - protected: - CmdBase* find(const std::string& name) { - for (auto command : _commands) { - if (command->is(name)) { - return command; - } - } - - return nullptr; - } - - CmdBase* find_default() { - for (auto command : _commands) { - if (command->name == "") { - return command; - } - } - - return nullptr; - } - - std::string usage() const { - std::stringstream ss { }; - ss << "Available parameters:\n\n"; - - for (const auto& command : _commands) { - ss << " " << command->command << "\t" << command->alternative; - - if (command->required == true) { - ss << "\t(required)"; - } - - ss << "\n " << command->description; - - if (command->required == false) { - ss << "\n " << "This parameter is optional. The default value is '" + command->print_value() << "'."; - } - - ss << "\n\n"; - } - - return ss.str(); - } - - void print_help(std::stringstream& ss) const { - if (has_help()) { - ss << "For more help use --help or -h.\n"; - } - } - - std::string howto_required(CmdBase* command) const { - std::stringstream ss { }; - ss << "The parameter " << command->name << " is required.\n"; - ss << command->description << '\n'; - print_help(ss); - return ss.str(); - } - - std::string howto_use(CmdBase* command) const { - std::stringstream ss { }; - ss << "The parameter " << command->name << " has invalid arguments.\n"; - ss << command->description << '\n'; - print_help(ss); - return ss.str(); - } - - std::string no_default() const { - std::stringstream ss { }; - ss << "No default parameter has been specified.\n"; - ss << "The given argument must be used with a parameter.\n"; - print_help(ss); - return ss.str(); - } - - private: - const std::string _appname; - std::vector _arguments; - std::vector _commands; - }; -} diff --git a/projects/hipcub/benchmark/common_benchmark_header.hpp b/projects/hipcub/benchmark/common_benchmark_header.hpp deleted file mode 100644 index a632840a815f..000000000000 --- a/projects/hipcub/benchmark/common_benchmark_header.hpp +++ /dev/null @@ -1,60 +0,0 @@ -// MIT License -// -// Copyright (c) 2020-2025 Advanced Micro Devices, Inc. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// Google Benchmark -#include "benchmark/benchmark.h" - -// CmdParser -#include "cmdparser.hpp" - -// HIP API -#include - -// benchmark_utils.hpp should only be included by this header. -// The following definition is used as guard in benchmark_utils.hpp -// Including benchmark_utils.hpp by itself will cause a compile error. -#define BENCHMARK_UTILS_INCLUDE_GUARD -#include "benchmark_utils.hpp" - -#define HIP_CHECK(condition) \ - { \ - hipError_t error = condition; \ - if(error != hipSuccess){ \ - std::cout << "HIP error: " << error << " line: " << __LINE__ << std::endl; \ - exit(error); \ - } \ - } - diff --git a/projects/hipcub/cmake/Dependencies.cmake b/projects/hipcub/cmake/Dependencies.cmake index 246a98af0b2e..77fa2dd431e7 100644 --- a/projects/hipcub/cmake/Dependencies.cmake +++ b/projects/hipcub/cmake/Dependencies.cmake @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -25,36 +25,42 @@ # ########################### # NOTE1: the reason we don't scope global state meddling using add_subdirectory -# is because CMake < 3.24 lacks CMAKE_FIND_PACKAGE_TARGETS_GLOBAL which -# would promote IMPORTED targets of find_package(CONFIG) to be visible -# by other parts of the build. So we save and restore global state. +# is because CMake < 3.24 lacks CMAKE_FIND_PACKAGE_TARGETS_GLOBAL which +# would promote IMPORTED targets of find_package(CONFIG) to be visible +# by other parts of the build. So we save and restore global state. # # NOTE2: We disable the ROCMChecks.cmake warning noting that we meddle with -# global state. This is consequence of abusing the CMake CXX language -# which HIP piggybacks on top of. This kind of HIP support has one chance -# at observing the global flags, at the find_package(HIP) invocation. -# The device compiler won't be able to pick up changes after that, hence -# the warning. +# global state. This is consequence of abusing the CMake CXX language +# which HIP piggybacks on top of. This kind of HIP support has one chance +# at observing the global flags, at the find_package(HIP) invocation. +# The device compiler won't be able to pick up changes after that, hence +# the warning. # # NOTE3: hipCUB and rocPRIM share CMake options for building tests, benchmarks -# and examples. Until that's not fixed, we have to save/restore them. +# and examples. Until that's not fixed, we have to save/restore them. set(USER_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + if(DEFINED BUILD_SHARED_LIBS) set(USER_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) endif() + set(USER_ROCM_WARN_TOOLCHAIN_VAR ${ROCM_WARN_TOOLCHAIN_VAR}) set(ROCM_WARN_TOOLCHAIN_VAR OFF CACHE BOOL "") + # Suppress ROCMChecks WARNING on third-party dependencies set(_HIPCUB_DISABLE_ROCM_CHECKS FALSE) + macro(rocm_check_toolchain_var var access value list_file) if(NOT _HIPCUB_DISABLE_ROCM_CHECKS) _rocm_check_toolchain_var("${var}" "${access}" "${value}" "${list_file}") endif() endmacro() + # Turn off warnings and errors for all warnings in dependencies separate_arguments(CXX_FLAGS_LIST NATIVE_COMMAND ${CMAKE_CXX_FLAGS}) list(REMOVE_ITEM CXX_FLAGS_LIST /WX -Werror -Werror=pendantic -pedantic-errors) + if(MSVC) list(FILTER CXX_FLAGS_LIST EXCLUDE REGEX "/[Ww]([0-4]?)(all)?") # Remove MSVC warning flags list(APPEND CXX_FLAGS_LIST /w) @@ -62,7 +68,9 @@ else() list(FILTER CXX_FLAGS_LIST EXCLUDE REGEX "-W(all|extra|everything)") # Remove GCC/LLVM flags list(APPEND CXX_FLAGS_LIST -w) endif() + list(JOIN CXX_FLAGS_LIST " " CMAKE_CXX_FLAGS) + # Don't build client dependencies as shared set(BUILD_SHARED_LIBS OFF CACHE BOOL "Global flag to cause add_library() to create shared libraries if on." FORCE) @@ -88,7 +96,7 @@ function(find_download_branch git_path branch) string(STRIP ${output} output) endif() - if(NOT (${output} MATCHES "[\t ]+refs/heads/${branch_value}(\n)?$")) + if(NOT(${output} MATCHES "[\t ]+refs/heads/${branch_value}(\n)?$")) message(WARNING "Unable to locate requested release branch \"${branch_value}\" in repository. Defaulting to the develop branch.") set(${branch} "develop" PARENT_SCOPE) else() @@ -100,6 +108,7 @@ endfunction() function(check_git_version git_path) execute_process(COMMAND ${git_path} "--version" OUTPUT_VARIABLE git_version_output) string(REGEX MATCH "([0-9]+\.[0-9]+\.[0-9]+)" GIT_VERSION_STRING ${git_version_output}) + if(DEFINED CMAKE_MATCH_0) set(GIT_VERSION ${CMAKE_MATCH_0} PARENT_SCOPE) else() @@ -119,18 +128,20 @@ function(fetch_dep method repo_name repo_path download_branch) # and the latter is what gets picked up by find_package(Git), since it's what's in PATH. # Check for a git binary in /usr/bin first, then if git < 2.25 is not found, use find_package(Git) to search # other locations. - if (NOT(GIT_PATH)) + if(NOT(GIT_PATH)) message(STATUS "Checking git version") set(GIT_MIN_VERSION_FOR_SPARSE_CHECKOUT 2.25) find_program(find_result git PATHS /usr/bin NO_DEFAULT_PATH) - if(NOT (${find_result} STREQUAL "find_result-NOTFOUND")) + + if(NOT(${find_result} STREQUAL "find_result-NOTFOUND")) set(GIT_PATH ${find_result} CACHE INTERNAL "Path to the git executable") check_git_version(${GIT_PATH}) endif() if(NOT GIT_VERSION OR "${GIT_VERSION}" LESS ${GIT_MIN_VERSION_FOR_SPARSE_CHECKOUT}) find_package(Git QUIET) + if(GIT_FOUND) set(GIT_PATH ${GIT_EXECUTABLE} CACHE INTERNAL "Path to the git executable") check_git_version(${GIT_PATH}) @@ -161,6 +172,7 @@ function(fetch_dep method repo_name repo_path download_branch) if(NOT ${${repo_name}_FOUND}) message(STATUS "No existing ${repo_name} package meeting the minimum version requirement (${MIN_ROCPRIM_PACKAGE_VERSION}) was found. Falling back to downloading it.") + # update local and parent variable values set(${method} "DOWNLOAD" PARENT_SCOPE) set(method_value "DOWNLOAD") @@ -181,31 +193,35 @@ function(fetch_dep method repo_name repo_path download_branch) set(FALLBACK_TO_DOWNLOAD ON) message(WARNING "Unable to locate ${repo_name} in parent monorepo (it's not at \"${CMAKE_CURRENT_SOURCE_DIR}/../../projects/${repo_name}/\").") message(STATUS "Checking if local monorepo is a sparse-checkout that we can add ${repo_name} to.") + if(NOT(GIT_PATH)) message(FATAL_ERROR "Git could not be found on the system. Since ${repo_name} could not be found in the local monorepo, git is required to download it.") endif() if(USE_SPARSE_CHECKOUT) execute_process(COMMAND ${GIT_PATH} "sparse-checkout" "list" OUTPUT_VARIABLE sparse_list ERROR_VARIABLE git_error RESULT_VARIABLE git_result - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../../) + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../../) if(NOT(git_result EQUAL 0) OR git_error) message(STATUS "The local monorepo does not appear to be a sparse-checkout.") else() message(STATUS "The local monorepo appears to be a sparse checkout. Attempting to add \"projects/${repo_name}\" to the checkout list.") + # Check if the dependency is already present in the checkout list. # Git lists sparse checkout directories each on a separate line. # Take care not to match something in the middle of a path, eg. "other_dir/projects/${repo_name}/sub_dir". string(REGEX MATCH "(^|\n)projects/${repo_name}($|\n)" find_result ${sparse_list}) + if(find_result) message(STATUS "Found existing entry for \"projects/${repo_name}\" in sparse-checkout list - has the directory structure been modified?") else() # Add project/${repo_name} to the sparse checkout execute_process(COMMAND ${GIT_PATH} "sparse-checkout" "add" "projects/${repo_name}" RESULT_VARIABLE sparse_checkout_result - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../../) + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../../) + # Note that in this case, we are forced to checkout the same branch that the sparse-checkout was created with. execute_process(COMMAND ${GIT_PATH} "checkout" RESULT_VARIABLE checkout_result - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../../) + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../../) if(sparse_checkout_result EQUAL 0 AND checkout_result EQUAL 0) message(STATUS "Added new checkout list entry.") @@ -213,6 +229,7 @@ function(fetch_dep method repo_name repo_path download_branch) else() message(STATUS "Unable to add new checkout list entry.") endif() + # Save the monorepo path in the parent scope set(${repo_path} "${CMAKE_CURRENT_SOURCE_DIR}/../../projects/${repo_name}" PARENT_SCOPE) endif() @@ -221,8 +238,9 @@ function(fetch_dep method repo_name repo_path download_branch) message(STATUS "The version of git installed on the system (${GIT_VERSION}) does not support sparse-checkout.") endif() - if (FALLBACK_TO_DOWNLOAD) + if(FALLBACK_TO_DOWNLOAD) message(WARNING "Unable to locate/fetch dependency ${repo_name} from monorepo. Falling back to downloading it.") + # update local and parent variable values set(${method} "DOWNLOAD" PARENT_SCOPE) set(method_value "DOWNLOAD") @@ -246,10 +264,12 @@ function(fetch_dep method repo_name repo_path download_branch) set(download_branch_value ${${download_branch}}) message(STATUS "Downloading ${repo_name} from https://github.com/ROCm/rocm-libraries.git") + if(${USE_SPARSE_CHECKOUT}) # In this case, we have access to git sparse-checkout. # Check if the dependency has already been downloaded in the past: find_path(found_path NAMES "." PATHS "${CMAKE_CURRENT_BINARY_DIR}/${repo_name}-src/" NO_CACHE NO_DEFAULT_PATH) + if(${found_path} STREQUAL "found_path-NOTFOUND") # First, git clone with options "--no-checkout" and "--filter=tree:0" to prevent files from being pulled immediately. # Use option "--depth=1" to avoid downloading past commit history. @@ -257,14 +277,14 @@ function(fetch_dep method repo_name repo_path download_branch) # Next, use git sparse-checkout to ensure we only pull the directory containing the desired repo. execute_process(COMMAND ${GIT_PATH} sparse-checkout init --cone - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${repo_name}-src) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${repo_name}-src) execute_process(COMMAND ${GIT_PATH} sparse-checkout set projects/${repo_name} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${repo_name}-src) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${repo_name}-src) # Finally, download the files using git checkout. execute_process(COMMAND ${GIT_PATH} checkout ${download_branch_value} - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${repo_name}-src) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${repo_name}-src) message(STATUS "${repo_name} download complete") else() @@ -277,9 +297,11 @@ function(fetch_dep method repo_name repo_path download_branch) # In this case, we do not have access to sparse-checkout, so we need to download the whole monorepo. # Check if the monorepo has already been downloaded to satisfy a previous dependency find_path(found_path NAMES "." PATHS "${CMAKE_CURRENT_BINARY_DIR}/monorepo-src/" NO_CACHE NO_DEFAULT_PATH) + if(${found_path} STREQUAL "found_path-NOTFOUND") # Warn the user that this will take some time. message(WARNING "The detected version of git (${GIT_VERSION}) is older than 2.25 and does not provide sparse-checkout functionality. Falling back to checking out the whole rocm-libraries repository (this may take a long time).") + # Avoid downloading anything related to branches other than the target branch (--single-branch), and avoid any past commit history information (--depth=1) execute_process(COMMAND ${GIT_PATH} clone --single-branch --branch=${download_branch_value} --depth=1 https://github.com/ROCm/rocm-libraries.git ${CMAKE_CURRENT_BINARY_DIR}/monorepo-src) message(STATUS "rocm-libraries download complete") @@ -302,21 +324,23 @@ if(USER_BUILD_TEST) # GTestConfig.cmake defines: GTest::gtest, GTest::gtest_main, GTest::gmock, GTest::gmock_main # # NOTE2: Finding GTest in MODULE mode, one cannot invoke find_package in CONFIG mode, because targets - # will be duplicately defined. + # will be duplicately defined. # # NOTE3: The following snippet first tries to find Google Test binary either in MODULE or CONFIG modes. - # If neither succeeds it goes on to import Google Test into this build either from a system - # source package (apt install googletest on Ubuntu 18.04 only) or GitHub and defines the MODULE - # mode targets. Otherwise if MODULE or CONFIG succeeded, then it prints the result to the - # console via a non-QUIET find_package call and if CONFIG succeeded, creates ALIAS targets - # with the MODULE IMPORTED names. + # If neither succeeds it goes on to import Google Test into this build either from a system + # source package (apt install googletest on Ubuntu 18.04 only) or GitHub and defines the MODULE + # mode targets. Otherwise if MODULE or CONFIG succeeded, then it prints the result to the + # console via a non-QUIET find_package call and if CONFIG succeeded, creates ALIAS targets + # with the MODULE IMPORTED names. if(NOT EXTERNAL_DEPS_FORCE_DOWNLOAD) find_package(GTest QUIET) endif() + if(NOT TARGET GTest::GTest AND NOT TARGET GTest::gtest) option(BUILD_GTEST "Builds the googletest subproject" ON) option(BUILD_GMOCK "Builds the googlemock subproject" OFF) option(INSTALL_GTEST "Enable installation of googletest." OFF) + if(EXISTS /usr/src/googletest AND NOT EXTERNAL_DEPS_FORCE_DOWNLOAD) FetchContent_Declare( googletest @@ -327,84 +351,62 @@ if(USER_BUILD_TEST) FetchContent_Declare( googletest GIT_REPOSITORY https://github.com/google/googletest.git - GIT_TAG release-1.11.0 + GIT_TAG release-1.11.0 ) endif() + set(_HIPCUB_DISABLE_ROCM_CHECKS TRUE) FetchContent_MakeAvailable(googletest) set(_HIPCUB_DISABLE_ROCM_CHECKS FALSE) add_library(GTest::GTest ALIAS gtest) - add_library(GTest::Main ALIAS gtest_main) + add_library(GTest::Main ALIAS gtest_main) else() find_package(GTest REQUIRED) + if(TARGET GTest::gtest_main AND NOT TARGET GTest::Main) add_library(GTest::GTest ALIAS GTest::gtest) - add_library(GTest::Main ALIAS GTest::gtest_main) + add_library(GTest::Main ALIAS GTest::gtest_main) endif() endif() endif(USER_BUILD_TEST) -if(USER_BUILD_BENCHMARK) - set(BENCHMARK_VERSION 1.8.0) - if(NOT EXTERNAL_DEPS_FORCE_DOWNLOAD) - find_package(benchmark CONFIG QUIET) - endif() - if(NOT TARGET benchmark::benchmark) - message(STATUS "Google Benchmark not found. Fetching...") - option(BENCHMARK_ENABLE_TESTING "Enable testing of the benchmark library." OFF) - option(BENCHMARK_ENABLE_INSTALL "Enable installation of benchmark." OFF) - FetchContent_Declare( - googlebench - GIT_REPOSITORY https://github.com/google/benchmark.git - GIT_TAG v${BENCHMARK_VERSION} - ) - set(HAVE_STD_REGEX ON) - set(RUN_HAVE_STD_REGEX 1) - set(_HIPCUB_DISABLE_ROCM_CHECKS TRUE) - FetchContent_MakeAvailable(googlebench) - set(_HIPCUB_DISABLE_ROCM_CHECKS FALSE) - if(NOT TARGET benchmark::benchmark) - add_library(benchmark::benchmark ALIAS benchmark) - endif() - else() - find_package(benchmark CONFIG REQUIRED) - endif() -endif(USER_BUILD_BENCHMARK) - # CUB (only for CUDA platform) if(HIP_COMPILER STREQUAL "nvcc") - set(CCCL_MINIMUM_VERSION 2.8.2) + set(CCCL_MINIMUM_VERSION 3.0.0) + if(NOT DOWNLOAD_CUB) find_package(CCCL ${CCCL_MINIMUM_VERSION} CONFIG) endif() - if (NOT CCCL_FOUND) + if(NOT CCCL_FOUND) message(STATUS "CCCL not found, downloading and extracting CCCL ${CCCL_MINIMUM_VERSION}") file(DOWNLOAD https://github.com/NVIDIA/cccl/archive/refs/tags/v${CCCL_MINIMUM_VERSION}.zip - ${CMAKE_CURRENT_BINARY_DIR}/cccl-${CCCL_MINIMUM_VERSION}.zip - STATUS cccl_download_status LOG cccl_download_log) + ${CMAKE_CURRENT_BINARY_DIR}/cccl-${CCCL_MINIMUM_VERSION}.zip + STATUS cccl_download_status LOG cccl_download_log) list(GET cccl_download_status 0 cccl_download_error_code) + if(cccl_download_error_code) message(FATAL_ERROR "Error: downloading " - "https://github.com/NVIDIA/cccl/archive/refs/tags/v${CCCL_MINIMUM_VERSION}.zip failed " - "error_code: ${cccl_download_error_code} " - "log: ${cccl_download_log}") + "https://github.com/NVIDIA/cccl/archive/refs/tags/v${CCCL_MINIMUM_VERSION}.zip failed " + "error_code: ${cccl_download_error_code} " + "log: ${cccl_download_log}") endif() - if (CMAKE_VERSION VERSION_GREATER_EQUAL 3.18) + if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18) file(ARCHIVE_EXTRACT INPUT ${CMAKE_CURRENT_BINARY_DIR}/cccl-${CCCL_MINIMUM_VERSION}.zip) else() execute_process(COMMAND "${CMAKE_COMMAND}" -E tar xf ${CMAKE_CURRENT_BINARY_DIR}/cccl-${CCCL_MINIMUM_VERSION}.zip - WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} - RESULT_VARIABLE cccl_unpack_error_code) + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + RESULT_VARIABLE cccl_unpack_error_code) + if(cccl_unpack_error_code) message(FATAL_ERROR "Error: unpacking ${CMAKE_CURRENT_BINARY_DIR}/cccl-${CCCL_MINIMUM_VERSION}.zip failed") endif() endif() find_package(CCCL ${CCCL_MINIMUM_VERSION} CONFIG REQUIRED NO_DEFAULT_PATH - PATHS ${CMAKE_CURRENT_BINARY_DIR}/cccl-${CCCL_MINIMUM_VERSION}) + PATHS ${CMAKE_CURRENT_BINARY_DIR}/cccl-${CCCL_MINIMUM_VERSION}) endif() else() # rocPRIM (only for ROCm platform) @@ -415,17 +417,19 @@ else() message(STATUS "Configuring rocPRIM") FetchContent_Declare( prim - SOURCE_DIR ${ROCPRIM_PATH} - INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/deps/rocprim - CMAKE_ARGS -DBUILD_TEST=OFF -DCMAKE_INSTALL_PREFIX= -DCMAKE_PREFIX_PATH=/opt/rocm + SOURCE_DIR ${ROCPRIM_PATH} + INSTALL_DIR ${CMAKE_CURRENT_BINARY_DIR}/deps/rocprim + CMAKE_ARGS -DBUILD_TEST=OFF -DCMAKE_INSTALL_PREFIX= -DCMAKE_PREFIX_PATH=/opt/rocm LOG_CONFIGURE TRUE - LOG_BUILD TRUE - LOG_INSTALL TRUE + LOG_BUILD TRUE + LOG_INSTALL TRUE ) FetchContent_MakeAvailable(prim) + if(NOT TARGET roc::rocprim) add_library(roc::rocprim ALIAS rocprim) endif() + if(NOT TARGET roc::rocprim_hip) add_library(roc::rocprim_hip ALIAS rocprim_hip) endif() @@ -438,9 +442,11 @@ endforeach() # Restore user global state set(CMAKE_CXX_FLAGS ${USER_CXX_FLAGS}) + if(DEFINED USER_BUILD_SHARED_LIBS) set(BUILD_SHARED_LIBS ${USER_BUILD_SHARED_LIBS}) else() - unset(BUILD_SHARED_LIBS CACHE ) + unset(BUILD_SHARED_LIBS CACHE) endif() + set(ROCM_WARN_TOOLCHAIN_VAR ${USER_ROCM_WARN_TOOLCHAIN_VAR} CACHE BOOL "") diff --git a/projects/hipcub/cmake/SetupNVCC.cmake b/projects/hipcub/cmake/SetupNVCC.cmake index 46b88d278332..48b59eb2f403 100644 --- a/projects/hipcub/cmake/SetupNVCC.cmake +++ b/projects/hipcub/cmake/SetupNVCC.cmake @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2018-2024 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2018-2026 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -123,6 +123,6 @@ if (NOT _HIPCUB_HIP_NVCC_FLAGS_SET) set(_HIPCUB_HIP_NVCC_FLAGS_SET ON CACHE INTERNAL "") endif() -# Ignore warnings about #pragma unroll +# Ignore warnings about _CCCL_PRAGMA_UNROLL_FULL() # and about deprecated CUDA function(s) used in hip/nvcc_detail/hip_runtime_api.h # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${HIP_CPP_CONFIG_FLAGS_STRIP} -Wno-unknown-pragmas -Wno-deprecated-declarations" CACHE STRING "compile flags" FORCE) diff --git a/projects/hipcub/examples/device/example_device_partition_flagged.cpp b/projects/hipcub/examples/device/example_device_partition_flagged.cpp index f56cae1d6e8d..35e4f529c866 100644 --- a/projects/hipcub/examples/device/example_device_partition_flagged.cpp +++ b/projects/hipcub/examples/device/example_device_partition_flagged.cpp @@ -82,10 +82,10 @@ void Initialize( unsigned short repeat; RandomBits(repeat); repeat = (unsigned short) ((float(repeat) * (float(max_segment) / float(max_short)))); - repeat = std::max(1, repeat); + repeat = _HIPCUB_STD::max(1, repeat); int j = i; - while (j < std::min(i + repeat, num_items)) + while(j < _HIPCUB_STD::min(i + repeat, num_items)) { h_flags[j] = 0; h_in[j] = key; diff --git a/projects/hipcub/examples/device/example_device_partition_if.cpp b/projects/hipcub/examples/device/example_device_partition_if.cpp index 7516f9a40d77..f12c8188f373 100644 --- a/projects/hipcub/examples/device/example_device_partition_if.cpp +++ b/projects/hipcub/examples/device/example_device_partition_if.cpp @@ -93,10 +93,10 @@ void Initialize( unsigned short repeat; RandomBits(repeat); repeat = (unsigned short) ((float(repeat) * (float(max_segment) / float(max_short)))); - repeat = std::max(1, repeat); + repeat = _HIPCUB_STD::max(1, repeat); int j = i; - while (j < std::min(i + repeat, num_items)) + while(j < _HIPCUB_STD::min(i + repeat, num_items)) { h_in[j] = key; j++; diff --git a/projects/hipcub/examples/device/example_device_select_flagged.cpp b/projects/hipcub/examples/device/example_device_select_flagged.cpp index 75c39789deeb..ba52705f01b0 100644 --- a/projects/hipcub/examples/device/example_device_select_flagged.cpp +++ b/projects/hipcub/examples/device/example_device_select_flagged.cpp @@ -83,10 +83,10 @@ void Initialize( unsigned short repeat; RandomBits(repeat); repeat = (unsigned short) ((float(repeat) * (float(max_segment) / float(max_short)))); - repeat = std::max(1, repeat); + repeat = _HIPCUB_STD::max(1, repeat); int j = i; - while (j < std::min(i + repeat, num_items)) + while(j < _HIPCUB_STD::min(i + repeat, num_items)) { h_flags[j] = 0; h_in[j] = key; diff --git a/projects/hipcub/examples/device/example_device_select_if.cpp b/projects/hipcub/examples/device/example_device_select_if.cpp index 40a81bc15bc7..0493d3c0c476 100644 --- a/projects/hipcub/examples/device/example_device_select_if.cpp +++ b/projects/hipcub/examples/device/example_device_select_if.cpp @@ -93,10 +93,10 @@ void Initialize( unsigned short repeat; RandomBits(repeat); repeat = (unsigned short) ((float(repeat) * (float(max_segment) / float(max_short)))); - repeat = std::max(1, repeat); + repeat = _HIPCUB_STD::max(1, repeat); int j = i; - while (j < std::max(i + repeat, num_items)) + while(j < _HIPCUB_STD::max(i + repeat, num_items)) { h_in[j] = key; j++; diff --git a/projects/hipcub/examples/device/example_device_select_unique.cpp b/projects/hipcub/examples/device/example_device_select_unique.cpp index d923b29cdb4d..196850ed2f9d 100644 --- a/projects/hipcub/examples/device/example_device_select_unique.cpp +++ b/projects/hipcub/examples/device/example_device_select_unique.cpp @@ -80,10 +80,10 @@ void Initialize( unsigned short repeat; RandomBits(repeat); repeat = (unsigned short) ((float(repeat) * (float(max_segment) / float(max_short)))); - repeat = std::max(1, repeat); + repeat = _HIPCUB_STD::max(1, repeat); int j = i; - while (j < std::min(i + repeat, num_items)) + while(j < _HIPCUB_STD::min(i + repeat, num_items)) { h_in[j] = key; j++; diff --git a/projects/hipcub/examples/example_utils.hpp b/projects/hipcub/examples/example_utils.hpp index deedbc937add..ef6ea06bc3b6 100644 --- a/projects/hipcub/examples/example_utils.hpp +++ b/projects/hipcub/examples/example_utils.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2024, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -34,9 +34,10 @@ #include #include -#include #include -#include +#include + +#include _HIPCUB_STD_INCLUDE(functional) #define AssertEquals(a, b) if ((a) != (b)) { std::cerr << "\n(" << __FILE__ << ": " << __LINE__ << ")\n"; exit(1);} @@ -75,8 +76,8 @@ struct CommandLineArgs std::vector args; hipDeviceProp_t deviceProp; float device_giga_bandwidth; - std::size_t device_free_physmem; - std::size_t device_total_physmem; + size_t device_free_physmem; + size_t device_total_physmem; /** * Constructor @@ -124,7 +125,7 @@ struct CommandLineArgs { using namespace std; - for (std::size_t i = 0; i < keys.size(); ++i) + for(size_t i = 0; i < keys.size(); ++i) { if (keys[i] == string(arg_name)) return true; @@ -146,8 +147,8 @@ struct CommandLineArgs /** * Returns the commandline parameter for a given index (not including flags) */ - template - void GetCmdLineArgument(std::size_t index, T &val) + template + void GetCmdLineArgument(size_t index, T& val) { using namespace std; if (index < args.size()) { @@ -164,7 +165,7 @@ struct CommandLineArgs { using namespace std; - for (std::size_t i = 0; i < keys.size(); ++i) + for(size_t i = 0; i < keys.size(); ++i) { if (keys[i] == string(arg_name)) { @@ -189,7 +190,7 @@ struct CommandLineArgs vals.clear(); // Recover from multi-value string - for (std::size_t i = 0; i < keys.size(); ++i) + for(size_t i = 0; i < keys.size(); ++i) { if (keys[i] == string(arg_name)) { @@ -384,7 +385,6 @@ int CompareResults(double* computed, double* reference, OffsetT len, bool verbos return 0; } - // /** // * Verify the contents of a device array match those // * of a host array @@ -392,7 +392,7 @@ int CompareResults(double* computed, double* reference, OffsetT len, bool verbos // int CompareDeviceResults( // hipcub::NullType */* h_reference */, // hipcub::NullType */* d_data */, -// std::size_t /* num_items */, +// size_t /* num_items */, // bool /* verbose */ = true, // bool /* display_data */ = false) // { @@ -407,7 +407,7 @@ int CompareResults(double* computed, double* reference, OffsetT len, bool verbos // int CompareDeviceResults( // S *h_reference, // rocprim::discard_iterator d_data, -// std::size_t num_items, +// size_t num_items, // bool verbose = true, // bool display_data = false) // { @@ -418,13 +418,9 @@ int CompareResults(double* computed, double* reference, OffsetT len, bool verbos * Verify the contents of a device array match those * of a host array */ -template +template int CompareDeviceResults( - S *h_reference, - T *d_data, - std::size_t num_items, - bool verbose = true, - bool display_data = false) + S* h_reference, T* d_data, size_t num_items, bool verbose = true, bool display_data = false) { // Allocate array on host T *h_data = (T*) malloc(num_items * sizeof(T)); @@ -436,12 +432,12 @@ int CompareDeviceResults( if (display_data) { printf("Reference:\n"); - for (std::size_t i = 0; i < num_items; i++) + for(size_t i = 0; i < num_items; i++) { std::cout << CoutCast(h_reference[i]) << ", "; } printf("\n\nComputed:\n"); - for (std::size_t i = 0; i < num_items; i++) + for(size_t i = 0; i < num_items; i++) { std::cout << CoutCast(h_data[i]) << ", "; } @@ -462,13 +458,9 @@ int CompareDeviceResults( * Verify the contents of a device array match those * of a device array */ -template +template int CompareDeviceDeviceResults( - T *d_reference, - T *d_data, - std::size_t num_items, - bool verbose = true, - bool display_data = false) + T* d_reference, T* d_data, size_t num_items, bool verbose = true, bool display_data = false) { // Allocate array on host T *h_reference = (T*) malloc(num_items * sizeof(T)); @@ -481,12 +473,12 @@ int CompareDeviceDeviceResults( // Display data if (display_data) { printf("Reference:\n"); - for (std::size_t i = 0; i < num_items; i++) + for(size_t i = 0; i < num_items; i++) { std::cout << CoutCast(h_reference[i]) << ", "; } printf("\n\nComputed:\n"); - for (std::size_t i = 0; i < num_items; i++) + for(size_t i = 0; i < num_items; i++) { std::cout << CoutCast(h_data[i]) << ", "; } @@ -506,13 +498,11 @@ int CompareDeviceDeviceResults( /** * Print the contents of a host array */ -template -void DisplayResults( - InputIteratorT h_data, - std::size_t num_items) +template +void DisplayResults(InputIteratorT h_data, size_t num_items) { // Display data - for (std::size_t i = 0; i < num_items; i++) + for(size_t i = 0; i < num_items; i++) { std::cout << CoutCast(h_data[i]) << ", "; } @@ -573,8 +563,8 @@ void RandomBits( int current_bit = j * WORD_BYTES * 8; unsigned int word = 0xffffffff; - word &= 0xffffffff << std::max(0, begin_bit - current_bit); - word &= 0xffffffff >> std::max(0, (current_bit + (WORD_BYTES * 8)) - end_bit); + word &= 0xffffffff << _HIPCUB_STD::max(0, begin_bit - current_bit); + word &= 0xffffffff >> _HIPCUB_STD::max(0, (current_bit + (WORD_BYTES * 8)) - end_bit); for (int i = 0; i <= entropy_reduction; i++) { @@ -589,7 +579,7 @@ void RandomBits( memcpy(&key, word_buff, sizeof(K)); K copy = key; - if HIPCUB_IF_CONSTEXPR(std::is_floating_point::value) + if constexpr(std::is_floating_point::value) #ifndef _WIN32 if(!std::isnan(copy)) #else diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/agent/single_pass_scan_operators.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/agent/single_pass_scan_operators.hpp index 8e223cbbf475..f05546715605 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/agent/single_pass_scan_operators.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/agent/single_pass_scan_operators.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -46,10 +46,9 @@ using BlockScanRunningPrefixOp = cub::BlockScanRunningPrefixOp; template> using TilePrefixCallbackOp - = cub::TilePrefixCallbackOp; + = cub::TilePrefixCallbackOp; template // IWYU pragma: export +#include + BEGIN_HIPCUB_NAMESPACE struct DeviceAdjacentDifference { template - static HIPCUB_RUNTIME_FUNCTION hipError_t SubtractLeftCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - InputIteratorT d_input, - OutputIteratorT d_output, - NumItemsT num_items, - DifferenceOpT difference_op = {}, - hipStream_t stream = 0) + typename DifferenceOpT = ::cuda::std::minus<>, + typename NumItemsT = uint32_t> + static HIPCUB_RUNTIME_FUNCTION + hipError_t SubtractLeftCopy(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_input, + OutputIteratorT d_output, + NumItemsT num_items, + DifferenceOpT difference_op = {}, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceAdjacentDifference::SubtractLeftCopy(d_temp_storage, @@ -60,39 +63,16 @@ struct DeviceAdjacentDifference stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION hipError_t - SubtractLeftCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - InputIteratorT d_input, - OutputIteratorT d_output, - NumItemsT num_items, - DifferenceOpT difference_op, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SubtractLeftCopy(d_temp_storage, - temp_storage_bytes, - d_input, - d_output, - num_items, - difference_op, - stream); - } - template - static HIPCUB_RUNTIME_FUNCTION hipError_t SubtractLeft(void* d_temp_storage, - std::size_t& temp_storage_bytes, - RandomAccessIteratorT d_input, - NumItemsT num_items, - DifferenceOpT difference_op = {}, - hipStream_t stream = 0) + typename DifferenceOpT = ::cuda::std::minus<>, + typename NumItemsT = uint32_t> + static HIPCUB_RUNTIME_FUNCTION + hipError_t SubtractLeft(void* d_temp_storage, + size_t& temp_storage_bytes, + RandomAccessIteratorT d_input, + NumItemsT num_items, + DifferenceOpT difference_op = {}, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceAdjacentDifference::SubtractLeft(d_temp_storage, @@ -103,38 +83,18 @@ struct DeviceAdjacentDifference stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION hipError_t - SubtractLeft(void* d_temp_storage, - std::size_t& temp_storage_bytes, - RandomAccessIteratorT d_input, - NumItemsT num_items, - DifferenceOpT difference_op, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SubtractLeft(d_temp_storage, - temp_storage_bytes, - d_input, - num_items, - difference_op, - stream); - } - template - static HIPCUB_RUNTIME_FUNCTION hipError_t SubtractRightCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - InputIteratorT d_input, - OutputIteratorT d_output, - NumItemsT num_items, - DifferenceOpT difference_op = {}, - hipStream_t stream = 0) + typename DifferenceOpT = ::cuda::std::minus<>, + typename NumItemsT = uint32_t> + static HIPCUB_RUNTIME_FUNCTION + hipError_t SubtractRightCopy(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_input, + OutputIteratorT d_output, + NumItemsT num_items, + DifferenceOpT difference_op = {}, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceAdjacentDifference::SubtractRightCopy(d_temp_storage, @@ -146,39 +106,16 @@ struct DeviceAdjacentDifference stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION hipError_t - SubtractRightCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - InputIteratorT d_input, - OutputIteratorT d_output, - NumItemsT num_items, - DifferenceOpT difference_op, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SubtractRightCopy(d_temp_storage, - temp_storage_bytes, - d_input, - d_output, - num_items, - difference_op, - stream); - } - template - static HIPCUB_RUNTIME_FUNCTION hipError_t SubtractRight(void* d_temp_storage, - std::size_t& temp_storage_bytes, - RandomAccessIteratorT d_input, - NumItemsT num_items, - DifferenceOpT difference_op = {}, - hipStream_t stream = 0) + typename DifferenceOpT = ::cuda::std::minus<>, + typename NumItemsT = uint32_t> + static HIPCUB_RUNTIME_FUNCTION + hipError_t SubtractRight(void* d_temp_storage, + size_t& temp_storage_bytes, + RandomAccessIteratorT d_input, + NumItemsT num_items, + DifferenceOpT difference_op = {}, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceAdjacentDifference::SubtractRight(d_temp_storage, @@ -188,27 +125,6 @@ struct DeviceAdjacentDifference difference_op, stream)); } - - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION hipError_t - SubtractRight(void* d_temp_storage, - std::size_t& temp_storage_bytes, - RandomAccessIteratorT d_input, - NumItemsT num_items, - DifferenceOpT difference_op, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SubtractRight(d_temp_storage, - temp_storage_bytes, - d_input, - num_items, - difference_op, - stream); - } }; END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_copy.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_copy.hpp index a6315c517241..64f22a982d59 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_copy.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_copy.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,13 +40,13 @@ BEGIN_HIPCUB_NAMESPACE struct DeviceCopy { template - static hipError_t Batched(void* d_temp_storage, - size_t& temp_storage_bytes, - InputBufferIt input_buffer_it, - OutputBufferIt output_buffer_it, - BufferSizeIteratorT buffer_sizes, - uint32_t num_buffers, - hipStream_t stream = 0) + static hipError_t Batched(void* d_temp_storage, + size_t& temp_storage_bytes, + InputBufferIt input_buffer_it, + OutputBufferIt output_buffer_it, + BufferSizeIteratorT buffer_sizes, + _HIPCUB_STD::int64_t num_buffers, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceCopy::Batched(d_temp_storage, temp_storage_bytes, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_for.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_for.hpp index 0f22c405179d..7bc48bafdde2 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_for.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_for.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,13 +33,11 @@ #include // IWYU pragma: export -#if __cccl_lib_mdspan - #include +#include BEGIN_HIPCUB_NAMESPACE template using extents = ::cuda::std::extents; END_HIPCUB_NAMESPACE -#endif // __cccl_lib_mdspan BEGIN_HIPCUB_NAMESPACE @@ -164,8 +162,7 @@ HIPCUB_RUNTIME_FUNCTION cub::DeviceFor::Bulk(d_temp_storage, temp_storage_bytes, shape, op, stream)); } -// ForEachInExtents only enables when the cccl mdspan extension is enabled -#ifdef __cccl_lib_mdspan + // ForEachInExtents only enables when the cccl mdspan extension is enabled template HIPCUB_RUNTIME_FUNCTION static hipError_t ForEachInExtents(void* d_temp_storage, @@ -189,7 +186,6 @@ HIPCUB_RUNTIME_FUNCTION { return hipCUDAErrorTohipError(cub::DeviceFor::ForEachInExtents(extents, op, stream)); } -#endif // __cccl_lib_mdspan }; END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_histogram.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_histogram.hpp index 60018ad68549..f184c4d2ebd7 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_histogram.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_histogram.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -61,31 +61,6 @@ struct DeviceHistogram stream)); } - template - HIPCUB_RUNTIME_FUNCTION HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static hipError_t - HistogramEven(void* d_temp_storage, - size_t& temp_storage_bytes, - SampleIteratorT d_samples, - CounterT* d_histogram, - int num_levels, - LevelT lower_level, - LevelT upper_level, - OffsetT num_samples, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return HistogramEven(d_temp_storage, - temp_storage_bytes, - d_samples, - d_histogram, - num_levels, - lower_level, - upper_level, - num_samples, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t HistogramEven(void* d_temp_storage, size_t& temp_storage_bytes, @@ -112,35 +87,6 @@ struct DeviceHistogram stream)); } - template - HIPCUB_RUNTIME_FUNCTION HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static hipError_t - HistogramEven(void* d_temp_storage, - size_t& temp_storage_bytes, - SampleIteratorT d_samples, - CounterT* d_histogram, - int num_levels, - LevelT lower_level, - LevelT upper_level, - OffsetT num_row_samples, - OffsetT num_rows, - size_t row_stride_bytes, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return HistogramEven(d_temp_storage, - temp_storage_bytes, - d_samples, - d_histogram, - num_levels, - lower_level, - upper_level, - num_row_samples, - num_rows, - row_stride_bytes, - stream); - } - template - HIPCUB_RUNTIME_FUNCTION static hipError_t - MultiHistogramEven(void* d_temp_storage, - size_t& temp_storage_bytes, - SampleIteratorT d_samples, - CounterT* d_histogram[NUM_ACTIVE_CHANNELS], - int num_levels[NUM_ACTIVE_CHANNELS], - LevelT lower_level[NUM_ACTIVE_CHANNELS], - LevelT upper_level[NUM_ACTIVE_CHANNELS], - OffsetT num_row_pixels, - OffsetT num_rows, - size_t row_stride_bytes, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t MultiHistogramEven(void* d_temp_storage, + size_t& temp_storage_bytes, + SampleIteratorT d_samples, + CounterT* d_histogram[NUM_ACTIVE_CHANNELS], + int num_levels[NUM_ACTIVE_CHANNELS], + LevelT lower_level[NUM_ACTIVE_CHANNELS], + LevelT upper_level[NUM_ACTIVE_CHANNELS], + OffsetT num_row_pixels, + OffsetT num_rows, + size_t row_stride_bytes, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceHistogram::MultiHistogramEven( @@ -232,7 +178,7 @@ struct DeviceHistogram num_row_pixels, num_rows, row_stride_bytes, - stream)); + reinterpret_cast(stream))); } template - HIPCUB_RUNTIME_FUNCTION HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static hipError_t - MultiHistogramEven(void* d_temp_storage, - size_t& temp_storage_bytes, - SampleIteratorT d_samples, - CounterT* d_histogram[NUM_ACTIVE_CHANNELS], - int num_levels[NUM_ACTIVE_CHANNELS], - LevelT lower_level[NUM_ACTIVE_CHANNELS], - LevelT upper_level[NUM_ACTIVE_CHANNELS], - OffsetT num_row_pixels, - OffsetT num_rows, - size_t row_stride_bytes, - hipStream_t stream, - bool debug_synchronous) +HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS +HIPCUB_RUNTIME_FUNCTION + static hipError_t MultiHistogramEven(void* d_temp_storage, + size_t& temp_storage_bytes, + SampleIteratorT d_samples, + CounterT* d_histogram[NUM_ACTIVE_CHANNELS], + int num_levels[NUM_ACTIVE_CHANNELS], + LevelT lower_level[NUM_ACTIVE_CHANNELS], + LevelT upper_level[NUM_ACTIVE_CHANNELS], + OffsetT num_row_pixels, + OffsetT num_rows, + size_t row_stride_bytes, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return MultiHistogramEven(d_temp_storage, - temp_storage_bytes, - d_samples, - d_histogram, - num_levels, - lower_level, - upper_level, - num_row_pixels, - num_rows, - row_stride_bytes, - stream); + + return MultiHistogramEven(d_temp_storage, + temp_storage_bytes, + d_samples, + d_histogram, + num_levels, + lower_level, + upper_level, + num_row_pixels, + num_rows, + row_stride_bytes, + stream); } template @@ -289,29 +242,6 @@ struct DeviceHistogram stream)); } - template - HIPCUB_RUNTIME_FUNCTION HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static hipError_t - HistogramRange(void* d_temp_storage, - size_t& temp_storage_bytes, - SampleIteratorT d_samples, - CounterT* d_histogram, - int num_levels, - LevelT* d_levels, - OffsetT num_samples, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return HistogramRange(d_temp_storage, - temp_storage_bytes, - d_samples, - d_histogram, - num_levels, - d_levels, - num_samples, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t HistogramRange(void* d_temp_storage, size_t& temp_storage_bytes, @@ -336,33 +266,6 @@ struct DeviceHistogram stream)); } - template - HIPCUB_RUNTIME_FUNCTION HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static hipError_t - HistogramRange(void* d_temp_storage, - size_t& temp_storage_bytes, - SampleIteratorT d_samples, - CounterT* d_histogram, - int num_levels, - LevelT* d_levels, - OffsetT num_row_samples, - OffsetT num_rows, - size_t row_stride_bytes, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return HistogramRange(d_temp_storage, - temp_storage_bytes, - d_samples, - d_histogram, - num_levels, - d_levels, - num_row_samples, - num_rows, - row_stride_bytes, - stream); - } - template - HIPCUB_RUNTIME_FUNCTION HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static hipError_t - MultiHistogramRange(void* d_temp_storage, - size_t& temp_storage_bytes, - SampleIteratorT d_samples, - CounterT* d_histogram[NUM_ACTIVE_CHANNELS], - int num_levels[NUM_ACTIVE_CHANNELS], - LevelT* d_levels[NUM_ACTIVE_CHANNELS], - OffsetT num_pixels, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return MultiHistogramRange(d_temp_storage, - temp_storage_bytes, - d_samples, - d_histogram, - num_levels, - d_levels, - num_pixels, - stream); - } - template - static hipError_t Batched(void* d_temp_storage, - size_t& temp_storage_bytes, - InputBufferIt input_buffer_it, - OutputBufferIt output_buffer_it, - BufferSizeIteratorT buffer_sizes, - uint32_t num_buffers, - hipStream_t stream = 0) + static hipError_t Batched(void* d_temp_storage, + size_t& temp_storage_bytes, + InputBufferIt input_buffer_it, + OutputBufferIt output_buffer_it, + BufferSizeIteratorT buffer_sizes, + _HIPCUB_STD::int64_t num_buffers, + hipStream_t stream = 0) { + if(num_buffers == 0) + { + temp_storage_bytes = 0; + return hipSuccess; + } + return hipCUDAErrorTohipError(::cub::DeviceMemcpy::Batched(d_temp_storage, temp_storage_bytes, input_buffer_it, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_merge.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_merge.hpp index f314f5a128ef..61c3b2dde593 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_merge.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_merge.hpp @@ -1,6 +1,6 @@ /****************************************************************************** - * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2025-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,6 +33,9 @@ #include // IWYU pragma: export +#include // IWYU pragma: export +using ::cuda::std::int64_t; + BEGIN_HIPCUB_NAMESPACE struct DeviceMerge @@ -44,11 +47,11 @@ struct DeviceMerge typename CompareOp = ::cuda::std::less<>> HIPCUB_RUNTIME_FUNCTION static hipError_t MergeKeys(void* d_temp_storage, - std::size_t& temp_storage_bytes, + size_t& temp_storage_bytes, KeyIteratorIn1 keys_in1, - int num_keys1, + int64_t num_keys1, KeyIteratorIn2 keys_in2, - int num_keys2, + int64_t num_keys2, KeyIteratorOut keys_out, CompareOp compare_op = {}, hipStream_t stream = 0) @@ -74,13 +77,13 @@ struct DeviceMerge typename CompareOp = ::cuda::std::less<>> HIPCUB_RUNTIME_FUNCTION static hipError_t MergePairs(void* d_temp_storage, - std::size_t& temp_storage_bytes, + size_t& temp_storage_bytes, KeyIteratorIn1 keys_in1, ValueIteratorIn1 values_in1, - int num_keys1, + int64_t num_keys1, KeyIteratorIn2 keys_in2, ValueIteratorIn2 values_in2, - int num_keys2, + int64_t num_keys2, KeyIteratorOut keys_out, ValueIteratorOut values_out, CompareOp compare_op = {}, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_merge_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_merge_sort.hpp index be8792534de1..c047b61beae3 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_merge_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_merge_sort.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,13 +40,14 @@ BEGIN_HIPCUB_NAMESPACE struct DeviceMergeSort { template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairs(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorT d_keys, - ValueIteratorT d_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyIteratorT d_keys, + ValueIteratorT d_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceMergeSort::SortPairs(d_temp_storage, temp_storage_bytes, @@ -57,42 +58,22 @@ struct DeviceMergeSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairs(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorT d_keys, - ValueIteratorT d_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortPairs(d_temp_storage, - temp_storage_bytes, - d_keys, - d_items, - num_items, - compare_op, - stream); - } - template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairsCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - ValueInputIteratorT d_input_items, - KeyIteratorT d_output_keys, - ValueIteratorT d_output_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairsCopy(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + ValueInputIteratorT d_input_items, + KeyIteratorT d_output_keys, + ValueIteratorT d_output_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceMergeSort::SortPairsCopy(d_temp_storage, temp_storage_bytes, @@ -105,43 +86,14 @@ struct DeviceMergeSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairsCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - ValueInputIteratorT d_input_items, - KeyIteratorT d_output_keys, - ValueIteratorT d_output_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortPairsCopy(d_temp_storage, - temp_storage_bytes, - d_input_keys, - d_input_items, - d_output_keys, - d_output_items, - num_items, - compare_op, - stream); - } - template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorT d_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyIteratorT d_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceMergeSort::SortKeys(d_temp_storage, temp_storage_bytes, @@ -151,31 +103,18 @@ struct DeviceMergeSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeys(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorT d_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortKeys(d_temp_storage, temp_storage_bytes, d_keys, num_items, compare_op, stream); - } - template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeysCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - KeyIteratorT d_output_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeysCopy(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + KeyIteratorT d_output_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceMergeSort::SortKeysCopy(d_temp_storage, @@ -187,39 +126,15 @@ struct DeviceMergeSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeysCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - KeyIteratorT d_output_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) - - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortKeysCopy(d_temp_storage, - temp_storage_bytes, - d_input_keys, - d_output_keys, - num_items, - compare_op, - stream); - } - template - HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortPairs(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorT d_keys, - ValueIteratorT d_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyIteratorT d_keys, + ValueIteratorT d_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceMergeSort::StableSortPairs(d_temp_storage, temp_storage_bytes, @@ -230,34 +145,14 @@ struct DeviceMergeSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortPairs(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorT d_keys, - ValueIteratorT d_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return StableSortPairs(d_temp_storage, - temp_storage_bytes, - d_keys, - d_items, - num_items, - compare_op, - stream); - } - template - HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortKeys(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorT d_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyIteratorT d_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceMergeSort::StableSortKeys(d_temp_storage, temp_storage_bytes, @@ -267,36 +162,18 @@ struct DeviceMergeSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortKeys(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorT d_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return StableSortKeys(d_temp_storage, - temp_storage_bytes, - d_keys, - num_items, - compare_op, - stream); - } - template - HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortKeysCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - KeyIteratorT d_output_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeysCopy(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + KeyIteratorT d_output_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceMergeSort::StableSortKeysCopy(d_temp_storage, temp_storage_bytes, @@ -311,15 +188,15 @@ struct DeviceMergeSort typename KeyIteratorT, typename OffsetT, typename CompareOpT> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortKeysCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - KeyIteratorT d_output_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeysCopy(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + KeyIteratorT d_output_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortKeysCopy(d_temp_storage, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_partition.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_partition.hpp index 767fd257268c..b7a6f12f99a3 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_partition.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_partition.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -64,33 +64,6 @@ struct DevicePartition stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t Flagged(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - FlagIterator d_flags, - OutputIteratorT d_out, - NumSelectedIteratorT d_num_selected_out, - NumItemsT num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return Flagged(d_temp_storage, - temp_storage_bytes, - d_in, - d_flags, - d_out, - d_num_selected_out, - num_items, - stream); - } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t If(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - NumSelectedIteratorT d_num_selected_out, - NumItemsT num_items, - SelectOp select_op, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return If(d_temp_storage, - temp_storage_bytes, - d_in, - d_out, - d_num_selected_out, - num_items, - select_op, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t If(void* d_temp_storage, - std::size_t& temp_storage_bytes, + size_t& temp_storage_bytes, InputIteratorT d_in, FirstOutputIteratorT d_first_part_out, SecondOutputIteratorT d_second_part_out, @@ -176,42 +122,6 @@ struct DevicePartition select_second_part_op, stream)); } - - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t If(void* d_temp_storage, - std::size_t& temp_storage_bytes, - InputIteratorT d_in, - FirstOutputIteratorT d_first_part_out, - SecondOutputIteratorT d_second_part_out, - UnselectedOutputIteratorT d_unselected_out, - NumSelectedIteratorT d_num_selected_out, - NumItemsT num_items, - SelectFirstPartOp select_first_part_op, - SelectSecondPartOp select_second_part_op, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return If(d_temp_storage, - temp_storage_bytes, - d_in, - d_first_part_out, - d_second_part_out, - d_unselected_out, - d_num_selected_out, - num_items, - select_first_part_op, - select_second_part_op, - stream); - } }; END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_radix_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_radix_sort.hpp index f2b396903480..30dd10b31800 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_radix_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_radix_sort.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -65,33 +65,6 @@ struct DeviceRadixSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - NumItemsT num_items, - int begin_bit, - int end_bit, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortPairs(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_keys_out, - d_values_in, - d_values_out, - num_items, - begin_bit, - end_bit, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static auto SortPairs(void* d_temp_storage, size_t& temp_storage_bytes, @@ -162,29 +135,6 @@ struct DeviceRadixSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - NumItemsT num_items, - int begin_bit, - int end_bit, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortPairs(d_temp_storage, - temp_storage_bytes, - d_keys, - d_values, - num_items, - begin_bit, - end_bit, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static auto SortPairs(void* d_temp_storage, size_t& temp_storage_bytes, @@ -252,33 +202,6 @@ struct DeviceRadixSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - NumItemsT num_items, - int begin_bit, - int end_bit, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortPairsDescending(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_keys_out, - d_values_in, - d_values_out, - num_items, - begin_bit, - end_bit, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static auto SortPairsDescending(void* d_temp_storage, size_t& temp_storage_bytes, @@ -352,29 +275,6 @@ struct DeviceRadixSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - NumItemsT num_items, - int begin_bit, - int end_bit, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortPairsDescending(d_temp_storage, - temp_storage_bytes, - d_keys, - d_values, - num_items, - begin_bit, - end_bit, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static auto SortPairsDescending(void* d_temp_storage, size_t& temp_storage_bytes, @@ -420,14 +320,15 @@ struct DeviceRadixSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - NumItemsT num_items, - int begin_bit = 0, - int end_bit = sizeof(KeyT) * 8, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + NumItemsT num_items, + int begin_bit = 0, + int end_bit = sizeof(KeyT) * 8, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceRadixSort::SortKeys(d_temp_storage, temp_storage_bytes, @@ -439,29 +340,6 @@ struct DeviceRadixSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - NumItemsT num_items, - int begin_bit, - int end_bit, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortKeys(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_keys_out, - num_items, - begin_bit, - end_bit, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static auto SortKeys(void* d_temp_storage, size_t& temp_storage_bytes, @@ -522,27 +400,6 @@ struct DeviceRadixSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - NumItemsT num_items, - int begin_bit, - int end_bit, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortKeys(d_temp_storage, - temp_storage_bytes, - d_keys, - num_items, - begin_bit, - end_bit, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static auto SortKeys(void* d_temp_storage, size_t& temp_storage_bytes, @@ -601,29 +458,6 @@ struct DeviceRadixSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - NumItemsT num_items, - int begin_bit, - int end_bit, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortKeysDescending(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_keys_out, - num_items, - begin_bit, - end_bit, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static auto SortKeysDescending(void* d_temp_storage, size_t& temp_storage_bytes, @@ -684,27 +518,6 @@ struct DeviceRadixSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - NumItemsT num_items, - int begin_bit, - int end_bit, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortKeysDescending(d_temp_storage, - temp_storage_bytes, - d_keys, - num_items, - begin_bit, - end_bit, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static auto SortKeysDescending(void* d_temp_storage, size_t& temp_storage_bytes, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_reduce.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_reduce.hpp index 8d81dce0e1ed..b8a5edf87bb2 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_reduce.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_reduce.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,9 +32,14 @@ #include "../../../config.hpp" #include "../../../util_deprecated.hpp" +#include "../../../util_type.hpp" #include // IWYU pragma: export +#include + +#include + BEGIN_HIPCUB_NAMESPACE class DeviceReduce @@ -64,33 +69,6 @@ class DeviceReduce stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - Reduce(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - NumItemsT num_items, - ReduceOpT reduction_op, - T init, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return Reduce(d_temp_storage, - temp_storage_bytes, - d_in, - d_out, - num_items, - reduction_op, - init, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t Sum(void* d_temp_storage, size_t& temp_storage_bytes, @@ -107,20 +85,6 @@ class DeviceReduce stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - Sum(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - NumItemsT num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return Sum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t Min(void* d_temp_storage, size_t& temp_storage_bytes, @@ -137,30 +101,50 @@ class DeviceReduce stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - Min(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - NumItemsT num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return Min(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream); - } - - template - HIPCUB_RUNTIME_FUNCTION + template + HIPCUB_RUNTIME_FUNCTION static hipError_t ArgMin(void* d_temp_storage, size_t& temp_storage_bytes, InputIteratorT d_in, ExtremumOutIteratorT d_min_out, IndexOutIteratorT d_index_out, - ::std::int64_t num_items, + NumItemsT num_items, hipStream_t stream = 0) { + using value_type = ::hipcub::detail::it_value_t; + using index_type = ::hipcub::detail::it_value_t; + + // CUB handles zero-length inputs in its internal dispatch layer. + // That behavior must be reproduced manually, so this case is handled here. + if(num_items == 0) + { + if(d_temp_storage == nullptr) + { + temp_storage_bytes = sizeof(int); + return hipSuccess; + } + + value_type init_value = ::cuda::std::numeric_limits::max(); + index_type init_index = 1; + + hipError_t e1 = hipMemcpyAsync(d_min_out, + &init_value, + sizeof(value_type), + hipMemcpyHostToDevice, + stream); + + hipError_t e2 = hipMemcpyAsync(d_index_out, + &init_index, + sizeof(index_type), + hipMemcpyHostToDevice, + stream); + + return (e1 != hipSuccess ? e1 : e2); + } + return hipCUDAErrorTohipError(::cub::DeviceReduce::ArgMin(d_temp_storage, temp_storage_bytes, d_in, @@ -185,28 +169,40 @@ class DeviceReduce NumItemsT num_items, hipStream_t stream = 0) { + using pair_type = ::hipcub::detail::it_value_t; + using value_type = decltype(pair_type::value); + using index_type = decltype(pair_type::key); + + if(num_items == 0) + { + if(d_temp_storage == nullptr) + { + temp_storage_bytes = sizeof(int); + return hipSuccess; + } + + pair_type init; + init.key = static_cast(1); + init.value = ::cuda::std::numeric_limits::max(); + + return hipMemcpyAsync(d_out, &init, sizeof(pair_type), hipMemcpyHostToDevice, stream); + } + + pair_type* out_pair = reinterpret_cast(d_out); + + value_type* d_min_out = &(out_pair->value); + index_type* d_index_out = &(out_pair->key); + _CCCL_SUPPRESS_DEPRECATED_PUSH - return hipCUDAErrorTohipError(::cub::DeviceReduce::ArgMin(d_temp_storage, - temp_storage_bytes, - d_in, - d_out, - num_items, - stream)); + auto status = ArgMin(d_temp_storage, + temp_storage_bytes, + d_in, + d_min_out, + d_index_out, + static_cast(num_items), + stream); _CCCL_SUPPRESS_DEPRECATED_POP - } - - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - ArgMin(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - NumItemsT num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return ArgMin(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream); + return status; } template @@ -225,30 +221,52 @@ class DeviceReduce stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - Max(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - NumItemsT num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return Max(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream); - } - - template + template HIPCUB_RUNTIME_FUNCTION static hipError_t ArgMax(void* d_temp_storage, size_t& temp_storage_bytes, InputIteratorT d_in, ExtremumOutIteratorT d_max_out, IndexOutIteratorT d_index_out, - ::std::int64_t num_items, - hipError_t stream = 0) + NumItemsT num_items, + hipStream_t stream = 0) { + using value_type = ::hipcub::detail::it_value_t; + using index_type = ::hipcub::detail::it_value_t; + + // CUB documentation claims zero-length inputs initialize with numeric_limits::max(), + // but the actual CUB implementation uses numeric_limits::lowest(). + // hipCUB matches the implementation. + + if(num_items == 0) + { + if(d_temp_storage == nullptr) + { + temp_storage_bytes = sizeof(int); + return hipSuccess; + } + + value_type init_value = ::cuda::std::numeric_limits::lowest(); + index_type init_index = 1; // hipCUB 1-based index + + hipError_t e1 = hipMemcpyAsync(d_max_out, + &init_value, + sizeof(value_type), + hipMemcpyHostToDevice, + stream); + + hipError_t e2 = hipMemcpyAsync(d_index_out, + &init_index, + sizeof(index_type), + hipMemcpyHostToDevice, + stream); + + return (e1 != hipSuccess ? e1 : e2); + } + return hipCUDAErrorTohipError(::cub::DeviceReduce::ArgMax(d_temp_storage, temp_storage_bytes, d_in, @@ -259,13 +277,11 @@ class DeviceReduce } template - HIPCUB_DEPRECATED_BECAUSE( - "CUB has superseded this interface in favor of the ArgMax interface " - "that takes two separate " - "iterators: one iterator to which the extremum is written and another " - "iterator to which the " - "index of the found extremum is written. ") - HIPCUB_RUNTIME_FUNCTION + HIPCUB_DEPRECATED_BECAUSE("CUB has superseded this interface in favor of the ArgMax interface " + "that takes two separate iterators: one iterator to which the " + "extremum is written and another " + "iterator to which the index of the found extremum is written. ") +HIPCUB_RUNTIME_FUNCTION static hipError_t ArgMax(void* d_temp_storage, size_t& temp_storage_bytes, InputIteratorT d_in, @@ -273,28 +289,40 @@ class DeviceReduce NumItemsT num_items, hipStream_t stream = 0) { + using pair_type = ::hipcub::detail::it_value_t; + using value_type = decltype(pair_type::value); + using index_type = decltype(pair_type::key); + + if(num_items == 0) + { + if(d_temp_storage == nullptr) + { + temp_storage_bytes = sizeof(int); + return hipSuccess; + } + + pair_type init; + init.key = static_cast(1); + init.value = ::cuda::std::numeric_limits::lowest(); + + return hipMemcpyAsync(d_out, &init, sizeof(pair_type), hipMemcpyHostToDevice, stream); + } + + pair_type* out_pair = reinterpret_cast(d_out); + + value_type* d_max_out = &(out_pair->value); + index_type* d_index_out = &(out_pair->key); + _CCCL_SUPPRESS_DEPRECATED_PUSH - return hipCUDAErrorTohipError(::cub::DeviceReduce::ArgMax(d_temp_storage, - temp_storage_bytes, - d_in, - d_out, - num_items, - stream)); + auto status = ArgMax(d_temp_storage, + temp_storage_bytes, + d_in, + d_max_out, + d_index_out, + static_cast(num_items), + stream); _CCCL_SUPPRESS_DEPRECATED_POP - } - - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - ArgMax(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - NumItemsT num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return ArgMax(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream); + return status; } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - ReduceByKey(void* d_temp_storage, - size_t& temp_storage_bytes, - KeysInputIteratorT d_keys_in, - UniqueOutputIteratorT d_unique_out, - ValuesInputIteratorT d_values_in, - AggregatesOutputIteratorT d_aggregates_out, - NumRunsOutputIteratorT d_num_runs_out, - ReductionOpT reduction_op, - NumItemsT num_items, - hipStream_t stream, - bool debug_synchronous) +private: + template + struct value_only_pair_output_iterator { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return ReduceByKey(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_unique_out, - d_values_in, - d_aggregates_out, - d_num_runs_out, - reduction_op, - num_items, - stream); - } + ScalarOutputIt out; + using value_type = ::cub::KeyValuePair; + HIPCUB_HOST_DEVICE + value_only_pair_output_iterator(ScalarOutputIt o) + : out(o) + {} + HIPCUB_HOST_DEVICE + value_only_pair_output_iterator& operator*() + { + return *this; + } + HIPCUB_HOST_DEVICE + value_only_pair_output_iterator& operator=(value_type const& p) + { + *out = p.value; + return *this; + } + HIPCUB_HOST_DEVICE + value_only_pair_output_iterator& operator++() + { + ++out; + return *this; + } + HIPCUB_HOST_DEVICE + value_only_pair_output_iterator operator++(int) + { + value_only_pair_output_iterator tmp = *this; + ++out; + return tmp; + } + }; }; END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_run_length_encode.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_run_length_encode.hpp index 824ec2228745..dbfae1b63b4a 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_run_length_encode.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_run_length_encode.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -63,32 +63,6 @@ class DeviceRunLengthEncode stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - Encode(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - UniqueOutputIteratorT d_unique_out, - LengthsOutputIteratorT d_counts_out, - NumRunsOutputIteratorT d_num_runs_out, - int num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return Encode(d_temp_storage, - temp_storage_bytes, - d_in, - d_unique_out, - d_counts_out, - d_num_runs_out, - num_items, - stream); - } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - NonTrivialRuns(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OffsetsOutputIteratorT d_offsets_out, - LengthsOutputIteratorT d_lengths_out, - NumRunsOutputIteratorT d_num_runs_out, - int num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return NonTrivialRuns(d_temp_storage, - temp_storage_bytes, - d_in, - d_offsets_out, - d_lengths_out, - d_num_runs_out, - num_items, - stream); - } }; END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_scan.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_scan.hpp index 72ad11f7bc8b..660a6942d4f3 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_scan.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_scan.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,9 +32,12 @@ #include "../../../config.hpp" #include "../../../util_deprecated.hpp" +#include "../thread/thread_operators.hpp" #include // IWYU pragma: export +#include + BEGIN_HIPCUB_NAMESPACE class DeviceScan @@ -57,20 +60,6 @@ class DeviceScan stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t InclusiveSum(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - NumItemsT num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return InclusiveSum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveSum(void* d_temp_storage, @@ -82,23 +71,6 @@ class DeviceScan return InclusiveSum(d_temp_storage, temp_storage_bytes, d_data, d_data, num_items, stream); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t InclusiveSum(void* d_temp_storage, - size_t& temp_storage_bytes, - IteratorT d_data, - int num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return InclusiveSum(d_temp_storage, - temp_storage_bytes, - d_data, - num_items, - stream); - } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t InclusiveScan(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - ScanOpT scan_op, - NumItemsT num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return InclusiveScan(d_temp_storage, - temp_storage_bytes, - d_in, - d_out, - scan_op, - num_items, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveScan(void* d_temp_storage, @@ -163,25 +111,6 @@ class DeviceScan stream); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t InclusiveScan(void* d_temp_storage, - size_t& temp_storage_bytes, - IteratorT d_data, - ScanOpT scan_op, - NumItemsT num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return InclusiveScan(d_temp_storage, - temp_storage_bytes, - d_data, - scan_op, - num_items, - stream); - } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t ExclusiveSum(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - NumItemsT num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return ExclusiveSum(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t ExclusiveSum(void* d_temp_storage, @@ -249,23 +164,6 @@ class DeviceScan return ExclusiveSum(d_temp_storage, temp_storage_bytes, d_data, d_data, num_items, stream); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t ExclusiveSum(void* d_temp_storage, - size_t& temp_storage_bytes, - IteratorT d_data, - NumItemsT num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return ExclusiveSum(d_temp_storage, - temp_storage_bytes, - d_data, - num_items, - stream); - } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t ExclusiveScan(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - ScanOpT scan_op, - InitValueT init_value, - NumItemsT num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return ExclusiveScan(d_temp_storage, - temp_storage_bytes, - d_in, - d_out, - scan_op, - init_value, - num_items, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t ExclusiveScan(void* d_temp_storage, @@ -338,27 +209,6 @@ class DeviceScan stream); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t ExclusiveScan(void* d_temp_storage, - size_t& temp_storage_bytes, - IteratorT d_data, - ScanOpT scan_op, - InitValueT init_value, - NumItemsT num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return ExclusiveScan(d_temp_storage, - temp_storage_bytes, - d_data, - scan_op, - init_value, - num_items, - stream); - } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t ExclusiveScan(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - ScanOpT scan_op, - FutureValue init_value, - NumItemsT num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return ExclusiveScan(d_temp_storage, - temp_storage_bytes, - d_in, - d_out, - scan_op, - init_value, - num_items, - stream); - } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t ExclusiveScan(void* d_temp_storage, - size_t& temp_storage_bytes, - IteratorT d_data, - ScanOpT scan_op, - FutureValue init_value, - NumItemsT num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return ExclusiveScan(d_temp_storage, - temp_storage_bytes, - d_data, - scan_op, - init_value, - num_items, - stream); - } - template + typename EqualityOpT = ::cuda::std::equal_to<>, + typename NumItemsT = std::uint32_t> HIPCUB_RUNTIME_FUNCTION static hipError_t ExclusiveSumByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -487,40 +284,13 @@ class DeviceScan stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t ExclusiveSumByKey(void* d_temp_storage, - size_t& temp_storage_bytes, - KeysInputIteratorT d_keys_in, - ValuesInputIteratorT d_values_in, - ValuesOutputIteratorT d_values_out, - NumItemsT num_items, - EqualityOpT equality_op, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return ExclusiveSumByKey(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_values_in, - d_values_out, - num_items, - equality_op, - stream); - } - template + typename EqualityOpT = ::cuda::std::equal_to<>, + typename NumItemsT = std::uint32_t> HIPCUB_RUNTIME_FUNCTION static hipError_t ExclusiveScanByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -548,41 +318,8 @@ class DeviceScan template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t ExclusiveScanByKey(void* d_temp_storage, - size_t& temp_storage_bytes, - KeysInputIteratorT d_keys_in, - ValuesInputIteratorT d_values_in, - ValuesOutputIteratorT d_values_out, - ScanOpT scan_op, - InitValueT init_value, - NumItemsT num_items, - EqualityOpT equality_op, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return ExclusiveScanByKey(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_values_in, - d_values_out, - scan_op, - init_value, - num_items, - equality_op, - stream); - } - - template + typename EqualityOpT = ::cuda::std::equal_to<>, + typename NumItemsT = std::uint32_t> HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveSumByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -603,39 +340,12 @@ class DeviceScan stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t InclusiveSumByKey(void* d_temp_storage, - size_t& temp_storage_bytes, - KeysInputIteratorT d_keys_in, - ValuesInputIteratorT d_values_in, - ValuesOutputIteratorT d_values_out, - NumItemsT num_items, - EqualityOpT equality_op, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return InclusiveSumByKey(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_values_in, - d_values_out, - num_items, - equality_op, - stream); - } - template + typename EqualityOpT = ::cuda::std::equal_to<>, + typename NumItemsT = std::uint32_t> HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveScanByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -662,8 +372,8 @@ class DeviceScan typename ValuesInputIteratorT, typename ValuesOutputIteratorT, typename ScanOpT, - typename EqualityOpT = ::hipcub::Equality, - typename NumItemsT = ::cuda::std::uint32_t> + typename EqualityOpT = ::cuda::std::equal_to<>, + typename NumItemsT = std::uint32_t> HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveScanByKey(void* d_temp_storage, size_t& temp_storage_bytes, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_radix_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_radix_sort.hpp index a69e17fee04c..8d4807eb72d1 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_radix_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_radix_sort.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -69,39 +69,6 @@ struct DeviceSegmentedRadixSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - int begin_bit, - int end_bit, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortPairs(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_keys_out, - d_values_in, - d_values_out, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - begin_bit, - end_bit, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairs(void* d_temp_storage, size_t& temp_storage_bytes, @@ -128,35 +95,6 @@ struct DeviceSegmentedRadixSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - int begin_bit, - int end_bit, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortPairs(d_temp_storage, - temp_storage_bytes, - d_keys, - d_values, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - begin_bit, - end_bit, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairsDescending(void* d_temp_storage, size_t& temp_storage_bytes, @@ -188,39 +126,6 @@ struct DeviceSegmentedRadixSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - int begin_bit, - int end_bit, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortPairsDescending(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_keys_out, - d_values_in, - d_values_out, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - begin_bit, - end_bit, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairsDescending(void* d_temp_storage, size_t& temp_storage_bytes, @@ -248,35 +153,6 @@ struct DeviceSegmentedRadixSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - int begin_bit, - int end_bit, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortPairsDescending(d_temp_storage, - temp_storage_bytes, - d_keys, - d_values, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - begin_bit, - end_bit, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, size_t& temp_storage_bytes, @@ -303,35 +179,6 @@ struct DeviceSegmentedRadixSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - int begin_bit, - int end_bit, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortKeys(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_keys_out, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - begin_bit, - end_bit, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, size_t& temp_storage_bytes, @@ -356,33 +203,6 @@ struct DeviceSegmentedRadixSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - int begin_bit, - int end_bit, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortKeys(d_temp_storage, - temp_storage_bytes, - d_keys, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - begin_bit, - end_bit, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeysDescending(void* d_temp_storage, size_t& temp_storage_bytes, @@ -410,35 +230,6 @@ struct DeviceSegmentedRadixSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - int begin_bit, - int end_bit, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortKeysDescending(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_keys_out, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - begin_bit, - end_bit, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeysDescending(void* d_temp_storage, size_t& temp_storage_bytes, @@ -463,33 +254,6 @@ struct DeviceSegmentedRadixSort end_bit, stream)); } - - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - int begin_bit, - int end_bit, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortKeysDescending(d_temp_storage, - temp_storage_bytes, - d_keys, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - begin_bit, - end_bit, - stream); - } }; END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_reduce.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_reduce.hpp index 4f21fbfac373..942643b3bfd6 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_reduce.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_reduce.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -44,16 +44,17 @@ struct DeviceSegmentedReduce typename OffsetIteratorT, typename ReductionOp, typename T> - HIPCUB_RUNTIME_FUNCTION static hipError_t Reduce(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - ReductionOp reduction_op, - T initial_value, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t Reduce(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + ReductionOp reduction_op, + T initial_value, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedReduce::Reduce(d_temp_storage, temp_storage_bytes, @@ -67,46 +68,16 @@ struct DeviceSegmentedReduce stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - Reduce(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - ReductionOp reduction_op, - T initial_value, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return Reduce(d_temp_storage, - temp_storage_bytes, - d_in, - d_out, - num_segments, - d_begin_offsets, - d_end_offsets, - reduction_op, - initial_value, - stream); - } - template - HIPCUB_RUNTIME_FUNCTION static hipError_t Sum(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t Sum(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedReduce::Sum(d_temp_storage, temp_storage_bytes, @@ -119,37 +90,15 @@ struct DeviceSegmentedReduce } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - Sum(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return Sum(d_temp_storage, - temp_storage_bytes, - d_in, - d_out, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t Min(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t Min(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedReduce::Min(d_temp_storage, temp_storage_bytes, @@ -162,37 +111,15 @@ struct DeviceSegmentedReduce } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - Min(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return Min(d_temp_storage, - temp_storage_bytes, - d_in, - d_out, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t ArgMin(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t ArgMin(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedReduce::ArgMin(d_temp_storage, temp_storage_bytes, @@ -205,37 +132,15 @@ struct DeviceSegmentedReduce } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - ArgMin(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return ArgMin(d_temp_storage, - temp_storage_bytes, - d_in, - d_out, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t Max(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t Max(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedReduce::Max(d_temp_storage, temp_storage_bytes, @@ -248,37 +153,15 @@ struct DeviceSegmentedReduce } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - Max(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return Max(d_temp_storage, - temp_storage_bytes, - d_in, - d_out, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t ArgMax(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t ArgMax(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedReduce::ArgMax(d_temp_storage, temp_storage_bytes, @@ -289,29 +172,6 @@ struct DeviceSegmentedReduce d_end_offsets, stream)); } - - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - ArgMax(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return ArgMax(d_temp_storage, - temp_storage_bytes, - d_in, - d_out, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } }; END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_sort.hpp index b31bada2f27d..5b193d15b7c7 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_segmented_sort.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -35,20 +35,24 @@ #include // IWYU pragma: export +#include // IWYU pragma: export +using ::cuda::std::int64_t; + BEGIN_HIPCUB_NAMESPACE struct DeviceSegmentedSort { template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int64_t num_items, + int64_t num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedSort::SortKeys(d_temp_storage, temp_storage_bytes, @@ -62,41 +66,16 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortKeys(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_keys_out, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int64_t num_items, + int64_t num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::SortKeysDescending(d_temp_storage, @@ -111,39 +90,15 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortKeysDescending(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_keys_out, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int64_t num_items, + int64_t num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedSort::SortKeys(d_temp_storage, temp_storage_bytes, @@ -156,38 +111,15 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortKeys(d_temp_storage, - temp_storage_bytes, - d_keys, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int64_t num_items, + int64_t num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::SortKeysDescending(d_temp_storage, @@ -201,38 +133,16 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortKeysDescending(d_temp_storage, - temp_storage_bytes, - d_keys, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int64_t num_items, + int64_t num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedSort::StableSortKeys(d_temp_storage, temp_storage_bytes, @@ -246,41 +156,16 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return StableSortKeys(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_keys_out, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int64_t num_items, + int64_t num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::StableSortKeysDescending(d_temp_storage, @@ -295,39 +180,15 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return StableSortKeysDescending(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_keys_out, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int64_t num_items, + int64_t num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedSort::StableSortKeys(d_temp_storage, temp_storage_bytes, @@ -340,38 +201,15 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return StableSortKeys(d_temp_storage, - temp_storage_bytes, - d_keys, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int64_t num_items, + int64_t num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::StableSortKeysDescending(d_temp_storage, @@ -384,44 +222,22 @@ struct DeviceSegmentedSort stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return StableSortKeysDescending(d_temp_storage, - temp_storage_bytes, - d_keys, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int64_t num_items, + int64_t num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedSort::SortPairs(d_temp_storage, temp_storage_bytes, @@ -440,50 +256,18 @@ struct DeviceSegmentedSort typename ValueT, typename BeginOffsetIteratorT, typename EndOffsetIteratorT> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortPairs(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_keys_out, - d_values_in, - d_values_out, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int64_t num_items, + int64_t num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::SortPairsDescending(d_temp_storage, @@ -503,47 +287,16 @@ struct DeviceSegmentedSort typename ValueT, typename BeginOffsetIteratorT, typename EndOffsetIteratorT> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortPairsDescending(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_keys_out, - d_values_in, - d_values_out, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int64_t num_items, + int64_t num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError(::cub::DeviceSegmentedSort::SortPairs(d_temp_storage, temp_storage_bytes, @@ -560,44 +313,16 @@ struct DeviceSegmentedSort typename ValueT, typename BeginOffsetIteratorT, typename EndOffsetIteratorT> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortPairs(d_temp_storage, - temp_storage_bytes, - d_keys, - d_values, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int64_t num_items, + int64_t num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::SortPairsDescending(d_temp_storage, @@ -615,45 +340,18 @@ struct DeviceSegmentedSort typename ValueT, typename BeginOffsetIteratorT, typename EndOffsetIteratorT> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return SortPairsDescending(d_temp_storage, - temp_storage_bytes, - d_keys, - d_values, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int64_t num_items, + int64_t num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::StableSortPairs(d_temp_storage, @@ -673,50 +371,18 @@ struct DeviceSegmentedSort typename ValueT, typename BeginOffsetIteratorT, typename EndOffsetIteratorT> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return StableSortPairs(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_keys_out, - d_values_in, - d_values_out, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int64_t num_items, + int64_t num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::StableSortPairsDescending(d_temp_storage, @@ -736,47 +402,16 @@ struct DeviceSegmentedSort typename ValueT, typename BeginOffsetIteratorT, typename EndOffsetIteratorT> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return StableSortPairsDescending(d_temp_storage, - temp_storage_bytes, - d_keys_in, - d_keys_out, - d_values_in, - d_values_out, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int64_t num_items, + int64_t num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::StableSortPairs(d_temp_storage, @@ -794,44 +429,16 @@ struct DeviceSegmentedSort typename ValueT, typename BeginOffsetIteratorT, typename EndOffsetIteratorT> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return StableSortPairs(d_temp_storage, - temp_storage_bytes, - d_keys, - d_values, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } - - template - HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int64_t num_items, + int64_t num_segments, + BeginOffsetIteratorT d_begin_offsets, + EndOffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return hipCUDAErrorTohipError( ::cub::DeviceSegmentedSort::StableSortPairsDescending(d_temp_storage, @@ -844,34 +451,6 @@ struct DeviceSegmentedSort d_end_offsets, stream)); } - - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - BeginOffsetIteratorT d_begin_offsets, - EndOffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return StableSortPairsDescending(d_temp_storage, - temp_storage_bytes, - d_keys, - d_values, - num_items, - num_segments, - d_begin_offsets, - d_end_offsets, - stream); - } }; END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_select.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_select.hpp index 6812c5cfeb72..9e8f76f1a730 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_select.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_select.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -66,32 +66,6 @@ class DeviceSelect stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t Flagged(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - FlagIterator d_flags, - OutputIteratorT d_out, - NumSelectedIteratorT d_num_selected_out, - int64_t num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return Flagged(d_temp_storage, - temp_storage_bytes, - d_in, - d_flags, - d_out, - d_num_selected_out, - num_items, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t Flagged(void* d_temp_storage, @@ -112,27 +86,6 @@ class DeviceSelect stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t Flagged(void* d_temp_storage, - size_t& temp_storage_bytes, - IteratorT d_data, - FlagIterator d_flags, - NumSelectedIteratorT d_num_selected_out, - int64_t num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return Flagged(d_temp_storage, - temp_storage_bytes, - d_data, - d_flags, - d_num_selected_out, - num_items, - stream); - } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t If(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - NumSelectedIteratorT d_num_selected_out, - int64_t num_items, - SelectOp select_op, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return If(d_temp_storage, - temp_storage_bytes, - d_in, - d_out, - d_num_selected_out, - num_items, - select_op, - stream); - } - template HIPCUB_RUNTIME_FUNCTION static hipError_t If(void* d_temp_storage, @@ -202,27 +129,6 @@ class DeviceSelect stream)); } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t If(void* d_temp_storage, - size_t& temp_storage_bytes, - IteratorT d_data, - NumSelectedIteratorT d_num_selected_out, - int64_t num_items, - SelectOp select_op, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return If(d_temp_storage, - temp_storage_bytes, - d_data, - d_num_selected_out, - num_items, - select_op, - stream); - } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t Unique(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - NumSelectedIteratorT d_num_selected_out, - int64_t num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - return Unique(d_temp_storage, - temp_storage_bytes, - d_in, - d_out, - d_num_selected_out, - num_items, - stream); - } - template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t UniqueByKey(void* d_temp_storage, - size_t& temp_storage_bytes, - KeyIteratorT d_keys_input, - ValueIteratorT d_values_input, - OutputKeyIteratorT d_keys_output, - OutputValueIteratorT d_values_output, - NumSelectedIteratorT d_num_selected_out, - NumItemsT num_items, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return UniqueByKey(d_temp_storage, temp_storage_bytes, d_keys_input, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_spmv.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_spmv.hpp deleted file mode 100644 index dbbd5e0cb10c..000000000000 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/device/device_spmv.hpp +++ /dev/null @@ -1,140 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ - -#ifndef HIPCUB_CUB_DEVICE_DEVICE_SPMV_HPP_ -#define HIPCUB_CUB_DEVICE_DEVICE_SPMV_HPP_ - -#include "../../../config.hpp" -#include "../../../util_deprecated.hpp" - -#include // IWYU pragma: export -#include // IWYU pragma: export - -BEGIN_HIPCUB_NAMESPACE - -class HIPCUB_DEPRECATED_BECAUSE("Use the cuSPARSE library instead") DeviceSpmv -{ - -public: - template ///< Signed integer type for sequence offsets - struct HIPCUB_DEPRECATED_BECAUSE("Use the cuSPARSE library instead") SpmvParams - { - ValueT* - d_values; ///< Pointer to the array of \p num_nonzeros values of the corresponding nonzero elements of matrix A. - OffsetT* - d_row_end_offsets; ///< Pointer to the array of \p m offsets demarcating the end of every row in \p d_column_indices and \p d_values - OffsetT* - d_column_indices; ///< Pointer to the array of \p num_nonzeros column-indices of the corresponding nonzero elements of matrix A. (Indices are zero-valued.) - ValueT* - d_vector_x; ///< Pointer to the array of \p num_cols values corresponding to the dense input vector x - ValueT* - d_vector_y; ///< Pointer to the array of \p num_rows values corresponding to the dense output vector y - int num_rows; ///< Number of rows of matrix A. - int num_cols; ///< Number of columns of matrix A. - int num_nonzeros; ///< Number of nonzero elements of matrix A. - ValueT alpha; ///< Alpha multiplicand - ValueT beta; ///< Beta addend-multiplicand - - ::cub::TexObjInputIterator t_vector_x; - }; - - template - HIPCUB_DEPRECATED_BECAUSE("Use the cuSPARSE library instead") - HIPCUB_RUNTIME_FUNCTION static hipError_t CsrMV(void* d_temp_storage, - size_t& temp_storage_bytes, - ValueT* d_values, - int* d_row_offsets, - int* d_column_indices, - ValueT* d_vector_x, - ValueT* d_vector_y, - int num_rows, - int num_cols, - int num_nonzeros, - hipStream_t stream = 0) - { - _CCCL_SUPPRESS_DEPRECATED_PUSH - ::cub::SpmvParams spmv_params; - _CCCL_SUPPRESS_DEPRECATED_POP - spmv_params.d_values = d_values; - spmv_params.d_row_end_offsets = d_row_offsets + 1; - spmv_params.d_column_indices = d_column_indices; - spmv_params.d_vector_x = d_vector_x; - spmv_params.d_vector_y = d_vector_y; - spmv_params.num_rows = num_rows; - spmv_params.num_cols = num_cols; - spmv_params.num_nonzeros = num_nonzeros; - spmv_params.alpha = 1.0; - spmv_params.beta = 0.0; - - _CCCL_SUPPRESS_DEPRECATED_PUSH - return static_cast( - ::cub::DispatchSpmv::Dispatch(d_temp_storage, - temp_storage_bytes, - spmv_params, - stream)); - _CCCL_SUPPRESS_DEPRECATED_POP - } - - template - HIPCUB_DEPRECATED_BECAUSE("Use the cuSPARSE library instead") - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - CsrMV(void* d_temp_storage, - size_t& temp_storage_bytes, - ValueT* d_values, - int* d_row_offsets, - int* d_column_indices, - ValueT* d_vector_x, - ValueT* d_vector_y, - int num_rows, - int num_cols, - int num_nonzeros, - hipStream_t stream, - bool debug_synchronous) - { - HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); - _CCCL_SUPPRESS_DEPRECATED_PUSH - return CsrMV(d_temp_storage, - temp_storage_bytes, - d_values, - d_row_offsets, - d_column_indices, - d_vector_x, - d_vector_y, - num_rows, - num_cols, - num_nonzeros, - stream); - _CCCL_SUPPRESS_DEPRECATED_POP - } -}; - -END_HIPCUB_NAMESPACE - -#endif // HIPCUB_CUB_DEVICE_DEVICE_SELECT_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/hipcub.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/hipcub.hpp index 2f8157c7bb46..299d843a7bda 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/hipcub.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/hipcub.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -67,7 +67,6 @@ #include "device/device_segmented_reduce.hpp" #include "device/device_segmented_sort.hpp" #include "device/device_select.hpp" -#include "device/device_spmv.hpp" #include "device/device_transform.hpp" // Grid diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/thread/thread_operators.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/thread/thread_operators.hpp index ab0a1f7630ca..0918d3ff0498 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/thread/thread_operators.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/thread/thread_operators.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -27,6 +27,8 @@ #include +#include + BEGIN_HIPCUB_NAMESPACE namespace detail @@ -37,6 +39,111 @@ using accumulator_t = ::cuda::std::__accumulator_t; } // namespace detail +//! deprecated [Since 5.0] +struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::equal_to instead.") Equality +{ + template + HIPCUB_HOST_DEVICE + inline constexpr bool operator()(T&& t, U&& u) const + { + return ::cuda::std::forward(t) == ::cuda::std::forward(u); + } +}; + +//! deprecated [Since 5.0] +struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::not_equal_to instead.") Inequality +{ + template + HIPCUB_HOST_DEVICE + inline constexpr bool operator()(T&& t, U&& u) const + { + return ::cuda::std::forward(t) != ::cuda::std::forward(u); + } +}; + +//! deprecated [Since 5.0] +struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::plus instead.") Sum +{ + template + HIPCUB_HOST_DEVICE + inline constexpr auto operator()(T&& t, U&& u) const -> decltype(auto) + { + return ::cuda::std::forward(t) + ::cuda::std::forward(u); + } +}; + +//! deprecated [Since 5.0] +struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::minus instead.") Difference +{ + template + HIPCUB_HOST_DEVICE + inline constexpr auto operator()(T&& t, U&& u) const -> decltype(auto) + { + return ::cuda::std::forward(t) - ::cuda::std::forward(u); + } +}; + +//! deprecated [Since 5.0] +struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::divides instead") Division +{ + template + HIPCUB_HOST_DEVICE + inline constexpr auto operator()(T&& t, U&& u) const -> decltype(auto) + { + return std::forward(t) / std::forward(u); + } +}; + +//! deprecated [Since 5.0] +struct HIPCUB_DEPRECATED_BECAUSE("Use hip::maximum instead.") Max +{ + template + HIPCUB_HOST_DEVICE + inline constexpr auto operator()(const T& t, const U& u) const -> + typename ::cuda::std::common_type::type + { + using R = typename ::cuda::std::common_type::type; + return (t < u) ? static_cast(u) : static_cast(t); + } +}; + +//! deprecated [Since 5.0] +struct HIPCUB_DEPRECATED_BECAUSE("Use hip::minimum instead") Min +{ + template + HIPCUB_HOST_DEVICE + inline constexpr auto operator()(const T& t, const U& u) const -> + typename ::cuda::std::common_type::type + { + using R = typename ::cuda::std::common_type::type; + return (u < t) ? static_cast(u) : static_cast(t); + } +}; + +struct ArgMax +{ + template + HIPCUB_HOST_DEVICE + inline constexpr ::cub::KeyValuePair + operator()(const ::cub::KeyValuePair& a, + const ::cub::KeyValuePair& b) const + { + return ((b.value > a.value) || ((a.value == b.value) && (b.key < a.key))) ? b : a; + } +}; + +struct ArgMin +{ + template + HIPCUB_HOST_DEVICE + inline constexpr ::cub::KeyValuePair + operator()(const ::cub::KeyValuePair& a, + const ::cub::KeyValuePair& b) const + { + return ((b.value < a.value) || ((a.value == b.value) && (b.key < a.key))) ? b : a; + } +}; + END_HIPCUB_NAMESPACE #endif // HIPCUB_CUB_THREAD_THREAD_OPERATORS_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/thread/thread_store.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/thread/thread_store.hpp new file mode 100644 index 000000000000..02a260e5c5f5 --- /dev/null +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/thread/thread_store.hpp @@ -0,0 +1,134 @@ +// Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +#ifndef HIPCUB_BACKEND_CUB_THREAD_STORE_HPP_ +#define HIPCUB_BACKEND_CUB_THREAD_STORE_HPP_ + +#include "../../../config.hpp" +#include "../util_type.hpp" + +#include // CUB thread store + +#include +#include + +BEGIN_HIPCUB_NAMESPACE + +enum CacheStoreModifier +{ + STORE_DEFAULT = 0, + STORE_WB = 1, + STORE_CG = 2, + STORE_CS = 3, + STORE_WT = 4, + STORE_VOLATILE = 5 +}; + +template +struct cub_cache_store_modifier_map +{ + static constexpr ::cub::CacheStoreModifier value = static_cast<::cub::CacheStoreModifier>(MOD); +}; + +template +HIPCUB_DEVICE +HIPCUB_FORCEINLINE void ThreadStoreVolatilePtr(T* ptr, T val, Fundamental /*is_fundamental*/) +{ + ::cub::ThreadStore<::cub::STORE_VOLATILE>(ptr, val); +} + +template +HIPCUB_DEVICE +HIPCUB_FORCEINLINE void ThreadStore(T* ptr, + T val, + ::std::integral_constant /*modifier*/, + ::std::true_type /*is_pointer*/) +{ + ::cub::ThreadStore::value>(ptr, val); +} + +template +HIPCUB_DEVICE +HIPCUB_FORCEINLINE void ThreadStore(OutputIteratorT itr, + T val, + ::std::integral_constant /*modifier*/, + ::std::false_type /*is_pointer*/) +{ + ThreadStore(&(*itr), + val, + ::std::integral_constant{}, + ::std::true_type{}); +} + +template +HIPCUB_DEVICE +HIPCUB_FORCEINLINE void ThreadStore(OutputIteratorT itr, T val) +{ + ThreadStore(itr, + val, + ::std::integral_constant{}, + ::std::bool_constant<_HIPCUB_STD::is_pointer::value>()); +} + +namespace detail +{ + +template +struct iterate_thread_store +{ + template + static HIPCUB_DEVICE + HIPCUB_FORCEINLINE void Store(T* ptr, T* vals) + { + ThreadStore(ptr + COUNT, + vals[COUNT], + ::std::integral_constant{}, + ::std::true_type{}); + iterate_thread_store::template Store(ptr, vals); + } + + template + static HIPCUB_DEVICE + HIPCUB_FORCEINLINE void Dereference(OutputIteratorT ptr, T* vals) + { + ptr[COUNT] = vals[COUNT]; + iterate_thread_store::Dereference(ptr, vals); + } +}; + +template +struct iterate_thread_store +{ + template + static HIPCUB_DEVICE + HIPCUB_FORCEINLINE void Store(T* /*ptr*/, T* /*vals*/) + {} + + template + static HIPCUB_DEVICE + HIPCUB_FORCEINLINE void Dereference(OutputIteratorT /*ptr*/, T* /*vals*/) + {} +}; + +} // namespace detail + +END_HIPCUB_NAMESPACE + +#endif // HIPCUB_BACKEND_CUB_THREAD_STORE_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/util_macro.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/util_macro.hpp index 794d527aade1..2489e3af162f 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/util_macro.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/util_macro.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -34,44 +34,4 @@ #include // IWYU pragma: export -BEGIN_HIPCUB_NAMESPACE - -/// Deprecated since rocm [7.1] -#ifndef HIPCUB_MAX - /// Select maximum(a, b) - #define HIPCUB_MAX(a, b) (((b) > (a)) ? (b) : (a)) -#endif - -/// Deprecated since rocm [7.1] -#ifndef HIPCUB_MIN - /// Select minimum(a, b) - #define HIPCUB_MIN(a, b) (((b) < (a)) ? (b) : (a)) -#endif - -/// Deprecated since rocm [7.1] -#ifndef HIPCUB_QUOTIENT_FLOOR - /// Quotient of x/y rounded down to nearest integer - #define HIPCUB_QUOTIENT_FLOOR(x, y) ((x) / (y)) -#endif - -/// Deprecated since rocm [7.1] -#ifndef HIPCUB_QUOTIENT_CEILING - /// Quotient of x/y rounded up to nearest integer - #define HIPCUB_QUOTIENT_CEILING(x, y) (((x) + (y)-1) / (y)) -#endif - -/// Deprecated since rocm [7.1] -#ifndef HIPCUB_ROUND_UP_NEAREST - /// x rounded up to the nearest multiple of y - #define HIPCUB_ROUND_UP_NEAREST(x, y) (HIPCUB_QUOTIENT_CEILING(x, y) * y) -#endif - -/// Deprecated since rocm [7.1] -#ifndef HIPCUB_ROUND_DOWN_NEAREST - /// x rounded down to the nearest multiple of y - #define HIPCUB_ROUND_DOWN_NEAREST(x, y) (((x) / (y)) * y) -#endif - -END_HIPCUB_NAMESPACE - #endif // HIPCUB_CUB_MACRO_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/util_temporary_storage.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/util_temporary_storage.hpp index fc67d645b142..daf1e265b1d1 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/cub/util_temporary_storage.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/util_temporary_storage.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2024, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -34,39 +34,4 @@ #include // IWYU pragma: export -BEGIN_HIPCUB_NAMESPACE - -/// \brief Alias temporaries to externally-allocated device storage (or simply return the amount of storage needed). -/// \tparam ALLOCATIONS The number of allocations that are needed. -/// \param d_temp_storage [in] Device-accessible allocation of temporary storage. When nullptr, the required allocation size is written to \p temp_storage_bytes and no work is done. -/// \param temp_storage_bytes [in,out] Size in bytes of \t d_temp_storage allocation. -/// \param allocations [out] Pointers to device allocations needed. -/// \param allocation_sizes [in] Sizes in bytes of device allocations needed. -template -HIPCUB_DEPRECATED_BECAUSE("Internal-only implementation detail") -HIPCUB_HOST_DEVICE HIPCUB_FORCEINLINE hipError_t - AliasTemporaries(void* d_temp_storage, - size_t& temp_storage_bytes, - void* (&allocations)[ALLOCATIONS], - const size_t (&allocation_sizes)[ALLOCATIONS]) -{ - cudaError_t error = ::cub::detail::AliasTemporaries(d_temp_storage, - temp_storage_bytes, - allocations, - allocation_sizes); - - if(cudaSuccess == error) - { - return hipSuccess; - } - else if(cudaErrorInvalidValue == error) - { - return hipErrorInvalidValue; - } - - return hipErrorUnknown; -} - -END_HIPCUB_NAMESPACE - #endif // HIPCUB_CUB_UTIL_TEMPORARY_STORAGE_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/cub/util_type.hpp b/projects/hipcub/hipcub/include/hipcub/backend/cub/util_type.hpp new file mode 100644 index 000000000000..3ec0ebd64dbc --- /dev/null +++ b/projects/hipcub/hipcub/include/hipcub/backend/cub/util_type.hpp @@ -0,0 +1,93 @@ +/****************************************************************************** + * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. + * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of the NVIDIA CORPORATION nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY + * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + ******************************************************************************/ + +#ifndef HIPCUB_CUB_UTIL_TYPE_HPP_ +#define HIPCUB_CUB_UTIL_TYPE_HPP_ + +#include "../../config.hpp" +#include "../../util_deprecated.hpp" + +#include _HIPCUB_STD_INCLUDE(iterator) +#include _HIPCUB_STD_INCLUDE(type_traits) + +#include // IWYU pragma: export + +BEGIN_HIPCUB_NAMESPACE + +namespace detail +{ +// the following iterator helpers are not named iter_value_t etc, like the C++20 facilities, because they are defined in +// terms of C++17 iterator_traits and not the new C++20 indirectly_readable trait etc. This allows them to detect nested +// value_type, difference_type and reference aliases, which the new C+20 traits do not consider (they only consider +// specializations of iterator_traits). Also, a value_type of void remains supported (needed by some output iterators). + +template +struct it_traits +{ + using value_type = typename _HIPCUB_STD::iterator_traits::value_type; + using reference = typename _HIPCUB_STD::iterator_traits::reference; + using difference_type = typename _HIPCUB_STD::iterator_traits::difference_type; + using pointer = typename _HIPCUB_STD::iterator_traits::pointer; +}; +template +struct it_traits> +{ + using value_type = typename It::value_type; + using reference = typename It::reference; + using difference_type = typename It::difference_type; + using pointer = typename It::pointer; +}; +template +using it_value_t = typename it_traits::value_type; +template +using it_reference_t = typename it_traits::reference; +template +using it_difference_t = typename it_traits::difference_type; +template +using it_pointer_t = typename it_traits::pointer; + +// use this whenever you need to lazily evaluate a trait. E.g., as an alternative in replace_if_use_default. +template typename Trait, typename... Args> +struct lazy_trait +{ + using type = Trait; +}; + +template +using int_constant_t = _HIPCUB_STD::integral_constant; + +} // namespace detail + +END_HIPCUB_NAMESPACE + +#endif // HIPCUB_CUB_UTIL_TYPE_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/agent/single_pass_scan_operators.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/agent/single_pass_scan_operators.hpp index 837ef4b1a98c..59060b602f8e 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/agent/single_pass_scan_operators.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/agent/single_pass_scan_operators.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -336,13 +336,11 @@ class ScanTileState * \tparam T Type of the values scanned. * \tparam ScanOpT Scan operation type. * \tparam ScanTileStateT Scan status type. - * \tparam LEGACY_PTX_ARCH [optional] Unused (deprecated). * \tparam DelayConstructorT [optional] Unused (CUB's implementation detail). */ template */> class TilePrefixCallbackOp { diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_merge_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_merge_sort.hpp index d306c834e3aa..29d29dee9391 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_merge_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_merge_sort.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. -* Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. +* Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,6 +32,7 @@ #include "../../../config.hpp" #include "../thread/thread_sort.hpp" +#include "../util_macro.hpp" #include "../util_math.hpp" #include "../util_type.hpp" @@ -95,7 +96,7 @@ HIPCUB_DEVICE __forceinline__ void SerialMerge(KeyT *keys_shared, KeyT key1 = keys_shared[keys1_beg]; KeyT key2 = keys_shared[keys2_beg]; -#pragma unroll + _CCCL_SORT_MAYBE_UNROLL() for (int item = 0; item < ITEMS_PER_THREAD; ++item) { bool p = (keys2_beg < keys2_end) && @@ -387,7 +388,7 @@ class BlockMergeSortStrategy // KeyT max_key = oob_default; - #pragma unroll + _CCCL_SORT_MAYBE_UNROLL() for (int item = WARP_SORT ? 1 : 0; item < ITEMS_PER_THREAD; ++item) { if (ITEMS_PER_THREAD * static_cast(linear_tid) + item < valid_items) @@ -411,7 +412,7 @@ class BlockMergeSortStrategy // each thread has sorted keys // merge sort keys in shared memory // - #pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for (int target_merged_threads_number = 2; target_merged_threads_number <= NUM_THREADS; target_merged_threads_number *= 2) @@ -423,7 +424,7 @@ class BlockMergeSortStrategy // store keys in shmem // - #pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for (int item = 0; item < ITEMS_PER_THREAD; ++item) { int idx = ITEMS_PER_THREAD * linear_tid + item; @@ -482,7 +483,7 @@ class BlockMergeSortStrategy // store keys in shmem // - #pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for (int item = 0; item < ITEMS_PER_THREAD; ++item) { int idx = ITEMS_PER_THREAD * linear_tid + item; @@ -493,7 +494,7 @@ class BlockMergeSortStrategy // gather items from shmem // - #pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for (int item = 0; item < ITEMS_PER_THREAD; ++item) { items[item] = temp_storage.items_shared[indices[item]]; diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_radix_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_radix_sort.hpp index 598f39a6ebee..b4ad51da814d 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_radix_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_radix_sort.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -74,7 +74,7 @@ constexpr auto tuple_bit_size_impl() template struct tuple_bit_size<::hipcub::tuple> - : public std::integral_constant, 0>()> + : public std::integral_constant, 0>()> {}; } // namespace detail diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_run_length_decode.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_run_length_decode.hpp index 8296c3a37a1b..39fe245f8b9c 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_run_length_decode.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_run_length_decode.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -37,7 +37,8 @@ #include "../util_type.hpp" #include "block_scan.hpp" -#include +#include _HIPCUB_STD_INCLUDE(limits) + #include BEGIN_HIPCUB_NAMESPACE @@ -247,7 +248,7 @@ class BlockRunLengthDecode { OffsetT lower_bound = 0; OffsetT upper_bound = num_items; - #pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for (int i = 0; i <= Log2::VALUE; i++) { OffsetT mid = hipcub::MidPoint(lower_bound, upper_bound); @@ -272,7 +273,7 @@ class BlockRunLengthDecode { // Keep the runs' items and the offsets of each run's beginning in the temporary storage RunOffsetT thread_dst_offset = static_cast(linear_tid) * static_cast(RUNS_PER_THREAD); - #pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for (int i = 0; i < RUNS_PER_THREAD; i++) { temp_storage.runs.run_values[thread_dst_offset] = run_values[i]; @@ -291,7 +292,7 @@ class BlockRunLengthDecode { // Compute the offset for the beginning of each run DecodedOffsetT run_offsets[RUNS_PER_THREAD]; - #pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for (int i = 0; i < RUNS_PER_THREAD; i++) { run_offsets[i] = static_cast(run_lengths[i]); @@ -347,7 +348,7 @@ class BlockRunLengthDecode ItemT val = temp_storage.runs.run_values[assigned_run]; - #pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for (DecodedOffsetT i = 0; i < DECODED_ITEMS_PER_THREAD; i++) { decoded_items[i] = val; diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_scan.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_scan.hpp index 3111e8224f2d..a4f406252298 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_scan.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_scan.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,10 +32,10 @@ #include "../../../config.hpp" -#include "../thread/thread_operators.hpp" - #include // IWYU pragma: export +#include _HIPCUB_STD_INCLUDE(functional) + #include BEGIN_HIPCUB_NAMESPACE @@ -121,12 +121,14 @@ class BlockScan } template - HIPCUB_DEVICE inline - void InclusiveSum(T input, T& output, BlockPrefixCallbackOp& block_prefix_callback_op) + HIPCUB_DEVICE + inline void InclusiveSum(T input, T& output, BlockPrefixCallbackOp& block_prefix_callback_op) { - base_type::inclusive_scan( - input, output, temp_storage_, block_prefix_callback_op, ::hipcub::Sum() - ); + base_type::inclusive_scan(input, + output, + temp_storage_, + block_prefix_callback_op, + _HIPCUB_STD::plus<>{}); } template @@ -145,13 +147,16 @@ class BlockScan } template - HIPCUB_DEVICE inline - void InclusiveSum(T(&input)[ITEMS_PER_THREAD], T(&output)[ITEMS_PER_THREAD], - BlockPrefixCallbackOp& block_prefix_callback_op) + HIPCUB_DEVICE + inline void InclusiveSum(T (&input)[ITEMS_PER_THREAD], + T (&output)[ITEMS_PER_THREAD], + BlockPrefixCallbackOp& block_prefix_callback_op) { - base_type::inclusive_scan( - input, output, temp_storage_, block_prefix_callback_op, ::hipcub::Sum() - ); + base_type::inclusive_scan(input, + output, + temp_storage_, + block_prefix_callback_op, + _HIPCUB_STD::plus<>{}); } template @@ -241,12 +246,14 @@ class BlockScan } template - HIPCUB_DEVICE inline - void ExclusiveSum(T input, T& output, BlockPrefixCallbackOp& block_prefix_callback_op) + HIPCUB_DEVICE + inline void ExclusiveSum(T input, T& output, BlockPrefixCallbackOp& block_prefix_callback_op) { - base_type::exclusive_scan( - input, output, temp_storage_, block_prefix_callback_op, ::hipcub::Sum() - ); + base_type::exclusive_scan(input, + output, + temp_storage_, + block_prefix_callback_op, + _HIPCUB_STD::plus<>{}); } template @@ -265,13 +272,16 @@ class BlockScan } template - HIPCUB_DEVICE inline - void ExclusiveSum(T(&input)[ITEMS_PER_THREAD], T(&output)[ITEMS_PER_THREAD], - BlockPrefixCallbackOp& block_prefix_callback_op) + HIPCUB_DEVICE + inline void ExclusiveSum(T (&input)[ITEMS_PER_THREAD], + T (&output)[ITEMS_PER_THREAD], + BlockPrefixCallbackOp& block_prefix_callback_op) { - base_type::exclusive_scan( - input, output, temp_storage_, block_prefix_callback_op, ::hipcub::Sum() - ); + base_type::exclusive_scan(input, + output, + temp_storage_, + block_prefix_callback_op, + _HIPCUB_STD::plus<>{}); } template diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_shuffle.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_shuffle.hpp index 8698d5767540..2c591898dd0a 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_shuffle.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/block_shuffle.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,8 +32,6 @@ #include "../../../config.hpp" -#include "../thread/thread_operators.hpp" - #include // IWYU pragma: export #include diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/radix_rank_sort_operations.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/radix_rank_sort_operations.hpp index 0ead22896253..4cb0033ff10e 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/radix_rank_sort_operations.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/block/radix_rank_sort_operations.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2011-2020, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -36,12 +36,17 @@ #define HIPCUB_ROCPRIM_BLOCK_RADIX_RANK_SORT_OPERATIONS_HPP_ #include "../../../config.hpp" +#include "../../../libcxx.hpp" #include "../util_type.hpp" #include // IWYU pragma: export #include // IWYU pragma: export #include // IWYU pragma: export +#include _HIPCUB_LIBCXX_INCLUDE(bit) + +#include + BEGIN_HIPCUB_NAMESPACE /** \brief Twiddling keys for radix sort. */ @@ -92,8 +97,9 @@ struct RadixSortTwiddle enum { - HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH FLOAT_KEY = TraitsT::CATEGORY == FLOATING_POINT, - HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP + FLOAT_KEY = _HIPCUB_STD::is_floating_point_v + || std::is_same_v + || std::is_same_v, }; static __device__ __forceinline__ UnsignedBits ProcessFloatMinusZero(UnsignedBits key) @@ -124,7 +130,19 @@ struct RadixSortTwiddle __device__ __forceinline__ uint32_t Digit(UnsignedBits key) { - return BFE(this->ProcessFloatMinusZero(key), bit_start, num_bits); + HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH + + uint32_t result = +#if _HIPCUB_HAS_DEVICE_SYSTEM_STD + _HIPCUB_LIBCXX::bitfield_extract +#else + BFE +#endif + (this->ProcessFloatMinusZero(key), bit_start, num_bits); + + HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP + + return result; } }; diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_adjacent_difference.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_adjacent_difference.hpp index e66f8b4bf9c6..5bf19740a626 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_adjacent_difference.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_adjacent_difference.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2022-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2022-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,24 +32,26 @@ #include "../../../config.hpp" #include "../../../util_deprecated.hpp" -#include #include // IWYU pragma: export +#include _HIPCUB_STD_INCLUDE(functional) + BEGIN_HIPCUB_NAMESPACE struct DeviceAdjacentDifference { template - static HIPCUB_RUNTIME_FUNCTION hipError_t SubtractLeftCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - InputIteratorT d_input, - OutputIteratorT d_output, - NumItemsT num_items, - DifferenceOpT difference_op = {}, - hipStream_t stream = 0) + typename DifferenceOpT = _HIPCUB_STD::minus<>, + typename NumItemsT = uint32_t> + static HIPCUB_RUNTIME_FUNCTION + hipError_t SubtractLeftCopy(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_input, + OutputIteratorT d_output, + NumItemsT num_items, + DifferenceOpT difference_op = {}, + hipStream_t stream = 0) { return ::rocprim::adjacent_difference(d_temp_storage, temp_storage_bytes, @@ -63,17 +65,17 @@ struct DeviceAdjacentDifference template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION hipError_t - SubtractLeftCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - InputIteratorT d_input, - OutputIteratorT d_output, - NumItemsT num_items, - DifferenceOpT difference_op, - hipStream_t stream, - bool debug_synchronous) + typename DifferenceOpT = _HIPCUB_STD::minus<>, + typename NumItemsT = uint32_t> + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION + hipError_t SubtractLeftCopy(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_input, + OutputIteratorT d_output, + NumItemsT num_items, + DifferenceOpT difference_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SubtractLeftCopy(d_temp_storage, @@ -86,14 +88,15 @@ struct DeviceAdjacentDifference } template - static HIPCUB_RUNTIME_FUNCTION hipError_t SubtractLeft(void* d_temp_storage, - std::size_t& temp_storage_bytes, - RandomAccessIteratorT d_input, - NumItemsT num_items, - DifferenceOpT difference_op = {}, - hipStream_t stream = 0) + typename DifferenceOpT = _HIPCUB_STD::minus<>, + typename NumItemsT = uint32_t> + static HIPCUB_RUNTIME_FUNCTION + hipError_t SubtractLeft(void* d_temp_storage, + size_t& temp_storage_bytes, + RandomAccessIteratorT d_input, + NumItemsT num_items, + DifferenceOpT difference_op = {}, + hipStream_t stream = 0) { return ::rocprim::adjacent_difference_inplace(d_temp_storage, temp_storage_bytes, @@ -105,16 +108,16 @@ struct DeviceAdjacentDifference } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION hipError_t - SubtractLeft(void* d_temp_storage, - std::size_t& temp_storage_bytes, - RandomAccessIteratorT d_input, - NumItemsT num_items, - DifferenceOpT difference_op, - hipStream_t stream, - bool debug_synchronous) + typename DifferenceOpT = _HIPCUB_STD::minus<>, + typename NumItemsT = uint32_t> + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION + hipError_t SubtractLeft(void* d_temp_storage, + size_t& temp_storage_bytes, + RandomAccessIteratorT d_input, + NumItemsT num_items, + DifferenceOpT difference_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SubtractLeft(d_temp_storage, @@ -127,15 +130,16 @@ struct DeviceAdjacentDifference template - static HIPCUB_RUNTIME_FUNCTION hipError_t SubtractRightCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - InputIteratorT d_input, - OutputIteratorT d_output, - NumItemsT num_items, - DifferenceOpT difference_op = {}, - hipStream_t stream = 0) + typename DifferenceOpT = _HIPCUB_STD::minus<>, + typename NumItemsT = uint32_t> + static HIPCUB_RUNTIME_FUNCTION + hipError_t SubtractRightCopy(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_input, + OutputIteratorT d_output, + NumItemsT num_items, + DifferenceOpT difference_op = {}, + hipStream_t stream = 0) { return ::rocprim::adjacent_difference_right(d_temp_storage, temp_storage_bytes, @@ -149,17 +153,17 @@ struct DeviceAdjacentDifference template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION hipError_t - SubtractRightCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - InputIteratorT d_input, - OutputIteratorT d_output, - NumItemsT num_items, - DifferenceOpT difference_op, - hipStream_t stream, - bool debug_synchronous) + typename DifferenceOpT = _HIPCUB_STD::minus<>, + typename NumItemsT = uint32_t> + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION + hipError_t SubtractRightCopy(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_input, + OutputIteratorT d_output, + NumItemsT num_items, + DifferenceOpT difference_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SubtractRightCopy(d_temp_storage, @@ -172,14 +176,15 @@ struct DeviceAdjacentDifference } template - static HIPCUB_RUNTIME_FUNCTION hipError_t SubtractRight(void* d_temp_storage, - std::size_t& temp_storage_bytes, - RandomAccessIteratorT d_input, - NumItemsT num_items, - DifferenceOpT difference_op = {}, - hipStream_t stream = 0) + typename DifferenceOpT = _HIPCUB_STD::minus<>, + typename NumItemsT = uint32_t> + static HIPCUB_RUNTIME_FUNCTION + hipError_t SubtractRight(void* d_temp_storage, + size_t& temp_storage_bytes, + RandomAccessIteratorT d_input, + NumItemsT num_items, + DifferenceOpT difference_op = {}, + hipStream_t stream = 0) { return ::rocprim::adjacent_difference_right_inplace(d_temp_storage, temp_storage_bytes, @@ -191,16 +196,16 @@ struct DeviceAdjacentDifference } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION hipError_t - SubtractRight(void* d_temp_storage, - std::size_t& temp_storage_bytes, - RandomAccessIteratorT d_input, - NumItemsT num_items, - DifferenceOpT difference_op, - hipStream_t stream, - bool debug_synchronous) + typename DifferenceOpT = _HIPCUB_STD::minus<>, + typename NumItemsT = uint32_t> + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS static HIPCUB_RUNTIME_FUNCTION + hipError_t SubtractRight(void* d_temp_storage, + size_t& temp_storage_bytes, + RandomAccessIteratorT d_input, + NumItemsT num_items, + DifferenceOpT difference_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SubtractRight(d_temp_storage, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_copy.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_copy.hpp index 8e41fa0fd330..68f2f2744ec6 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_copy.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_copy.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,13 +40,13 @@ BEGIN_HIPCUB_NAMESPACE struct DeviceCopy { template - static hipError_t Batched(void* d_temp_storage, - size_t& temp_storage_bytes, - InputBufferIt input_buffer_it, - OutputBufferIt output_buffer_it, - BufferSizeIteratorT buffer_sizes, - uint32_t num_buffers, - hipStream_t stream = 0) + static hipError_t Batched(void* d_temp_storage, + size_t& temp_storage_bytes, + InputBufferIt input_buffer_it, + OutputBufferIt output_buffer_it, + BufferSizeIteratorT buffer_sizes, + _HIPCUB_STD::int64_t num_buffers, + hipStream_t stream = 0) { return rocprim::batch_copy(d_temp_storage, temp_storage_bytes, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_for.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_for.hpp index 682eb673a388..32a672b5d4c7 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_for.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_for.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,12 +31,11 @@ #include "../../../config.hpp" -#include "../iterator/counting_input_iterator.hpp" -#include "../iterator/discard_output_iterator.hpp" -#include "../thread/thread_operators.hpp" #include "../util_mdspan.hpp" #include // IWYU pragma: export +#include // IWYU pragma: export +#include // IWYU pragma: export #include @@ -117,15 +116,14 @@ struct DeviceFor OpT op, hipStream_t stream = 0) -> std::enable_if_t()), - typename std::iterator_traits< - RandomAccessIteratorT>::value_type>::value, + detail::it_value_t>::value, hipError_t> { - using T = typename std::iterator_traits::value_type; - using OutputIterator = typename rocprim::discard_iterator; + using T = detail::it_value_t; + detail::bulk::OpWrapper wrapper_op = {op}; - OutputIterator output; + auto output = rocprim::make_discard_iterator(); return rocprim::transform(first, output, @@ -137,14 +135,15 @@ struct DeviceFor template HIPCUB_RUNTIME_FUNCTION - static auto - ForEachN(RandomAccessIteratorT first, OffsetT num_items, OpT op, hipStream_t stream = 0) - -> std::enable_if_t()), - typename std::iterator_traits< - RandomAccessIteratorT>::value_type>::value, - hipError_t> + static auto ForEachN(RandomAccessIteratorT first, + OffsetT num_items, + OpT op, + hipStream_t stream = 0) + -> std::enable_if_t()), + detail::it_value_t>::value, + hipError_t> { - using T = typename std::iterator_traits::value_type; + using T = detail::it_value_t; detail::bulk::OpWrapper wrapper_op = {op}; @@ -207,7 +206,7 @@ HIPCUB_RUNTIME_FUNCTION OpT op, hipStream_t stream = 0) { - using offset_t = typename std::iterator_traits::difference_type; + using offset_t = detail::it_difference_t; const offset_t num_items = static_cast(std::distance(first, last)); return ForEachN(first, num_items, op, stream); @@ -265,11 +264,12 @@ HIPCUB_RUNTIME_FUNCTION { static_assert(std::is_integral::value, "ShapeT must be an integral type"); using InputIterator = typename rocprim::counting_iterator; - using OutputIterator = typename rocprim::discard_iterator; + detail::bulk::OpWrapper wrapper_op = {op}; InputIterator input(ShapeT(0)); - OutputIterator output; + + auto output = rocprim::make_discard_iterator(); return rocprim::transform(input, output, @@ -350,14 +350,13 @@ HIPCUB_RUNTIME_FUNCTION // rocprim::counting_iterator only holds the index, not the data. using InputIterator = typename rocprim::counting_iterator; - // We don't actually need the output, so we use rocprim::discard_iterator here as a placeholder. - using OutputIterator = typename rocprim::discard_iterator; // How many times rocprim::transform will iterate. constexpr auto ext_size = ::hipcub::extents_size::value; InputIterator input(IndexType(0)); // Initialize the input iterator, starting from 0. - OutputIterator output; + + auto output = rocprim::make_discard_iterator(); // `ForEachInExtents` only iterates over the extents on device and does not guarantee ordering. // We only need to invoke `$op` `$ext_size` times. Therefore, `rocprim::transform` is suitable. diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_histogram.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_histogram.hpp index 0dd5d978cb34..ab51c2e02acd 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_histogram.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_histogram.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -61,24 +61,25 @@ HIPCUB_FORCEINLINE bool may_overflow(LevelT lower_level, ::std::true_type /* is_integral */) { return static_cast(upper_level - lower_level) - > (::std::numeric_limits::max() / static_cast(num_bins)); + > (_HIPCUB_STD::numeric_limits::max() + / static_cast(num_bins)); } template struct int_arithmetic_t { - using type = ::std::conditional_t< - sizeof(SampleT) + sizeof(CommonT) <= sizeof(uint32_t), - uint32_t, -#if HIPCUB_IS_INT128_ENABLED - ::std::conditional_t<(::std::is_same::value - || ::std::is_same::value), - CommonT, - uint64_t> + using type + = ::std::conditional_t + || ::std::is_same_v), + CommonT, + uint64_t> #else - uint64_t + uint64_t #endif - >; + >; }; // If potential overflow is detected, returns hipErrorInvalidValue, otherwise hipSuccess. @@ -86,7 +87,7 @@ template HIPCUB_HOST_DEVICE HIPCUB_FORCEINLINE hipError_t check_overflow(LevelT lower_level, LevelT upper_level, int num_levels) { - using sample_type = typename std::iterator_traits::value_type; + using sample_type = it_value_t; using common_type = typename hipcub::common_type::type; static_assert(std::is_convertible::value, "The common type of `LevelT` and `SampleT` must be " diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_memcpy.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_memcpy.hpp index a5828b7ad4d5..36dac611cff8 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_memcpy.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_memcpy.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2011-2022, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2023-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2023-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,14 +40,20 @@ BEGIN_HIPCUB_NAMESPACE struct DeviceMemcpy { template - static hipError_t Batched(void* d_temp_storage, - size_t& temp_storage_bytes, - InputBufferIt input_buffer_it, - OutputBufferIt output_buffer_it, - BufferSizeIteratorT buffer_sizes, - uint32_t num_buffers, - hipStream_t stream = 0) + static hipError_t Batched(void* d_temp_storage, + size_t& temp_storage_bytes, + InputBufferIt input_buffer_it, + OutputBufferIt output_buffer_it, + BufferSizeIteratorT buffer_sizes, + _HIPCUB_STD::int64_t num_buffers, + hipStream_t stream = 0) { + if(num_buffers == 0) + { + temp_storage_bytes = 0; + return hipSuccess; + } + return rocprim::batch_memcpy(d_temp_storage, temp_storage_bytes, input_buffer_it, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_merge.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_merge.hpp index 014695e5ba01..dc80f72a2491 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_merge.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_merge.hpp @@ -1,6 +1,6 @@ /****************************************************************************** - * Copyright (c) 2025, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2025, Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2025-2026, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2025-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -43,15 +43,15 @@ struct DeviceMerge typename KeyIteratorOut, typename CompareOp = ::rocprim::less<>> HIPCUB_RUNTIME_FUNCTION - static hipError_t MergeKeys(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorIn1 keys_in1, - int num_keys1, - KeyIteratorIn2 keys_in2, - int num_keys2, - KeyIteratorOut keys_out, - CompareOp compare_op = {}, - hipStream_t stream = 0) + static hipError_t MergeKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyIteratorIn1 keys_in1, + _HIPCUB_STD::int64_t num_keys1, + KeyIteratorIn2 keys_in2, + _HIPCUB_STD::int64_t num_keys2, + KeyIteratorOut keys_out, + CompareOp compare_op = {}, + hipStream_t stream = 0) { return ::rocprim::merge(d_temp_storage, @@ -74,18 +74,18 @@ struct DeviceMerge typename ValueIteratorOut, typename CompareOp = ::rocprim::less<>> HIPCUB_RUNTIME_FUNCTION - static hipError_t MergePairs(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorIn1 keys_in1, - ValueIteratorIn1 values_in1, - int num_keys1, - KeyIteratorIn2 keys_in2, - ValueIteratorIn2 values_in2, - int num_keys2, - KeyIteratorOut keys_out, - ValueIteratorOut values_out, - CompareOp compare_op = {}, - hipStream_t stream = 0) + static hipError_t MergePairs(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyIteratorIn1 keys_in1, + ValueIteratorIn1 values_in1, + _HIPCUB_STD::int64_t num_keys1, + KeyIteratorIn2 keys_in2, + ValueIteratorIn2 values_in2, + _HIPCUB_STD::int64_t num_keys2, + KeyIteratorOut keys_out, + ValueIteratorOut values_out, + CompareOp compare_op = {}, + hipStream_t stream = 0) { return ::rocprim::merge(d_temp_storage, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_merge_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_merge_sort.hpp index 0c9f4df4d82a..5bf1e08e9e96 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_merge_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_merge_sort.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -42,13 +42,14 @@ BEGIN_HIPCUB_NAMESPACE struct DeviceMergeSort { template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairs(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorT d_keys, - ValueIteratorT d_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyIteratorT d_keys, + ValueIteratorT d_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return ::rocprim::merge_sort(d_temp_storage, temp_storage_bytes, @@ -63,15 +64,15 @@ struct DeviceMergeSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairs(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorT d_keys, - ValueIteratorT d_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyIteratorT d_keys, + ValueIteratorT d_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortPairs(d_temp_storage, @@ -89,15 +90,16 @@ struct DeviceMergeSort typename ValueIteratorT, typename OffsetT, typename CompareOpT> - HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairsCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - ValueInputIteratorT d_input_items, - KeyIteratorT d_output_keys, - ValueIteratorT d_output_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairsCopy(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + ValueInputIteratorT d_input_items, + KeyIteratorT d_output_keys, + ValueIteratorT d_output_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return ::rocprim::merge_sort(d_temp_storage, temp_storage_bytes, @@ -117,17 +119,17 @@ struct DeviceMergeSort typename ValueIteratorT, typename OffsetT, typename CompareOpT> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairsCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - ValueInputIteratorT d_input_items, - KeyIteratorT d_output_keys, - ValueIteratorT d_output_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairsCopy(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + ValueInputIteratorT d_input_items, + KeyIteratorT d_output_keys, + ValueIteratorT d_output_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortPairsCopy(d_temp_storage, @@ -142,12 +144,13 @@ struct DeviceMergeSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorT d_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyIteratorT d_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return ::rocprim::merge_sort(d_temp_storage, temp_storage_bytes, @@ -160,14 +163,14 @@ struct DeviceMergeSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeys(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorT d_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyIteratorT d_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortKeys(d_temp_storage, temp_storage_bytes, d_keys, num_items, compare_op, stream); @@ -177,13 +180,14 @@ struct DeviceMergeSort typename KeyIteratorT, typename OffsetT, typename CompareOpT> - HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeysCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - KeyIteratorT d_output_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeysCopy(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + KeyIteratorT d_output_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return ::rocprim::merge_sort(d_temp_storage, @@ -200,15 +204,15 @@ struct DeviceMergeSort typename KeyIteratorT, typename OffsetT, typename CompareOpT> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeysCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - KeyIteratorT d_output_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeysCopy(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + KeyIteratorT d_output_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); @@ -222,13 +226,14 @@ struct DeviceMergeSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortPairs(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorT d_keys, - ValueIteratorT d_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyIteratorT d_keys, + ValueIteratorT d_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return ::rocprim::merge_sort(d_temp_storage, temp_storage_bytes, @@ -243,15 +248,15 @@ struct DeviceMergeSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortPairs(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorT d_keys, - ValueIteratorT d_items, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyIteratorT d_keys, + ValueIteratorT d_items, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortPairs(d_temp_storage, @@ -264,12 +269,13 @@ struct DeviceMergeSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortKeys(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorT d_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyIteratorT d_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return ::rocprim::merge_sort(d_temp_storage, temp_storage_bytes, @@ -282,14 +288,14 @@ struct DeviceMergeSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortKeys(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyIteratorT d_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyIteratorT d_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortKeys(d_temp_storage, @@ -304,13 +310,14 @@ struct DeviceMergeSort typename KeyIteratorT, typename OffsetT, typename CompareOpT> - HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortKeysCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - KeyIteratorT d_output_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeysCopy(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + KeyIteratorT d_output_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream = 0) { return ::rocprim::merge_sort(d_temp_storage, temp_storage_bytes, @@ -326,15 +333,15 @@ struct DeviceMergeSort typename KeyIteratorT, typename OffsetT, typename CompareOpT> - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortKeysCopy(void* d_temp_storage, - std::size_t& temp_storage_bytes, - KeyInputIteratorT d_input_keys, - KeyIteratorT d_output_keys, - OffsetT num_items, - CompareOpT compare_op, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeysCopy(void* d_temp_storage, + size_t& temp_storage_bytes, + KeyInputIteratorT d_input_keys, + KeyIteratorT d_output_keys, + OffsetT num_items, + CompareOpT compare_op, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortKeysCopy(d_temp_storage, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_partition.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_partition.hpp index 55ca243d0a71..cf50fe274167 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_partition.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_partition.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -155,7 +155,7 @@ struct DevicePartition typename NumItemsT> HIPCUB_RUNTIME_FUNCTION static hipError_t If(void* d_temp_storage, - std::size_t& temp_storage_bytes, + size_t& temp_storage_bytes, InputIteratorT d_in, FirstOutputIteratorT d_first_part_out, SecondOutputIteratorT d_second_part_out, @@ -190,7 +190,7 @@ struct DevicePartition typename NumItemsT> HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t If(void* d_temp_storage, - std::size_t& temp_storage_bytes, + size_t& temp_storage_bytes, InputIteratorT d_in, FirstOutputIteratorT d_first_part_out, SecondOutputIteratorT d_second_part_out, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_reduce.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_reduce.hpp index 8f21617f81ec..41b697eb3a52 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_reduce.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_reduce.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -46,8 +46,11 @@ #include // hip_bfloat16 #include // __half +#include _HIPCUB_LIBCXX_INCLUDE(functional) +#include _HIPCUB_STD_INCLUDE(functional) +#include _HIPCUB_STD_INCLUDE(limits) + #include -#include BEGIN_HIPCUB_NAMESPACE namespace detail @@ -66,7 +69,7 @@ HIPCUB_HOST_DEVICE T set_half_bits(uint16_t value) template HIPCUB_HOST_DEVICE inline T get_lowest_value() { - return std::numeric_limits::lowest(); + return _HIPCUB_STD::numeric_limits::lowest(); } template<> @@ -86,7 +89,7 @@ HIPCUB_HOST_DEVICE inline hip_bfloat16 get_lowest_value() template HIPCUB_HOST_DEVICE inline T get_max_value() { - return std::numeric_limits::max(); + return _HIPCUB_STD::numeric_limits::max(); } template<> @@ -116,7 +119,7 @@ template inline auto get_lowest_special_value() -> typename std::enable_if_t::value, T> { - return -std::numeric_limits::infinity(); + return -_HIPCUB_STD::numeric_limits::infinity(); } template<> @@ -146,7 +149,7 @@ template inline auto get_max_special_value() -> typename std::enable_if_t::value, T> { - return std::numeric_limits::infinity(); + return _HIPCUB_STD::numeric_limits::infinity(); } template<> @@ -229,15 +232,15 @@ class DeviceReduce NumItemsT num_items, hipStream_t stream = 0) { - using InputT = typename std::iterator_traits::value_type; - using OutputT = typename std::iterator_traits::value_type; - using InitT = hipcub::detail::non_void_value_t; + using InputT = detail::it_value_t; + using OutputT = detail::it_value_t; + using InitT = detail::non_void_value_t; return Reduce(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, - ::hipcub::Sum(), + _HIPCUB_STD::plus<>{}, InitT(0), stream); } @@ -264,13 +267,17 @@ class DeviceReduce NumItemsT num_items, hipStream_t stream = 0) { - using T = typename std::iterator_traits::value_type; + using T = detail::it_value_t; return Reduce(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, - ::hipcub::Min(), +#if _HIPCUB_HAS_DEVICE_SYSTEM_STD + _HIPCUB_LIBCXX::minimum<>{}, +#else + [] (auto a, auto b) { return a > b ? b : a;}, +#endif detail::get_max_value(), stream); } @@ -302,8 +309,8 @@ class DeviceReduce hipStream_t stream = 0) { using OffsetT = NumItemsT; - using T = typename std::iterator_traits::value_type; - using O = typename std::iterator_traits::value_type; + using T = detail::it_value_t; + using O = detail::it_value_t; using OutputTupleT = hipcub::detail::non_void_value_t>; using OutputValueT = typename OutputTupleT::Value; @@ -381,13 +388,17 @@ class DeviceReduce NumItemsT num_items, hipStream_t stream = 0) { - using T = typename std::iterator_traits::value_type; + using T = detail::it_value_t; return Reduce(d_temp_storage, temp_storage_bytes, d_in, d_out, num_items, - ::hipcub::Max(), +#if _HIPCUB_HAS_DEVICE_SYSTEM_STD + _HIPCUB_LIBCXX::maximum<>{}, +#else + [] (auto a, auto b) { return a > b ? a : b;}, +#endif detail::get_lowest_value(), stream); } @@ -419,9 +430,9 @@ class DeviceReduce hipStream_t stream = 0) { using OffsetT = NumItemsT; - using T = typename std::iterator_traits::value_type; - using O = typename std::iterator_traits::value_type; - using OutputTupleT = hipcub::detail::non_void_value_t>; + using T = detail::it_value_t; + using O = detail::it_value_t; + using OutputTupleT = detail::non_void_value_t>; using OutputValueT = typename OutputTupleT::Value; using IteratorT = ArgIndexInputIterator; @@ -546,8 +557,7 @@ class DeviceReduce NumItemsT num_items, hipStream_t stream = 0) { - using key_compare_op - = ::rocprim::equal_to::value_type>; + using key_compare_op = ::rocprim::equal_to>; return ::rocprim::reduce_by_key(d_temp_storage, temp_storage_bytes, d_keys_in, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_scan.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_scan.hpp index 2e8d70581a3a..de2be6e0859d 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_scan.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_scan.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -40,6 +40,8 @@ #include // IWYU pragma: export #include // IWYU pragma: export +#include _HIPCUB_STD_INCLUDE(functional) + BEGIN_HIPCUB_NAMESPACE class DeviceScan @@ -58,7 +60,7 @@ class DeviceScan temp_storage_bytes, d_in, d_out, - ::hipcub::Sum(), + _HIPCUB_STD::plus<>{}, num_items, stream); } @@ -230,12 +232,12 @@ class DeviceScan NumItemsT num_items, hipStream_t stream = 0) { - using T = typename std::iterator_traits::value_type; + using T = detail::it_value_t; return ExclusiveScan(d_temp_storage, temp_storage_bytes, d_in, d_out, - ::hipcub::Sum(), + _HIPCUB_STD::plus<>{}, T(0), num_items, stream); @@ -492,8 +494,8 @@ class DeviceScan template + typename EqualityOpT = _HIPCUB_STD::equal_to<>, + typename NumItemsT = uint32_t> HIPCUB_RUNTIME_FUNCTION static hipError_t ExclusiveSumByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -504,14 +506,14 @@ class DeviceScan EqualityOpT equality_op = EqualityOpT(), hipStream_t stream = 0) { - using in_value_type = typename std::iterator_traits::value_type; + using in_value_type = detail::it_value_t; return ExclusiveScanByKey(d_temp_storage, temp_storage_bytes, d_keys_in, d_values_in, d_values_out, - ::hipcub::Sum(), + _HIPCUB_STD::plus<>{}, static_cast(0), num_items, equality_op, @@ -521,8 +523,8 @@ class DeviceScan template + typename EqualityOpT = _HIPCUB_STD::equal_to<>, + typename NumItemsT = uint32_t> HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t ExclusiveSumByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -550,8 +552,8 @@ class DeviceScan typename ValuesOutputIteratorT, typename ScanOpT, typename InitValueT, - typename EqualityOpT = ::hipcub::Equality, - typename NumItemsT = std::uint32_t> + typename EqualityOpT = _HIPCUB_STD::equal_to<>, + typename NumItemsT = uint32_t> HIPCUB_RUNTIME_FUNCTION static hipError_t ExclusiveScanByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -591,8 +593,8 @@ class DeviceScan typename ValuesOutputIteratorT, typename ScanOpT, typename InitValueT, - typename EqualityOpT = ::hipcub::Equality, - typename NumItemsT = std::uint32_t> + typename EqualityOpT = _HIPCUB_STD::equal_to<>, + typename NumItemsT = uint32_t> HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t ExclusiveScanByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -622,8 +624,8 @@ class DeviceScan template + typename EqualityOpT = _HIPCUB_STD::equal_to<>, + typename NumItemsT = uint32_t> HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveSumByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -639,7 +641,7 @@ class DeviceScan d_keys_in, d_values_in, d_values_out, - ::hipcub::Sum(), + _HIPCUB_STD::plus<>{}, num_items, equality_op, stream); @@ -648,8 +650,8 @@ class DeviceScan template + typename EqualityOpT = _HIPCUB_STD::equal_to<>, + typename NumItemsT = uint32_t> HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveSumByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -676,8 +678,8 @@ class DeviceScan typename ValuesInputIteratorT, typename ValuesOutputIteratorT, typename ScanOpT, - typename EqualityOpT = ::hipcub::Equality, - typename NumItemsT = std::uint32_t> + typename EqualityOpT = _HIPCUB_STD::equal_to<>, + typename NumItemsT = uint32_t> HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveScanByKey(void* d_temp_storage, size_t& temp_storage_bytes, @@ -689,8 +691,7 @@ class DeviceScan EqualityOpT equality_op = EqualityOpT(), hipStream_t stream = 0) { - using acc_t = ::rocprim:: - accumulator_t::value_type>; + using acc_t = ::rocprim::accumulator_t>; return ::rocprim::inclusive_scan_by_key<::rocprim::default_config, KeysInputIteratorT, @@ -714,8 +715,8 @@ class DeviceScan typename ValuesInputIteratorT, typename ValuesOutputIteratorT, typename ScanOpT, - typename EqualityOpT = ::hipcub::Equality, - typename NumItemsT = std::uint32_t> + typename EqualityOpT = _HIPCUB_STD::equal_to<>, + typename NumItemsT = uint32_t> HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t InclusiveScanByKey(void* d_temp_storage, size_t& temp_storage_bytes, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_segmented_reduce.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_segmented_reduce.hpp index 8f6494471209..1a05059891de 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_segmented_reduce.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_segmented_reduce.hpp @@ -42,9 +42,12 @@ #include // IWYU pragma: export #include // IWYU pragma: export +#include _HIPCUB_LIBCXX_INCLUDE(functional) +#include _HIPCUB_STD_INCLUDE(functional) +#include _HIPCUB_STD_INCLUDE(limits) + #include #include -#include BEGIN_HIPCUB_NAMESPACE @@ -86,7 +89,7 @@ inline hipError_t launch_segmented_arg_minmax(::rocprim::detail::target current_ const unsigned int segment_id = ::rocprim::detail::block_id<0>(); // Large indices need bigger offset type than unsigned int - using offset_type = typename std::iterator_traits::value_type; + using offset_type = it_value_t; const offset_type begin_offset = begin_offsets[segment_id]; const offset_type end_offset = end_offsets[segment_id]; @@ -133,7 +136,7 @@ inline hipError_t segmented_arg_minmax(void* temporary_storage, InitValueType empty_value, hipStream_t stream) { - using input_type = typename std::iterator_traits::value_type; + using input_type = detail::it_value_t; using result_type = ::rocprim::accumulator_t; using selector = ::rocprim::detail::segmented_reduce_config_selector; @@ -156,7 +159,7 @@ inline hipError_t segmented_arg_minmax(void* temporary_storage, std::chrono::high_resolution_clock::time_point start; - if HIPCUB_IF_CONSTEXPR(HIPCUB_DETAIL_DEBUG_SYNC_VALUE) + if constexpr(HIPCUB_DETAIL_DEBUG_SYNC_VALUE) { start = std::chrono::high_resolution_clock::now(); } @@ -188,16 +191,16 @@ struct DeviceSegmentedReduce typename ReductionOp, typename T> HIPCUB_RUNTIME_FUNCTION - static hipError_t Reduce(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - ReductionOp reduction_op, - T initial_value, - hipStream_t stream = 0) + static hipError_t Reduce(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + ReductionOp reduction_op, + T initial_value, + hipStream_t stream = 0) { return ::rocprim::segmented_reduce( d_temp_storage, @@ -218,17 +221,17 @@ struct DeviceSegmentedReduce typename ReductionOp, typename T> HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t Reduce(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - ReductionOp reduction_op, - T initial_value, - hipStream_t stream, - bool debug_synchronous) + static hipError_t Reduce(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + ReductionOp reduction_op, + T initial_value, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return Reduce(d_temp_storage, @@ -245,16 +248,16 @@ struct DeviceSegmentedReduce template HIPCUB_RUNTIME_FUNCTION - static hipError_t Sum(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + static hipError_t Sum(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { - using input_type = typename std::iterator_traits::value_type; + using input_type = detail::it_value_t; return Reduce(d_temp_storage, temp_storage_bytes, @@ -263,22 +266,22 @@ struct DeviceSegmentedReduce num_segments, d_begin_offsets, d_end_offsets, - ::hipcub::Sum(), + _HIPCUB_STD::plus<>{}, input_type(), stream); } template HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t Sum(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + static hipError_t Sum(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return Sum(d_temp_storage, @@ -293,16 +296,16 @@ struct DeviceSegmentedReduce template HIPCUB_RUNTIME_FUNCTION - static hipError_t Min(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + static hipError_t Min(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { - using input_type = typename std::iterator_traits::value_type; + using input_type = detail::it_value_t; return Reduce(d_temp_storage, temp_storage_bytes, @@ -311,22 +314,26 @@ struct DeviceSegmentedReduce num_segments, d_begin_offsets, d_end_offsets, - ::hipcub::Min(), - std::numeric_limits::max(), +#if _HIPCUB_HAS_DEVICE_SYSTEM_STD + _HIPCUB_LIBCXX::minimum<>{}, +#else + [] (auto a, auto b) { return a > b ? b : a;}, +#endif + _HIPCUB_STD::numeric_limits::max(), stream); } template HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t Min(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + static hipError_t Min(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return Min(d_temp_storage, @@ -341,20 +348,20 @@ struct DeviceSegmentedReduce template HIPCUB_RUNTIME_FUNCTION - static hipError_t ArgMin(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + static hipError_t ArgMin(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { using OffsetT = int; - using T = typename std::iterator_traits::value_type; - using O = typename std::iterator_traits::value_type; - using OutputTupleT = typename std:: - conditional::value, KeyValuePair, O>::type; + using T = hipcub::detail::it_value_t; + using O = hipcub::detail::it_value_t; + using OutputTupleT = + typename std::conditional, KeyValuePair, O>::type; using OutputValueT = typename OutputTupleT::Value; using IteratorT = ArgIndexInputIterator; @@ -362,7 +369,7 @@ struct DeviceSegmentedReduce IteratorT d_indexed_in(d_in); // true maximum value of the full range // key is ::max because ArgMin finds the lowest value that has the lowest key - const OutputTupleT init(std::numeric_limits::max(), + const OutputTupleT init(_HIPCUB_STD::numeric_limits::max(), detail::get_max_special_value()); // special value for empty segments const OutputTupleT empty_value(1, detail::get_max_value()); @@ -382,15 +389,15 @@ struct DeviceSegmentedReduce template HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t ArgMin(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + static hipError_t ArgMin(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return ArgMin(d_temp_storage, @@ -405,16 +412,16 @@ struct DeviceSegmentedReduce template HIPCUB_RUNTIME_FUNCTION - static hipError_t Max(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + static hipError_t Max(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { - using input_type = typename std::iterator_traits::value_type; + using input_type = detail::it_value_t; return Reduce(d_temp_storage, temp_storage_bytes, @@ -423,22 +430,26 @@ struct DeviceSegmentedReduce num_segments, d_begin_offsets, d_end_offsets, - ::hipcub::Max(), - std::numeric_limits::lowest(), +#if _HIPCUB_HAS_DEVICE_SYSTEM_STD + _HIPCUB_LIBCXX::maximum<>{}, +#else + [] (auto a, auto b) { return a > b ? a : b;}, +#endif + _HIPCUB_STD::numeric_limits::lowest(), stream); } template HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t Max(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + static hipError_t Max(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return Max(d_temp_storage, @@ -453,20 +464,20 @@ struct DeviceSegmentedReduce template HIPCUB_RUNTIME_FUNCTION - static hipError_t ArgMax(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + static hipError_t ArgMax(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { using OffsetT = int; - using T = typename std::iterator_traits::value_type; - using O = typename std::iterator_traits::value_type; - using OutputTupleT = typename std:: - conditional::value, KeyValuePair, O>::type; + using T = hipcub::detail::it_value_t; + using O = hipcub::detail::it_value_t; + using OutputTupleT = + typename std::conditional, KeyValuePair, O>::type; using OutputValueT = typename OutputTupleT::Value; using IteratorT = ArgIndexInputIterator; @@ -474,7 +485,7 @@ struct DeviceSegmentedReduce IteratorT d_indexed_in(d_in); // true minimum value of the full range // key is ::max because ArgMax finds the highest value that has the lowest key - const OutputTupleT init(std::numeric_limits::max(), + const OutputTupleT init(_HIPCUB_STD::numeric_limits::max(), detail::get_lowest_special_value()); // special value for empty segments const OutputTupleT empty_value(1, detail::get_lowest_value()); @@ -494,15 +505,15 @@ struct DeviceSegmentedReduce template HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION - static hipError_t ArgMax(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + static hipError_t ArgMax(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return ArgMax(d_temp_storage, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_segmented_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_segmented_sort.hpp index e266bd635aa0..6765a273d7de 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_segmented_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_segmented_sort.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -42,17 +42,18 @@ BEGIN_HIPCUB_NAMESPACE struct DeviceSegmentedSort { template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return ::rocprim::segmented_radix_sort_pairs(d_temp_storage, temp_storage_bytes, @@ -71,19 +72,19 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortPairs(d_temp_storage, @@ -100,15 +101,16 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { ::rocprim::double_buffer d_keys_db = detail::to_double_buffer(d_keys); ::rocprim::double_buffer d_values_db = detail::to_double_buffer(d_values); @@ -130,17 +132,17 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortPairs(d_temp_storage, @@ -155,17 +157,18 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return ::rocprim::segmented_radix_sort_pairs_desc(d_temp_storage, temp_storage_bytes, @@ -184,19 +187,19 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortPairsDescending(d_temp_storage, @@ -213,15 +216,16 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { ::rocprim::double_buffer d_keys_db = detail::to_double_buffer(d_keys); ::rocprim::double_buffer d_values_db = detail::to_double_buffer(d_values); @@ -244,17 +248,17 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t SortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortPairsDescending(d_temp_storage, @@ -269,15 +273,16 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return ::rocprim::segmented_radix_sort_keys(d_temp_storage, temp_storage_bytes, @@ -294,17 +299,17 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortKeys(d_temp_storage, @@ -319,14 +324,15 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { ::rocprim::double_buffer d_keys_db = detail::to_double_buffer(d_keys); hipError_t error = ::rocprim::segmented_radix_sort_keys(d_temp_storage, @@ -345,16 +351,16 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortKeys(d_temp_storage, @@ -368,15 +374,16 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return ::rocprim::segmented_radix_sort_keys_desc(d_temp_storage, temp_storage_bytes, @@ -393,17 +400,17 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortKeysDescending(d_temp_storage, @@ -418,14 +425,15 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { ::rocprim::double_buffer d_keys_db = detail::to_double_buffer(d_keys); hipError_t error @@ -445,16 +453,16 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - SortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t SortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return SortKeysDescending(d_temp_storage, @@ -468,17 +476,18 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return SortPairs(d_temp_storage, temp_storage_bytes, @@ -494,19 +503,19 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortPairs(d_temp_storage, @@ -523,15 +532,16 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return SortPairs(d_temp_storage, temp_storage_bytes, @@ -545,17 +555,17 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortPairs(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortPairs(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortPairs(d_temp_storage, @@ -570,18 +580,18 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return SortPairsDescending(d_temp_storage, temp_storage_bytes, @@ -597,19 +607,19 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - const ValueT* d_values_in, - ValueT* d_values_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + const ValueT* d_values_in, + ValueT* d_values_out, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortPairsDescending(d_temp_storage, @@ -626,16 +636,16 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return SortPairsDescending(d_temp_storage, temp_storage_bytes, @@ -649,17 +659,17 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortPairsDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - DoubleBuffer& d_values, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortPairsDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + DoubleBuffer& d_values, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortPairsDescending(d_temp_storage, @@ -674,15 +684,16 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return SortKeys(d_temp_storage, temp_storage_bytes, @@ -696,17 +707,17 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortKeys(d_temp_storage, @@ -721,14 +732,15 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t StableSortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return SortKeys(d_temp_storage, temp_storage_bytes, @@ -741,16 +753,16 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortKeys(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeys(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortKeys(d_temp_storage, @@ -764,16 +776,16 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return SortKeysDescending(d_temp_storage, temp_storage_bytes, @@ -787,17 +799,17 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - const KeyT* d_keys_in, - KeyT* d_keys_out, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + const KeyT* d_keys_in, + KeyT* d_keys_out, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortKeysDescending(d_temp_storage, @@ -812,15 +824,15 @@ struct DeviceSegmentedSort } template - HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream = 0) + HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream = 0) { return SortKeysDescending(d_temp_storage, temp_storage_bytes, @@ -833,16 +845,16 @@ struct DeviceSegmentedSort } template - HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION static hipError_t - StableSortKeysDescending(void* d_temp_storage, - size_t& temp_storage_bytes, - DoubleBuffer& d_keys, - int num_items, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream, - bool debug_synchronous) + HIPCUB_DETAIL_DEPRECATED_DEBUG_SYNCHRONOUS HIPCUB_RUNTIME_FUNCTION + static hipError_t StableSortKeysDescending(void* d_temp_storage, + size_t& temp_storage_bytes, + DoubleBuffer& d_keys, + int64_t num_items, + int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream, + bool debug_synchronous) { HIPCUB_DETAIL_RUNTIME_LOG_DEBUG_SYNCHRONOUS(); return StableSortKeysDescending(d_temp_storage, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_select.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_select.hpp index 090b0f4a285d..5360ed4220f4 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_select.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_select.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,10 +33,10 @@ #include "../../../config.hpp" #include "../../../util_deprecated.hpp" -#include "../thread/thread_operators.hpp" - #include // IWYU pragma: export +#include _HIPCUB_STD_INCLUDE(functional) + #include BEGIN_HIPCUB_NAMESPACE @@ -352,7 +352,7 @@ class DeviceSelect d_out, d_num_selected_out, num_items, - hipcub::Equality(), + _HIPCUB_STD::equal_to<>(), stream, HIPCUB_DETAIL_DEBUG_SYNC_VALUE); } @@ -437,7 +437,7 @@ class DeviceSelect d_values_output, d_num_selected_out, num_items, - hipcub::Equality(), + _HIPCUB_STD::equal_to<>(), stream); } diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_spmv.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_spmv.hpp deleted file mode 100644 index e5c75322dfbe..000000000000 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/device/device_spmv.hpp +++ /dev/null @@ -1,196 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ - -#ifndef HIPCUB_ROCPRIM_DEVICE_DEVICE_SPMV_HPP_ -#define HIPCUB_ROCPRIM_DEVICE_DEVICE_SPMV_HPP_ - -#include "../../../config.hpp" -#include "../../../util_deprecated.hpp" - -#include "../iterator/tex_obj_input_iterator.hpp" - -#include "../util_sync.hpp" - -#include - -BEGIN_HIPCUB_NAMESPACE - -class HIPCUB_DEPRECATED_BECAUSE("Use the hipSPARSE library instead") DeviceSpmv -{ - -public: - template ///< Signed integer type for sequence offsets - struct HIPCUB_DEPRECATED_BECAUSE("Use the rocSPARSE library instead") SpmvParams - { - ValueT* - d_values; ///< Pointer to the array of \p num_nonzeros values of the corresponding nonzero elements of matrix A. - OffsetT* - d_row_end_offsets; ///< Pointer to the array of \p m offsets demarcating the end of every row in \p d_column_indices and \p d_values - OffsetT* - d_column_indices; ///< Pointer to the array of \p num_nonzeros column-indices of the corresponding nonzero elements of matrix A. (Indices are zero-valued.) - ValueT* - d_vector_x; ///< Pointer to the array of \p num_cols values corresponding to the dense input vector x - ValueT* - d_vector_y; ///< Pointer to the array of \p num_rows values corresponding to the dense output vector y - int num_rows; ///< Number of rows of matrix A. - int num_cols; ///< Number of columns of matrix A. - int num_nonzeros; ///< Number of nonzero elements of matrix A. - ValueT alpha; ///< Alpha multiplicand - ValueT beta; ///< Beta addend-multiplicand - - ::hipcub::TexObjInputIterator t_vector_x; - }; - -static constexpr uint32_t CsrMVKernel_MaxThreads = 256; - -template -static __global__ void -CsrMVKernel(SpmvParams spmv_params) -{ - __shared__ ValueT partial; - - const int32_t row_id = blockIdx.x; - - if(threadIdx.x == 0) - { - partial = spmv_params.beta * spmv_params.d_vector_y[row_id]; - } - __syncthreads(); - - int32_t row_offset = (row_id == 0) ? (0) : (spmv_params.d_row_end_offsets[row_id - 1]); - for(uint32_t thread_offset = 0; thread_offset < spmv_params.num_cols / blockDim.x; - thread_offset++) - { - int32_t offset = row_offset + thread_offset * blockDim.x + threadIdx.x; - - if(offset < spmv_params.d_row_end_offsets[row_id]) - { - ValueT t_value = - spmv_params.alpha * - spmv_params.d_values[offset] * - spmv_params.d_vector_x[spmv_params.d_column_indices[offset]]; - - atomicAdd(&partial, t_value); - - __syncthreads(); - - if(threadIdx.x == 0) - { - spmv_params.d_vector_y[row_id] = partial; - } - } - } -} - -template -HIPCUB_DEPRECATED_BECAUSE("Use the rocSPARSE library instead") -HIPCUB_RUNTIME_FUNCTION static hipError_t CsrMV(void* d_temp_storage, - size_t& temp_storage_bytes, - ValueT* d_values, - int* d_row_offsets, - int* d_column_indices, - ValueT* d_vector_x, - ValueT* d_vector_y, - int num_rows, - int num_cols, - int num_nonzeros, - hipStream_t stream = 0) -{ - HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH - SpmvParams spmv_params; - HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP - spmv_params.d_values = d_values; - spmv_params.d_row_end_offsets = d_row_offsets + 1; - spmv_params.d_column_indices = d_column_indices; - spmv_params.d_vector_x = d_vector_x; - spmv_params.d_vector_y = d_vector_y; - spmv_params.num_rows = num_rows; - spmv_params.num_cols = num_cols; - spmv_params.num_nonzeros = num_nonzeros; - spmv_params.alpha = 1.0; - spmv_params.beta = 0.0; - - if(d_temp_storage == nullptr) - { - // Make sure user won't try to allocate 0 bytes memory, because - // hipMalloc will return nullptr when size is zero. - temp_storage_bytes = 4; - return hipError_t(0); - } else - { - size_t block_size = min(num_cols, static_cast(DeviceSpmv::CsrMVKernel_MaxThreads)); - size_t grid_size = num_rows; - - std::chrono::high_resolution_clock::time_point start; - if HIPCUB_IF_CONSTEXPR(HIPCUB_DETAIL_DEBUG_SYNC_VALUE) - { - start = std::chrono::high_resolution_clock::now(); - } - HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH - CsrMVKernel<<>>(spmv_params); - HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH - HIPCUB_DETAIL_HIP_SYNC_AND_RETURN_ON_ERROR("CsrMV", block_size * grid_size, start); - } - return hipSuccess; -} - -template -HIPCUB_DEPRECATED_BECAUSE("Use the rocSPARSE library instead") -HIPCUB_RUNTIME_FUNCTION static hipError_t CsrMV(void* d_temp_storage, - size_t& temp_storage_bytes, - ValueT* d_values, - int* d_row_offsets, - int* d_column_indices, - ValueT* d_vector_x, - ValueT* d_vector_y, - int num_rows, - int num_cols, - int num_nonzeros, - hipStream_t stream, - bool /*debug_synchronous*/) -{ - return CsrMV(d_temp_storage, - temp_storage_bytes, - d_values, - d_row_offsets, - d_column_indices, - d_vector_x, - d_vector_y, - num_rows, - num_cols, - num_nonzeros, - stream); -} -}; - -END_HIPCUB_NAMESPACE - -#endif // HIPCUB_CUB_DEVICE_DEVICE_SELECT_HPP_ - diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/grid/grid_even_share.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/grid/grid_even_share.hpp index 8311ec12c0b0..88a80c2dad6c 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/grid/grid_even_share.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/grid/grid_even_share.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2024, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -123,12 +123,12 @@ struct GridEvenShare HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH this->total_tiles = static_cast(hipcub::DivideAndRoundUp(num_items_, tile_items)); HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP - this->grid_size = min(total_tiles, max_grid_size); + this->grid_size = _HIPCUB_STD::min(total_tiles, max_grid_size); int avg_tiles_per_block = total_tiles / grid_size; // leftover grains go to big blocks: this->big_shares = total_tiles - (avg_tiles_per_block * grid_size); - this->normal_share_items = avg_tiles_per_block * tile_items; - this->normal_base_offset = big_shares * tile_items; + this->normal_share_items = static_cast(avg_tiles_per_block) * tile_items; + this->normal_base_offset = static_cast(big_shares) * tile_items; this->big_share_items = normal_share_items + tile_items; } @@ -154,7 +154,7 @@ struct GridEvenShare { // This thread block gets a normal share of grains (avg_tiles_per_block) block_offset = normal_base_offset + (block_id * normal_share_items); - block_end = min(num_items, block_offset + normal_share_items); + block_end = _HIPCUB_STD::min(num_items, block_offset + normal_share_items); } // Else default past-the-end } diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/hipcub.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/hipcub.hpp index ad8e1131179d..3124a142c2ff 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/hipcub.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/hipcub.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -66,7 +66,6 @@ #include "device/device_segmented_reduce.hpp" #include "device/device_segmented_sort.hpp" #include "device/device_select.hpp" -#include "device/device_spmv.hpp" #include "device/device_transform.hpp" // Grid @@ -79,15 +78,10 @@ #include "iterator/arg_index_input_iterator.hpp" #include "iterator/cache_modified_input_iterator.hpp" #include "iterator/cache_modified_output_iterator.hpp" -#include "iterator/constant_input_iterator.hpp" -#include "iterator/counting_input_iterator.hpp" -#include "iterator/discard_output_iterator.hpp" #include "iterator/tex_obj_input_iterator.hpp" -#include "iterator/transform_input_iterator.hpp" // Thread #include "thread/thread_load.hpp" -#include "thread/thread_operators.hpp" #include "thread/thread_reduce.hpp" #include "thread/thread_scan.hpp" #include "thread/thread_search.hpp" diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/arg_index_input_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/arg_index_input_iterator.hpp index 1acdb5072780..51754688bf2a 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/arg_index_input_iterator.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/arg_index_input_iterator.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,6 +32,8 @@ #include "../../../config.hpp" +#include "../util_type.hpp" + #include "iterator_category.hpp" #include "iterator_wrapper.hpp" @@ -45,7 +47,7 @@ BEGIN_HIPCUB_NAMESPACE template::value_type> + class InputValueType = detail::it_value_t> class ArgIndexInputIterator : public detail::IteratorWrapper< rocprim::arg_index_iterator, diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/constant_input_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/constant_input_iterator.hpp deleted file mode 100644 index e135817dbb0a..000000000000 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/constant_input_iterator.hpp +++ /dev/null @@ -1,76 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ - -#ifndef HIPCUB_ROCPRIM_ITERATOR_CONSTANT_INPUT_ITERATOR_HPP_ -#define HIPCUB_ROCPRIM_ITERATOR_CONSTANT_INPUT_ITERATOR_HPP_ - -#include "../../../config.hpp" -#include "../../../util_deprecated.hpp" - -#include "iterator_category.hpp" -#include "iterator_wrapper.hpp" - -#include // IWYU pragma: export - -#include - -BEGIN_HIPCUB_NAMESPACE - -#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document - -template -class HIPCUB_DEPRECATED_BECAUSE( - "Use rocprim::constant_iterator or rocthrust::constant_iterator instead") ConstantInputIterator - : public detail::IteratorWrapper, - ConstantInputIterator> -{ - using Iterator = rocprim::constant_iterator; - using Base = detail::IteratorWrapper>; - -public: - using iterator_category = typename detail::IteratorCategory::type; - using self_type = typename Iterator::self_type; - - __host__ __device__ __forceinline__ ConstantInputIterator( - const typename Iterator::value_type value, const size_t index = 0) - : Base(Iterator(value, index)) - {} - - // Cast from wrapped iterator to class itself - __host__ __device__ __forceinline__ explicit ConstantInputIterator(Iterator iterator) - : Base(iterator) - {} -}; - -#endif - -END_HIPCUB_NAMESPACE - -#endif // HIPCUB_ROCPRIM_ITERATOR_CONSTANT_INPUT_ITERATOR_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/counting_input_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/counting_input_iterator.hpp deleted file mode 100644 index 06802ffc8bc3..000000000000 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/counting_input_iterator.hpp +++ /dev/null @@ -1,77 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ - -#ifndef HIPCUB_ROCPRIM_ITERATOR_COUNTING_INPUT_ITERATOR_HPP_ -#define HIPCUB_ROCPRIM_ITERATOR_COUNTING_INPUT_ITERATOR_HPP_ - -#include "../../../config.hpp" -#include "../../../util_deprecated.hpp" - -#include "iterator_category.hpp" -#include "iterator_wrapper.hpp" - -#include // IWYU pragma: export - -#include - -BEGIN_HIPCUB_NAMESPACE - -#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document - -template -class HIPCUB_DEPRECATED_BECAUSE( - "Use rocprim::counting_iterator or rocthrust::counting_iterator instead") CountingInputIterator - : public detail::IteratorWrapper, - CountingInputIterator> -{ - using Iterator = rocprim::counting_iterator; - using Base - = detail::IteratorWrapper>; - -public: - using iterator_category = typename detail::IteratorCategory::type; - using self_type = typename Iterator::self_type; - - __host__ __device__ __forceinline__ CountingInputIterator( - const typename Iterator::value_type value) - : Base(Iterator(value)) - {} - - // Cast from wrapped iterator to class itself - __host__ __device__ __forceinline__ explicit CountingInputIterator(Iterator iterator) - : Base(iterator) - {} -}; - -#endif - -END_HIPCUB_NAMESPACE - -#endif // HIPCUB_ROCPRIM_ITERATOR_COUNTING_INPUT_ITERATOR_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/discard_output_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/discard_output_iterator.hpp deleted file mode 100644 index 4896530fb83a..000000000000 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/discard_output_iterator.hpp +++ /dev/null @@ -1,227 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2020-2025, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ - -#ifndef HIPCUB_ROCPRIM_ITERATOR_DISCARD_OUTPUT_ITERATOR_HPP_ -#define HIPCUB_ROCPRIM_ITERATOR_DISCARD_OUTPUT_ITERATOR_HPP_ - -#include "../../../config.hpp" -#include "../../../util_deprecated.hpp" - -#include "iterator_category.hpp" - -#include // IWYU pragma: export - -#include -#include - -BEGIN_HIPCUB_NAMESPACE - -/** - * \addtogroup UtilIterator - * @{ - */ - - -/** - * \brief A discard iterator - */ -template -class HIPCUB_DEPRECATED_BECAUSE( - "Use rocprim::discard_iterator or rocthrust::discard_iterator instead") DiscardOutputIterator -{ -public: - // Required iterator traits - using self_type = DiscardOutputIterator; ///< My own type - using difference_type - = OffsetT; ///< Type to express the result of subtracting one iterator from another - using value_type = void; ///< The type of the element the iterator can point to - using pointer = void; ///< The type of a pointer to an element the iterator can point to - using reference = void; ///< The type of a reference to an element the iterator can point to - using iterator_category = - typename detail::IteratorCategory::type; ///< The iterator category - -private: - - OffsetT offset; - -public: - - /// Constructor - __host__ __device__ __forceinline__ DiscardOutputIterator( - OffsetT offset = 0) ///< Base offset - : - offset(offset) - {} - - /** - * @typedef self_type - * @brief Postfix increment - */ - __host__ __device__ __forceinline__ self_type operator++(int) - { - self_type retval = *this; - offset++; - return retval; - } - - /** - * @typedef self_type - * @brief Postfix increment - */ - __host__ __device__ __forceinline__ self_type operator++() - { - offset++; - return *this; - } - - /** - * @typedef self_type - * @brief Indirection - */ - __host__ __device__ __forceinline__ self_type& operator*() - { - // return self reference, which can be assigned to anything - return *this; - } - - /** - * @typedef self_type - * @brief Addition - */ - template - __host__ __device__ __forceinline__ self_type operator+(Distance n) const - { - self_type retval(offset + n); - return retval; - } - - /** - * @typedef self_type - * @brief Addition assignment - */ - template - __host__ __device__ __forceinline__ self_type& operator+=(Distance n) - { - offset += n; - return *this; - } - - /** - * @typedef self_type - * @brief Subtraction assignment - */ - template - __host__ __device__ __forceinline__ self_type operator-(Distance n) const - { - self_type retval(offset - n); - return retval; - } - - /** - * @typedef self_type - * @brief Subtraction assignment - */ - template - __host__ __device__ __forceinline__ self_type& operator-=(Distance n) - { - offset -= n; - return *this; - } - - /** - * @typedef self_type - * @brief Distance - */ - __host__ __device__ __forceinline__ difference_type operator-(self_type other) const - { - return offset - other.offset; - } - - /** - * @typedef self_type - * @brief Array subscript - */ - template - __host__ __device__ __forceinline__ self_type& operator[](Distance) - { - // return self reference, which can be assigned to anything - return *this; - } - - /// Structure dereference - __host__ __device__ __forceinline__ pointer operator->() - { - return; - } - - /// Assignment to anything else (no-op) - template - __host__ __device__ __forceinline__ void operator=(T const&) - {} - - /// Cast to void* operator - __host__ __device__ __forceinline__ operator void*() const - { - return nullptr; - } - - /** - * @typedef self_type - * @brief Equal to - */ - __host__ __device__ __forceinline__ bool operator==(const self_type& rhs) const - { - return (offset == rhs.offset); - } - - /** - * @typedef self_type - * @brief Not equal to - */ - __host__ __device__ __forceinline__ bool operator!=(const self_type& rhs) const - { - return (offset != rhs.offset); - } - - /** - * @typedef self_type - * @brief ostream operator - */ - HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH - friend std::ostream& operator<<(std::ostream& os, const self_type& itr) - { - os << "[" << itr.offset << "]"; - return os; - } - HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP -}; - -END_HIPCUB_NAMESPACE - -#endif // HIPCUB_ROCPRIM_ITERATOR_DISCARD_OUTPUT_ITERATOR_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/transform_input_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/transform_input_iterator.hpp deleted file mode 100644 index 2b1efba7bf2b..000000000000 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/iterator/transform_input_iterator.hpp +++ /dev/null @@ -1,87 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ - -#ifndef HIPCUB_ROCPRIM_ITERATOR_TRANSFORM_INPUT_ITERATOR_HPP_ -#define HIPCUB_ROCPRIM_ITERATOR_TRANSFORM_INPUT_ITERATOR_HPP_ - -#include "../../../config.hpp" - -#include "iterator_category.hpp" -#include "iterator_wrapper.hpp" - -#include // IWYU pragma: export -#include // IWYU pragma: export - -#include -#include -#include - -BEGIN_HIPCUB_NAMESPACE - -#ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document - -template -class HIPCUB_DEPRECATED_BECAUSE( - "Use rocprim::transform_iterator or rocthrust::transform_iterator instead") - TransformInputIterator - : public detail::IteratorWrapper< - rocprim::transform_iterator, - TransformInputIterator> -{ - using Iterator = rocprim::transform_iterator; - using Base = detail::IteratorWrapper< - Iterator, - TransformInputIterator>; - -public: - using iterator_category = typename detail::IteratorCategory::type; - using self_type = typename Iterator::self_type; - using unary_function = typename Iterator::unary_function; - - __host__ __device__ __forceinline__ TransformInputIterator(InputIteratorT iterator, - ConversionOp transform) - : Base(Iterator(iterator, transform)) - {} - - // Cast from wrapped iterator to class itself - __host__ __device__ __forceinline__ explicit TransformInputIterator(Iterator iterator) - : Base(iterator) - {} -}; - -#endif - -END_HIPCUB_NAMESPACE - -#endif // HIPCUB_ROCPRIM_ITERATOR_TRANSFORM_INPUT_ITERATOR_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_load.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_load.hpp index 655814697d4f..e06e14edc945 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_load.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_load.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -77,10 +77,10 @@ HIPCUB_FORCEINLINE T ThreadLoadVolatilePointer(T* ptr, Fundamental /* is_fundame template HIPCUB_DEVICE -HIPCUB_FORCEINLINE typename std::iterator_traits::value_type - ThreadLoad(InputIteratorT itr, - detail::int_constant_t /*modifier*/, - ::std::false_type /*is_pointer*/) +HIPCUB_FORCEINLINE detail::it_value_t + ThreadLoad(InputIteratorT itr, + detail::int_constant_t /*modifier*/, + ::std::false_type /*is_pointer*/) { return rocprim::thread_load(itr); } @@ -96,12 +96,11 @@ HIPCUB_FORCEINLINE T ThreadLoad(T* ptr, template HIPCUB_DEVICE -HIPCUB_FORCEINLINE - typename std::iterator_traits::value_type ThreadLoad(InputIteratorT itr) +HIPCUB_FORCEINLINE detail::it_value_t ThreadLoad(InputIteratorT itr) { return ThreadLoad(itr, detail::int_constant_t(), - ::std::bool_constant<::std::is_pointer::value>()); + ::std::bool_constant<_HIPCUB_STD::is_pointer::value>()); } END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_operators.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_operators.hpp index 193817f486e3..68be944b0315 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_operators.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_operators.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -38,12 +38,14 @@ #include // IWYU pragma: export #include +#include _HIPCUB_STD_INCLUDE(functional) + #include BEGIN_HIPCUB_NAMESPACE -// TODO: this is deprecated in cub, we should also mark this as deprecated when we have libhipcxx -struct Equality +//! deprecated [Since 5.0] +struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::equal_to instead.") Equality { template HIPCUB_HOST_DEVICE inline constexpr bool operator()(T&& t, U&& u) const @@ -52,8 +54,8 @@ struct Equality } }; -// TODO: this is deprecated in cub, we should also mark this as deprecated when we have libhipcxx -struct Inequality +//! deprecated [Since 5.0] +struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::not_equal_to instead.") Inequality { template HIPCUB_HOST_DEVICE inline constexpr bool operator()(T&& t, U&& u) const @@ -62,9 +64,9 @@ struct Inequality } }; -// TODO: this is deprecated in cub, we should also mark this as deprecated when we have libhipcxx -template -struct InequalityWrapper +//! deprecated [Since 5.0] +template +struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::not_equal_to instead.") InequalityWrapper { EqualityOp op; @@ -78,8 +80,8 @@ struct InequalityWrapper } }; -// TODO: this is deprecated in cub, we should also mark this as deprecated when we have libhipcxx -struct Sum +//! deprecated [Since 5.0] +struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::plus instead.") Sum { template HIPCUB_HOST_DEVICE inline constexpr auto operator()(T&& t, U&& u) const -> decltype(auto) @@ -88,8 +90,8 @@ struct Sum } }; -// TODO: this is deprecated in cub, we should also mark this as deprecated when we have libhipcxx -struct Difference +//! deprecated [Since 5.0] +struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::minus instead") Difference { template HIPCUB_HOST_DEVICE inline constexpr auto operator()(T&& t, U&& u) const -> decltype(auto) @@ -98,8 +100,8 @@ struct Difference } }; -// TODO: this is deprecated in cub, we should also mark this as deprecated when we have libhipcxx -struct Division +//! deprecated [Since 5.0] +struct HIPCUB_DEPRECATED_BECAUSE("Use hip::std::divides instead") Division { template HIPCUB_HOST_DEVICE inline constexpr auto operator()(T&& t, U&& u) const -> decltype(auto) @@ -108,26 +110,24 @@ struct Division } }; -// TODO: this is deprecated in cub, we should also mark this as deprecated when we have libhipcxx -struct Max +//! deprecated [Since 5.0] +struct HIPCUB_DEPRECATED_BECAUSE("Use hip::maximum instead.") Max { template HIPCUB_HOST_DEVICE inline constexpr typename std::common_type::type operator()(T&& t, U&& u) const { - // TODO: change to use hip::std::max after libhipcxx is ready return (((u) > (t)) ? (u) : (t)); } }; -// TODO: this is deprecated in cub, we should also mark this as deprecated when we have libhipcxx -struct Min +//! deprecated [Since 5.0] +struct HIPCUB_DEPRECATED_BECAUSE("Use hip::minimum instead.") Min { template HIPCUB_HOST_DEVICE inline constexpr typename std::common_type::type operator()(T&& t, U&& u) const { - // TODO: change to use hip::std::min after libhipcxx is ready return (((u) < (t)) ? (u) : (t)); } }; @@ -256,30 +256,6 @@ struct ReduceByKeyOp } }; -template -struct BinaryFlip -{ - BinaryOpT binary_op; - - HIPCUB_HOST_DEVICE - explicit BinaryFlip(BinaryOpT binary_op) : binary_op(binary_op) - { - } - - template - HIPCUB_DEVICE auto operator()(T&& t, U&& u) -> decltype(auto) - { - return binary_op(std::forward(u), std::forward(t)); - } -}; - -template -HIPCUB_HOST_DEVICE -BinaryFlip MakeBinaryFlip(BinaryOpT binary_op) -{ - return BinaryFlip(binary_op); -} - namespace internal { @@ -300,7 +276,7 @@ struct [[deprecated( constexpr uint32_t operator()(int32_t t, int32_t u) const { - return HIPCUB_MIN(t, u); + return _HIPCUB_STD::min(t, u); } }; @@ -314,7 +290,7 @@ struct [[deprecated( constexpr uint32_t operator()(uint32_t t, uint32_t u) const { - return HIPCUB_MIN(t, u); + return _HIPCUB_STD::min(t, u); } }; @@ -330,7 +306,7 @@ struct [[deprecated( __half2 operator()(__half2 t, __half2 u) const { - return HIPCUB_MIN(t, u); + return _HIPCUB_STD::min(t, u); } }; #endif // !defined(__HIP_NO_HALF_OPERATORS__) @@ -346,7 +322,7 @@ struct [[deprecated("SIMD intrinsics are currently not supported on HIP, use Min __hip_bfloat162 operator()(__hip_bfloat162 t, __hip_bfloat162 u) const { - return HIPCUB_MIN(t, u); + return _HIPCUB_STD::min(t, u); } }; @@ -367,7 +343,7 @@ struct [[deprecated( constexpr uint32_t operator()(int32_t t, int32_t u) const { - return HIPCUB_MAX(t, u); + return _HIPCUB_STD::max(t, u); } }; @@ -381,7 +357,7 @@ struct [[deprecated( constexpr uint32_t operator()(uint32_t t, uint32_t u) const { - return HIPCUB_MAX(t, u); + return _HIPCUB_STD::max(t, u); } }; @@ -396,7 +372,7 @@ struct [[deprecated( __half2 operator()(__half2 t, __half2 u) const { - return HIPCUB_MAX(t, u); + return _HIPCUB_STD::max(t, u); } }; #endif // !defined(__HIP_NO_HALF_OPERATORS__) @@ -411,7 +387,7 @@ struct [[deprecated("SIMD intrinsics are currently not supported on HIP, use Max __hip_bfloat162 operator()(__hip_bfloat162 t, __hip_bfloat162 u) const { - return HIPCUB_MAX(t, u); + return _HIPCUB_STD::max(t, u); } }; @@ -583,7 +559,7 @@ namespace detail // Non-void value type. template using non_void_value_t = - typename std::conditional::value, FallbackT, IteratorT>::type; + typename std::conditional, FallbackT, IteratorT>::type; /// Intermediate accumulator type. template @@ -594,16 +570,16 @@ using accumulator_t = ::rocprim::accumulator_t; // // /// The output value type // using OutputT = -// typename If<(Equals::value_type, +// typename If<(Equals, // void>::VALUE), // OutputT = (if output iterator's value type is void) ? -// typename std::iterator_traits< -// InputIteratorT>::value_type, // ... then the input iterator's value type, -// typename std::iterator_traits::value_type>:: +// it_value_t< +// InputIteratorT>, // ... then the input iterator's value type, +// it_value_t>:: // Type; // ... else the output iterator's value type // // rocPRIM (as well as Thrust) uses result type of BinaryFunction instead (if not void): // -// using input_type = typename std::iterator_traits::value_type; +// using input_type = detail::it_value_t; // using result_type = ::rocprim::accumulator_t; // // For short -> float using Sum() @@ -618,8 +594,8 @@ template< > struct convert_result_type_wrapper { - using input_type = typename std::iterator_traits::value_type; - using output_type = typename std::iterator_traits::value_type; + using input_type = detail::it_value_t; + using output_type = it_value_t; using result_type = non_void_value_t; convert_result_type_wrapper(BinaryFunction op) : op(op) {} @@ -660,7 +636,7 @@ convert_result_type(BinaryFunction op) template struct convert_binary_result_type_wrapper { - using input_type = typename std::iterator_traits::value_type; + using input_type = detail::it_value_t; using init_type = InitT; using accum_type = accumulator_t; diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_reduce.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_reduce.hpp index 7547974fc008..b9188933220f 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_reduce.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_reduce.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -45,7 +45,7 @@ AccumType { AccumType retval = static_cast(prefix); constexpr int length = ::hipcub::detail::static_size_v(); -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(int i = 0; i < length; ++i) { retval = reduction_op(retval, input[i]); @@ -60,7 +60,7 @@ AccumType ThreadReduceSequential(const InputType& input, ReductionOp reduction_o { AccumType retval = input[0]; constexpr int length = ::hipcub::detail::static_size_v(); -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(int i = 1; i < length; ++i) { retval = reduction_op(retval, input[i]); @@ -76,10 +76,10 @@ __device__ __forceinline__ AccumType ThreadReduceBinaryTree(const InputType& input, ReductionOp reduction_op) { constexpr auto length = ::hipcub::detail::static_size_v(); -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(int i = 1; i < length; i *= 2) { -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(int j = 0; j + i < length; j += i * 2) { input[j] = reduction_op(input[j], input[j + i]); @@ -95,10 +95,10 @@ __device__ __forceinline__ AccumType ThreadReduceTernaryTree(const InputType& input, ReductionOp reduction_op) { constexpr auto length = ::hipcub::detail::static_size_v(); -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(int i = 1; i < length; i *= 3) { -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(int j = 0; j + i < length; j += i * 3) { auto value = reduction_op(input[j], input[j + i]); diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_scan.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_scan.hpp index 4995d72382e6..ae32a2c84063 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_scan.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_scan.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -61,7 +61,7 @@ HIPCUB_FORCEINLINE ScanOp scan_op, ///< [in] Binary scan operator detail::int_constant_t /*length*/) { -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(int i = 0; i < LENGTH; ++i) { inclusive = scan_op(exclusive, input[i]); @@ -144,7 +144,7 @@ HIPCUB_FORCEINLINE ScanOp scan_op, ///< [in] Binary scan operator detail::int_constant_t /*length*/) { -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(int i = 0; i < LENGTH; ++i) { inclusive = scan_op(inclusive, input[i]); diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_search.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_search.hpp index f383d0b4386f..6fca005dc50f 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_search.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_search.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2024, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -32,6 +32,8 @@ #include "../../../config.hpp" +#include _HIPCUB_STD_INCLUDE(functional) + #include BEGIN_HIPCUB_NAMESPACE @@ -55,8 +57,8 @@ __host__ __device__ __forceinline__ void MergePathSearch( OffsetT b_len, CoordinateT& path_coordinate) { - OffsetT split_min = CUB_MAX(diagonal - b_len, 0); - OffsetT split_max = CUB_MIN(diagonal, a_len); + OffsetT split_min = _HIPCUB_STD::max(diagonal - b_len, 0); + OffsetT split_max = _HIPCUB_STD::min(diagonal, a_len); while (split_min < split_max) { @@ -73,7 +75,7 @@ __host__ __device__ __forceinline__ void MergePathSearch( } } - path_coordinate.x = CUB_MIN(split_min, a_len); + path_coordinate.x = _HIPCUB_STD::min(split_min, a_len); path_coordinate.y = diagonal - split_min; } diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_sort.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_sort.hpp index 8e7a97b66f8f..bf3c6937b13a 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_sort.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_sort.hpp @@ -1,6 +1,6 @@ /****************************************************************************** * Copyright (c) 2011-2021, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -31,23 +31,19 @@ #include "../../../config.hpp" +#include "../util_macro.hpp" #include "../util_ptx.hpp" #include "../util_type.hpp" #include // IWYU pragma: export -BEGIN_HIPCUB_NAMESPACE +#include -// Should be deprecated once hip::std::swap is available in this scope. -template -HIPCUB_DEVICE -HIPCUB_FORCEINLINE void Swap(T& lhs, T& rhs) -{ - T temp = lhs; - lhs = rhs; - rhs = temp; -} +#if defined(__HIP_PLATFORM_NVIDIA__) + #include +#endif +BEGIN_HIPCUB_NAMESPACE /** * @brief Sorts data using odd-even sort method @@ -89,22 +85,28 @@ StableOddEvenSort(KeyT (&keys)[ITEMS_PER_THREAD], { constexpr bool KEYS_ONLY = ::rocprim::Equals::VALUE; - #pragma unroll - for (int i = 0; i < ITEMS_PER_THREAD; ++i) + _CCCL_SORT_MAYBE_UNROLL() + for(int i = 0; i < ITEMS_PER_THREAD; ++i) { - #pragma unroll - for (int j = 1 & i; j < ITEMS_PER_THREAD - 1; j += 2) - { - if (compare_op(keys[j + 1], keys[j])) + _CCCL_SORT_MAYBE_UNROLL() + for(int j = 1 & i; j < ITEMS_PER_THREAD - 1; j += 2) { - Swap(keys[j], keys[j + 1]); - if (!KEYS_ONLY) - { - Swap(items[j], items[j + 1]); - } - } - } // inner loop - } // outer loop + if(compare_op(keys[j + 1], keys[j])) + { + +#if defined(__HIP_PLATFORM_NVIDIA__) + using ::cuda::std::swap; +#else + using ::rocprim::swap; +#endif + swap(keys[j], keys[j + 1]); + if(!KEYS_ONLY) + { + swap(items[j], items[j + 1]); + } + } + } // inner loop + } // outer loop } diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_store.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_store.hpp index 775848a122d5..32651a8408b6 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_store.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/thread/thread_store.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -84,7 +84,7 @@ HIPCUB_FORCEINLINE void ThreadStore(OutputIteratorT itr, T val) ThreadStore(itr, val, detail::int_constant_t{}, - ::std::bool_constant<::std::is_pointer::value>()); + ::std::bool_constant<_HIPCUB_STD::is_pointer::value>()); } namespace detail @@ -128,8 +128,5 @@ struct iterate_thread_store } // namespace detail -template -using IterateThreadStore HIPCUB_DEPRECATED = detail::iterate_thread_store; - END_HIPCUB_NAMESPACE #endif diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_macro.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_macro.hpp index 0278f5a5cea0..054811a078f1 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_macro.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_macro.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2011-2026, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -36,68 +36,12 @@ BEGIN_HIPCUB_NAMESPACE -/** - * \addtogroup UtilModule - * @{ - */ - -#ifndef DOXYGEN_SHOULD_SKIP_THIS - #define HIPCUB_PREVENT_MACRO_SUBSTITUTION -template -constexpr __host__ __device__ -auto min HIPCUB_PREVENT_MACRO_SUBSTITUTION(T&& t, U&& u) - -> decltype(t < u ? std::forward(t) : std::forward(u)) -{ - return t < u ? std::forward(t) : std::forward(u); -} - -template -constexpr __host__ __device__ -auto max HIPCUB_PREVENT_MACRO_SUBSTITUTION(T&& t, U&& u) - -> decltype(t < u ? std::forward(u) : std::forward(t)) -{ - return t < u ? std::forward(u) : std::forward(t); -} - #undef HIPCUB_PREVENT_MACRO_SUBSTITUTION -#endif - -/// Deprecated since rocm [7.1] -#ifndef HIPCUB_MAX - /// Select maximum(a, b) - #define HIPCUB_MAX(a, b) (((b) > (a)) ? (b) : (a)) -#endif - -/// Deprecated since rocm [7.1] -#ifndef HIPCUB_MIN - /// Select minimum(a, b) - #define HIPCUB_MIN(a, b) (((b) < (a)) ? (b) : (a)) -#endif - -/// Deprecated since rocm [7.1] -#ifndef HIPCUB_QUOTIENT_FLOOR - /// Quotient of x/y rounded down to nearest integer - #define HIPCUB_QUOTIENT_FLOOR(x, y) ((x) / (y)) -#endif - -/// Deprecated since rocm [7.1] -#ifndef HIPCUB_QUOTIENT_CEILING - /// Quotient of x/y rounded up to nearest integer - #define HIPCUB_QUOTIENT_CEILING(x, y) (((x) + (y)-1) / (y)) -#endif - -/// Deprecated since rocm [7.1] -#ifndef HIPCUB_ROUND_UP_NEAREST - /// x rounded up to the nearest multiple of y - #define HIPCUB_ROUND_UP_NEAREST(x, y) (HIPCUB_QUOTIENT_CEILING(x, y) * y) -#endif - -/// Deprecated since rocm [7.1] -#ifndef HIPCUB_ROUND_DOWN_NEAREST - /// x rounded down to the nearest multiple of y - #define HIPCUB_ROUND_DOWN_NEAREST(x, y) (((x) / (y)) * y) -#endif - -/** @} */ // end group UtilModule +// RAPIDS cuDF needs to avoid unrolling some loops in sort to prevent compile time issues +#if defined(CCCL_AVOID_SORT_UNROLL) + #define _CCCL_SORT_MAYBE_UNROLL() _CCCL_PRAGMA_NOUNROLL() +#else // ^^^ CCCL_AVOID_SORT_UNROLL ^^^ / vvv !CCCL_AVOID_SORT_UNROLL vvv + #define _CCCL_SORT_MAYBE_UNROLL() _CCCL_PRAGMA_UNROLL_FULL() +#endif // !CCCL_AVOID_SORT_UNROLL END_HIPCUB_NAMESPACE diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_ptx.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_ptx.hpp index 3df30881bcd6..b4484569c533 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_ptx.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_ptx.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -44,12 +44,6 @@ BEGIN_HIPCUB_NAMESPACE // * ThreadExit - not supported // * LogicShiftLeft // * LogicShiftRight -// * ThreadTrap - not supported, deprecated in CUB -// * FFMA_RZ, FMUL_RZ - not supported, deprecated in CUB -// * SHFL_IDX_SYNC - not supported, deprecated in CUB -// * WARP_SYNC - deprecated, deprecated in CUB -// * CTA_SYNC_AND - not supported, deprecated in CUB -// * CTA_SYNC_OR - not supported, deprecated in CUB // * MatchAny - not in CUB public API // // Differences: @@ -69,18 +63,6 @@ HIPCUB_FORCEINLINE int RowMajorTid(int block_dim_x, int block_dim_y, int block_d + hipThreadIdx_x; } -HIPCUB_DEPRECATED_BECAUSE("use ::rocprim::lane_id() instead") -HIPCUB_DEVICE HIPCUB_FORCEINLINE unsigned int LaneId() -{ - return ::rocprim::lane_id(); -} - -HIPCUB_DEPRECATED_BECAUSE("use ::rocprim::warp_id instead") -HIPCUB_DEVICE HIPCUB_FORCEINLINE unsigned int WarpId() -{ - return ::rocprim::warp_id(); -} - template HIPCUB_DEVICE HIPCUB_FORCEINLINE uint64_t WarpMask(unsigned int warp_id) @@ -97,34 +79,6 @@ HIPCUB_FORCEINLINE uint64_t WarpMask(unsigned int warp_id) return member_mask; } -// Returns the warp lane mask of all lanes less than the calling thread -HIPCUB_DEPRECATED_BECAUSE("use ::rocprim::get_sreg_lanemask_lt instead") -HIPCUB_DEVICE HIPCUB_FORCEINLINE uint64_t LaneMaskLt() -{ - return (uint64_t(1) << ::rocprim::lane_id()) - 1; -} - -// Returns the warp lane mask of all lanes less than or equal to the calling thread -HIPCUB_DEPRECATED_BECAUSE("use ::rocprim::get_sreg_lanemask_le instead") -HIPCUB_DEVICE HIPCUB_FORCEINLINE uint64_t LaneMaskLe() -{ - return ((uint64_t(1) << ::rocprim::lane_id()) << 1) - 1; -} - -// Returns the warp lane mask of all lanes greater than the calling thread -HIPCUB_DEPRECATED_BECAUSE("use ::rocprim::get_sreg_lanemask_gt instead") -HIPCUB_DEVICE HIPCUB_FORCEINLINE uint64_t LaneMaskGt() -{ - return uint64_t(-1)^LaneMaskLe(); -} - -// Returns the warp lane mask of all lanes greater than or equal to the calling thread -HIPCUB_DEPRECATED_BECAUSE("use ::rocprim::get_sreg_lanemask_ge instead") -HIPCUB_DEVICE HIPCUB_FORCEINLINE uint64_t LaneMaskGe() -{ - return uint64_t(-1)^LaneMaskLt(); -} - // Shuffle funcs template @@ -167,22 +121,6 @@ HIPCUB_FORCEINLINE T ShuffleIndex(T input, int src_lane, unsigned int member_mas ); } -// Other - -HIPCUB_DEPRECATED_BECAUSE("will be removed in the next major release") -HIPCUB_DEVICE HIPCUB_FORCEINLINE - unsigned int SHR_ADD(unsigned int x, unsigned int shift, unsigned int addend) -{ - return (x >> shift) + addend; -} - -HIPCUB_DEPRECATED_BECAUSE("will be removed in the next major release") -HIPCUB_DEVICE HIPCUB_FORCEINLINE - unsigned int SHL_ADD(unsigned int x, unsigned int shift, unsigned int addend) -{ - return (x << shift) + addend; -} - namespace detail { template @@ -217,20 +155,23 @@ HIPCUB_FORCEINLINE auto // Extracts \p num_bits from \p source starting at bit-offset \p bit_start. // The input \p source may be an 8b, 16b, 32b, or 64b unsigned integer type. template +//! deprecated [Since 5.0] +HIPCUB_DEPRECATED_BECAUSE("Use hip::bitfield_extract()") HIPCUB_DEVICE -HIPCUB_FORCEINLINE unsigned int - BFE(UnsignedBits source, unsigned int bit_start, unsigned int num_bits) +HIPCUB_FORCEINLINE + unsigned int BFE(UnsignedBits source, unsigned int bit_start, unsigned int num_bits) { static_assert(std::is_unsigned::value, "UnsignedBits must be unsigned"); return detail::unsigned_bit_extract(source, bit_start, num_bits); } -#if HIPCUB_IS_INT128_ENABLED +#if _CCCL_HAS_INT128() /** * Bitfield-extract for 128-bit types. */ template -HIPCUB_DEVICE +//! deprecated [Since 5.0] +HIPCUB_DEPRECATED_BECAUSE("Use hip::bitfield_extract()") HIPCUB_DEVICE HIPCUB_FORCEINLINE unsigned int BFE(UnsignedBits source, unsigned int bit_start, unsigned int num_bits, @@ -241,78 +182,6 @@ HIPCUB_FORCEINLINE unsigned int BFE(UnsignedBits source, } #endif -// Bitfield insert. -// Inserts the \p num_bits least significant bits of \p y into \p x at bit-offset \p bit_start. -HIPCUB_DEPRECATED_BECAUSE("will be removed in the next major release") -HIPCUB_DEVICE HIPCUB_FORCEINLINE void BFI(unsigned int& ret, - unsigned int x, - unsigned int y, - unsigned int bit_start, - unsigned int num_bits) -{ - #ifdef __HIP_PLATFORM_AMD__ - ret = __bitinsert_u32(x, y, bit_start, num_bits); - #else - x <<= bit_start; - unsigned int MASK_X = ((1 << num_bits) - 1) << bit_start; - unsigned int MASK_Y = ~MASK_X; - ret = (y & MASK_Y) | (x & MASK_X); - #endif // __HIP_PLATFORM_AMD__ -} - -HIPCUB_DEPRECATED_BECAUSE("will be removed in the next major release") -HIPCUB_DEVICE HIPCUB_FORCEINLINE unsigned int IADD3(unsigned int x, unsigned int y, unsigned int z) -{ - return x + y + z; -} - -HIPCUB_DEPRECATED_BECAUSE("will be removed in the next major release") -HIPCUB_DEVICE HIPCUB_FORCEINLINE int PRMT(unsigned int a, unsigned int b, unsigned int index) -{ - return ::__byte_perm(a, b, index); -} - -HIPCUB_DEPRECATED_BECAUSE("will be removed in the next major release") -HIPCUB_DEVICE HIPCUB_FORCEINLINE void BAR(int count) -{ - (void) count; - __syncthreads(); -} - -HIPCUB_DEPRECATED_BECAUSE("use __syncthreads() instead") -HIPCUB_DEVICE HIPCUB_FORCEINLINE void CTA_SYNC() -{ - __syncthreads(); -} - -HIPCUB_DEPRECATED_BECAUSE("use ::rocprim::wave_barrier() instead") -HIPCUB_DEVICE HIPCUB_FORCEINLINE void WARP_SYNC(unsigned int member_mask) -{ - (void) member_mask; - ::rocprim::wave_barrier(); -} - -HIPCUB_DEPRECATED_BECAUSE("use ::__any(predicate) instead") -HIPCUB_DEVICE HIPCUB_FORCEINLINE int WARP_ANY(int predicate, uint64_t member_mask) -{ - (void) member_mask; - return ::__any(predicate); -} - -HIPCUB_DEPRECATED_BECAUSE("use ::__all(predicate) instead") -HIPCUB_DEVICE HIPCUB_FORCEINLINE int WARP_ALL(int predicate, uint64_t member_mask) -{ - (void) member_mask; - return ::__all(predicate); -} - -HIPCUB_DEPRECATED_BECAUSE("use ::__ballot(predicate) instead") -HIPCUB_DEVICE HIPCUB_FORCEINLINE int64_t WARP_BALLOT(int predicate, uint64_t member_mask) -{ - (void) member_mask; - return __ballot(predicate); -} - namespace detail { diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_sync.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_sync.hpp index 0ea8de316417..8a798b3ead67 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_sync.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_sync.hpp @@ -1,5 +1,5 @@ /****************************************************************************** - * Copyright (c) 2024, Advanced Micro Devices, Inc. All rights reserved. + * Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -41,7 +41,7 @@ { \ return _error; \ } \ - if HIPCUB_IF_CONSTEXPR(HIPCUB_DETAIL_DEBUG_SYNC_VALUE) \ + if constexpr(HIPCUB_DETAIL_DEBUG_SYNC_VALUE) \ { \ std::cout << name << "(" << size << ")"; \ auto __error = hipStreamSynchronize(stream); \ diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_temporary_storage.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_temporary_storage.hpp index c2e465393daf..f5bef3e3c4b5 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_temporary_storage.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_temporary_storage.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2024, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2024-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -42,7 +42,7 @@ BEGIN_HIPCUB_NAMESPACE namespace detail { // Base case: When N == 0 -template +template HIPCUB_HOST_DEVICE typename std::enable_if::type generate_partition(void* d_temp_storage, size_t& temp_storage_bytes, @@ -56,7 +56,7 @@ typename std::enable_if::type generate_partition(void* d_t } // Recursive case: When N > 0 -template +template HIPCUB_HOST_DEVICE typename std::enable_if<(N > 0), hipError_t>::type generate_partition(void* d_temp_storage, size_t& temp_storage_bytes, Generator gen, Ts... args) diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_type.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_type.hpp index 52ae8a3b3dfc..c17208595c9e 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_type.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/util_type.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -42,7 +42,9 @@ #include #include -#include +#include _HIPCUB_STD_INCLUDE(limits) +#include _HIPCUB_STD_INCLUDE(iterator) + #include BEGIN_HIPCUB_NAMESPACE @@ -53,9 +55,12 @@ using NullType = ::rocprim::empty_type; #endif -#ifndef HIPCUB_IS_INT128_ENABLED - #define HIPCUB_IS_INT128_ENABLED 1 -#endif // !defined(HIPCUB_IS_INT128_ENABLED) +// This API needs to be deprecated once libhipcxx is available. +#if defined(__SIZEOF_INT128__) + #define _CCCL_HAS_INT128() 1 +#else + #define _CCCL_HAS_INT128() 0 +#endif template struct [[deprecated("[Since 1.16] If is deprecated use std::conditional instead.")]] If @@ -90,6 +95,30 @@ struct PowerOfTwo namespace detail { +// the following iterator helpers are not named iter_value_t etc, like the C++20 facilities, because they are defined in +// terms of C++17 iterator_traits and not the new C++20 indirectly_readable trait etc. This allows them to detect nested +// value_type, difference_type and reference aliases, which the new C+20 traits do not consider (they only consider +// specializations of iterator_traits). Also, a value_type of void remains supported (needed by some output iterators). + +template +using it_value_t = typename _HIPCUB_STD::iterator_traits::value_type; + +template +using it_reference_t = typename _HIPCUB_STD::iterator_traits::reference; + +template +using it_difference_t = typename _HIPCUB_STD::iterator_traits::difference_type; + +template +using it_pointer_t = typename _HIPCUB_STD::iterator_traits::pointer; + +// use this whenever you need to lazily evaluate a trait. E.g., as an alternative in replace_if_use_default. +template typename Trait, typename... Args> +struct lazy_trait +{ + using type = Trait; +}; + template struct Log2Impl { @@ -147,12 +176,6 @@ struct DoubleBuffer } }; -template -struct HIPCUB_DEPRECATED_BECAUSE("Use ::std::integral_constant instead") Int2Type -{ - enum {VALUE = A}; -}; - #ifndef DOXYGEN_SHOULD_SKIP_THIS // Do not document template< @@ -196,8 +219,9 @@ using is_integral_or_enum = } -// CUB deprecated this API, and suggests to use `::cuda::ceil_div` instead, +// CUB removed this API, and suggests to use `::cuda::ceil_div` instead, // which is implemented in file `libcudacxx/include/cuda/__cmath/ceil_div.h`. +// Remove when hip::ceil_div is implemented. template HIPCUB_DEPRECATED_BECAUSE("Use hip::ceil_div instead from 'libhipcxx'") HIPCUB_HOST_DEVICE __forceinline__ constexpr NumeratorT @@ -467,9 +491,9 @@ struct Uninitialized /// Biggest memory-access word that T is a whole multiple of and is not larger than the alignment of T using DeviceWord = typename UnitWord::DeviceWord; - static constexpr std::size_t DATA_SIZE = sizeof(T); - static constexpr std::size_t WORD_SIZE = sizeof(DeviceWord); - static constexpr std::size_t WORDS = DATA_SIZE / WORD_SIZE; + static constexpr size_t DATA_SIZE = sizeof(T); + static constexpr size_t WORD_SIZE = sizeof(DeviceWord); + static constexpr size_t WORDS = DATA_SIZE / WORD_SIZE; /// Backing storage DeviceWord storage[WORDS]; @@ -492,7 +516,7 @@ struct Uninitialized * This enum is deprecated, please use instead. Or if you have * libhipcxx, please use the type_traits system in libhipcxx. */ -enum HIPCUB_DEPRECATED_BECAUSE("Use instead.") Category +enum Category { NOT_A_NUMBER, SIGNED_INTEGER, @@ -500,48 +524,27 @@ enum HIPCUB_DEPRECATED_BECAUSE("Use instead.") Category FLOATING_POINT }; -/** - * \brief Basic type traits - */ -HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH -template +namespace detail +{ +struct is_primitive_impl; + +template struct BaseTraits { - /// Category - HIPCUB_DEPRECATED_BECAUSE("Use instead.") - static constexpr Category CATEGORY = _CATEGORY; - enum - { - PRIMITIVE HIPCUB_DEPRECATED_BECAUSE("Use instead.") = _PRIMITIVE, - nullptr_TYPE = _nullptr_TYPE, - }; +private: + friend struct is_primitive_impl; + + static constexpr bool is_primitive = _PRIMITIVE; }; -HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP -/** - * Basic type traits (unsigned primitive specialization) - */ -HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH -template -struct BaseTraits +template +struct BaseTraits { - using UnsignedBits = _UnsignedBits; - HIPCUB_DEPRECATED_BECAUSE("Use instead.") - static constexpr Category CATEGORY = UNSIGNED_INTEGER; + using UnsignedBits = _UnsignedBits; static constexpr UnsignedBits LOWEST_KEY = UnsignedBits(0); static constexpr UnsignedBits MAX_KEY = UnsignedBits(-1); - enum - { - PRIMITIVE HIPCUB_DEPRECATED_BECAUSE("Use instead.") = true, - nullptr_TYPE = false, - }; - using key_codec = decltype(::rocprim::traits::get().template radix_key_codec()); static HIPCUB_HOST_DEVICE __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key) @@ -554,6 +557,8 @@ struct BaseTraits return key; } + //! deprecated [Since 5.0] + HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::max()") static HIPCUB_HOST_DEVICE __forceinline__ T Max() { UnsignedBits retval_bits = MAX_KEY; @@ -562,6 +567,8 @@ struct BaseTraits return retval; } + //! deprecated [Since 5.0] + HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::lowest()") static HIPCUB_HOST_DEVICE __forceinline__ T Lowest() { UnsignedBits retval_bits = LOWEST_KEY; @@ -569,30 +576,22 @@ struct BaseTraits memcpy(&retval, &retval_bits, sizeof(T)); return retval; } + +private: + friend struct is_primitive_impl; + + static constexpr bool is_primitive = true; }; -HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP -/** - * Basic type traits (signed primitive specialization) - */ -HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH -template -struct BaseTraits +template +struct BaseTraits { using UnsignedBits = _UnsignedBits; - HIPCUB_DEPRECATED_BECAUSE("Use instead.") - static constexpr Category CATEGORY = SIGNED_INTEGER; static constexpr UnsignedBits HIGH_BIT = UnsignedBits(1) << ((sizeof(UnsignedBits) * 8) - 1); static constexpr UnsignedBits LOWEST_KEY = HIGH_BIT; static constexpr UnsignedBits MAX_KEY = UnsignedBits(-1) ^ HIGH_BIT; - enum - { - PRIMITIVE HIPCUB_DEPRECATED_BECAUSE("Use instead.") = true, - nullptr_TYPE = false, - }; - using key_codec = decltype(::rocprim::traits::get().template radix_key_codec()); static HIPCUB_HOST_DEVICE __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key) @@ -605,103 +604,39 @@ struct BaseTraits return key ^ HIGH_BIT; }; + //! deprecated [Since 5.0] + HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::max()") static HIPCUB_HOST_DEVICE __forceinline__ T Max() { UnsignedBits retval = MAX_KEY; return reinterpret_cast(retval); } + //! deprecated [Since 5.0] + HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::lowest()") static HIPCUB_HOST_DEVICE __forceinline__ T Lowest() { UnsignedBits retval = LOWEST_KEY; return reinterpret_cast(retval); } -}; -HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP - -// This API needs to be deprecated once libhipcxx is available. -template -struct FpLimits; - -// This API needs to be deprecated once libhipcxx is available. -template <> -struct FpLimits -{ - static HIPCUB_HOST_DEVICE __forceinline__ float Max() { - return std::numeric_limits::max(); - } - - static HIPCUB_HOST_DEVICE __forceinline__ float Lowest() { - return std::numeric_limits::max() * float(-1); - } -}; -// This API needs to be deprecated once libhipcxx is available. -template <> -struct FpLimits -{ - static HIPCUB_HOST_DEVICE __forceinline__ double Max() { - return std::numeric_limits::max(); - } - - static HIPCUB_HOST_DEVICE __forceinline__ double Lowest() { - return std::numeric_limits::max() * double(-1); - } -}; - -// This API needs to be deprecated once libhipcxx is available. -template <> -struct FpLimits<__half> -{ - static HIPCUB_HOST_DEVICE __forceinline__ __half Max() { - unsigned short max_word = 0x7BFF; - return reinterpret_cast<__half&>(max_word); - } - - static HIPCUB_HOST_DEVICE __forceinline__ __half Lowest() { - unsigned short lowest_word = 0xFBFF; - return reinterpret_cast<__half&>(lowest_word); - } -}; - -// This API needs to be deprecated once libhipcxx is available. -template <> -struct FpLimits -{ - static HIPCUB_HOST_DEVICE __forceinline__ hip_bfloat16 Max() { - unsigned short max_word = 0x7F7F; - return reinterpret_cast(max_word); - } +private: + friend struct is_primitive_impl; - static HIPCUB_HOST_DEVICE __forceinline__ hip_bfloat16 Lowest() { - unsigned short lowest_word = 0xFF7F; - return reinterpret_cast(lowest_word); - } + static constexpr bool is_primitive = true; }; -/** - * Basic type traits (fp primitive specialization) - */ -HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH -template -struct BaseTraits +template +struct BaseTraits { using UnsignedBits = _UnsignedBits; - HIPCUB_DEPRECATED_BECAUSE("Use instead.") - static constexpr Category CATEGORY = FLOATING_POINT; static constexpr UnsignedBits HIGH_BIT = UnsignedBits(1) << ((sizeof(UnsignedBits) * 8) - 1); static constexpr UnsignedBits LOWEST_KEY = UnsignedBits(-1); static constexpr UnsignedBits MAX_KEY = UnsignedBits(-1) ^ HIGH_BIT; using key_codec = decltype(::rocprim::traits::get().template radix_key_codec()); - enum - { - PRIMITIVE HIPCUB_DEPRECATED_BECAUSE("Use instead.") = true, - nullptr_TYPE = false, - }; - static HIPCUB_HOST_DEVICE __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key) { UnsignedBits mask = (key & HIGH_BIT) ? UnsignedBits(-1) : HIGH_BIT; @@ -714,52 +649,103 @@ struct BaseTraits return key ^ mask; }; - static HIPCUB_HOST_DEVICE __forceinline__ T Max() { - return FpLimits::Max(); + //! deprecated [Since 5.0] + HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::max()") + static HIPCUB_HOST_DEVICE __forceinline__ + T Max() + { + return _HIPCUB_STD::numeric_limits::max(); } - static HIPCUB_HOST_DEVICE __forceinline__ T Lowest() { - return FpLimits::Lowest(); + //! deprecated [Since 5.0] + HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::lowest()") + static HIPCUB_HOST_DEVICE __forceinline__ + T Lowest() + { + return _HIPCUB_STD::numeric_limits::lowest(); } + +private: + friend struct is_primitive_impl; + + static constexpr bool is_primitive = true; }; -HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP +} // namespace detail -/** - * \brief Numeric type traits - */ -HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH -template struct NumericTraits : BaseTraits {}; +//! Use this class as base when specializing \ref NumericTraits for primitive signed/unsigned integers or floating-point +//! types. +template +using BaseTraits = detail::BaseTraits<_CATEGORY, _PRIMITIVE, _UnsignedBits, T>; -template <> struct NumericTraits : BaseTraits {}; +//! Numeric type traits for radix sort key operations, decoupled lookback and tuning. You can specialize this template +//! for your own types if: +//! * There is an unsigned integral type of equal size +//! * The size of the type is smaller than 64bits +//! * The arithmetic throughput of the type is similar to other built-in types of the same size +//! For other types, if you want to use them with radix sort, please use the decomposer interface of the radix sort. -template <> struct NumericTraits : BaseTraits<(std::numeric_limits::is_signed) ? SIGNED_INTEGER : UNSIGNED_INTEGER, true, false, unsigned char, char> {}; -template <> struct NumericTraits : BaseTraits {}; -template <> struct NumericTraits : BaseTraits {}; -template <> struct NumericTraits : BaseTraits {}; -template <> struct NumericTraits : BaseTraits {}; -template <> struct NumericTraits : BaseTraits {}; +template +struct NumericTraits : BaseTraits +{}; + +template<> +struct NumericTraits : BaseTraits +{}; + +template<> +struct NumericTraits + : BaseTraits<(_HIPCUB_STD::numeric_limits::is_signed) ? SIGNED_INTEGER : UNSIGNED_INTEGER, + true, + unsigned char, + char> +{}; +template<> +struct NumericTraits : BaseTraits +{}; +template<> +struct NumericTraits : BaseTraits +{}; +template<> +struct NumericTraits : BaseTraits +{}; +template<> +struct NumericTraits : BaseTraits +{}; +template<> +struct NumericTraits : BaseTraits +{}; -template <> struct NumericTraits : BaseTraits {}; -template <> struct NumericTraits : BaseTraits {}; -template <> struct NumericTraits : BaseTraits {}; -template <> struct NumericTraits : BaseTraits {}; -template <> struct NumericTraits : BaseTraits {}; +template<> +struct NumericTraits + : BaseTraits +{}; +template<> +struct NumericTraits + : BaseTraits +{}; +template<> +struct NumericTraits + : BaseTraits +{}; +template<> +struct NumericTraits + : BaseTraits +{}; +template<> +struct NumericTraits + : BaseTraits +{}; - #if HIPCUB_IS_INT128_ENABLED + #if _CCCL_HAS_INT128() template<> struct NumericTraits<__uint128_t> { using T = __uint128_t; using UnsignedBits = __uint128_t; - static constexpr Category CATEGORY = UNSIGNED_INTEGER; static constexpr UnsignedBits LOWEST_KEY = UnsignedBits(0); static constexpr UnsignedBits MAX_KEY = UnsignedBits(-1); - HIPCUB_DEPRECATED_BECAUSE("Use instead.") - static constexpr bool PRIMITIVE = false; - static constexpr bool nullptr_TYPE = false; - using key_codec = decltype(::rocprim::traits::get().template radix_key_codec()); static __host__ __device__ __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key) @@ -772,11 +758,15 @@ struct NumericTraits<__uint128_t> return key; } + //! deprecated [Since 5.0] + HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::max()") static __host__ __device__ __forceinline__ T Max() { return MAX_KEY; } + //! deprecated [Since 5.0] + HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::lowest()") static __host__ __device__ __forceinline__ T Lowest() { return LOWEST_KEY; @@ -789,15 +779,10 @@ struct NumericTraits<__int128_t> using T = __int128_t; using UnsignedBits = __uint128_t; - static constexpr Category CATEGORY = SIGNED_INTEGER; static constexpr UnsignedBits HIGH_BIT = UnsignedBits(1) << ((sizeof(UnsignedBits) * 8) - 1); static constexpr UnsignedBits LOWEST_KEY = HIGH_BIT; static constexpr UnsignedBits MAX_KEY = UnsignedBits(-1) ^ HIGH_BIT; - HIPCUB_DEPRECATED_BECAUSE("Use instead.") - static constexpr bool PRIMITIVE = false; - static constexpr bool nullptr_TYPE = false; - using key_codec = decltype(::rocprim::traits::get().template radix_key_codec()); static __host__ __device__ __forceinline__ UnsignedBits TwiddleIn(UnsignedBits key) @@ -810,42 +795,86 @@ struct NumericTraits<__int128_t> return key ^ HIGH_BIT; }; + //! deprecated [Since 5.0] + HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::max()") static __host__ __device__ __forceinline__ T Max() { UnsignedBits retval = MAX_KEY; return reinterpret_cast(retval); } + //! deprecated [Since 5.0] + HIPCUB_DEPRECATED_BECAUSE("Use hip::std::numeric_limits::lowest()") static __host__ __device__ __forceinline__ T Lowest() { UnsignedBits retval = LOWEST_KEY; return reinterpret_cast(retval); } + +private: + friend struct detail::is_primitive_impl; + + static constexpr bool is_primitive = false; }; #endif -template <> struct NumericTraits : BaseTraits {}; -template <> struct NumericTraits : BaseTraits {}; -template <> struct NumericTraits<__half> : BaseTraits {}; -template <> struct NumericTraits : BaseTraits {}; +template<> +struct NumericTraits : BaseTraits +{}; +template<> +struct NumericTraits : BaseTraits +{}; +template<> +struct NumericTraits<__half> : BaseTraits +{ + using UnsignedBits = unsigned short; +}; +template<> +struct NumericTraits + : BaseTraits +{ + using UnsignedBits = unsigned short; +}; -template <> struct NumericTraits : BaseTraits::VolatileWord, bool> {}; -HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP +template<> +struct NumericTraits + : BaseTraits::VolatileWord, bool> +{}; -/** - * \brief Type traits - */ +namespace detail +{ template struct Traits : NumericTraits::type> {}; +} // namespace detail + +//! \brief Query type traits for radix sort key operations, decoupled lookback and tunings. To add support for your own +//! primitive types please specialize \ref NumericTraits. +template +using Traits = detail::Traits; + namespace detail { -// __uint128_t and __int128_t are not primitive +// we cannot befriend is_primitive on GCC < 11, since it's a template (bug) +struct is_primitive_impl +{ + // must be a struct instead of an alias, so the access of Traits::is_primitive happens in the context of this class + template + struct is_primitive : _HIPCUB_STD::bool_constant::is_primitive> + {}; +}; +// This trait serves two purposes: +// 1. It is used for tunings to detect whether we have a build-in arithmetic type for which we can expect certain +// arithmetic throughput. E.g.: we expect all primitive types of the same size to show roughly similar performance. +// 2. Decoupled lookback uses this trait to determine whether there is a machine word twice the size of T which can be +// loaded/stored with a single instruction. +// TODO(bgruber): for 2. we should probably just check whether sizeof(T) * 2 <= sizeof(int128) (or 256-bit on SM100) +// Users must be able to hook into both scenarios with their custom types, so this trait must depend on cub::Traits HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH template -struct is_primitive : ::std::bool_constant::PRIMITIVE> +struct is_primitive : is_primitive_impl::is_primitive {}; template @@ -878,8 +907,8 @@ template struct is_extended_fp : std::integral_constant< bool, - std::is_same<__half, typename std::remove_cv::type>::value - || std::is_same::type>::value> + _HIPCUB_STD::is_same<__half, typename std::remove_cv::type>::value + || _HIPCUB_STD::is_same::type>::value> {}; // Gets "raw" type: drops reference and const qualifier. diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_exchange.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_exchange.hpp index 20a85eab8d5f..7fac1a910fbc 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_exchange.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_exchange.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -63,7 +63,6 @@ using InternalWarpExchangeImpl template class WarpExchange : private detail::InternalWarpExchangeImpl struct LoadInternal { - using WarpExchangeT = WarpExchange< - InputT, - ITEMS_PER_THREAD, - LOGICAL_WARP_THREADS, - ARCH - >; + using WarpExchangeT = WarpExchange; using TempStorage = typename WarpExchangeT::TempStorage; TempStorage& temp_storage; int linear_tid; diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_reduce.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_reduce.hpp index 3d6e938b7337..c17896227c4e 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_reduce.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_reduce.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,7 +33,6 @@ #include "../../../config.hpp" #include "../util_ptx.hpp" -#include "../thread/thread_operators.hpp" #include // IWYU pragma: export diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_scan.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_scan.hpp index 60a20e6ecb9d..b5c6071d6c7c 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_scan.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_scan.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,7 +33,6 @@ #include "../../../config.hpp" #include "../util_ptx.hpp" -#include "../thread/thread_operators.hpp" #include // IWYU pragma: export diff --git a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_store.hpp b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_store.hpp index 23e0c52f6cda..a3be7d521c6d 100644 --- a/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_store.hpp +++ b/projects/hipcub/hipcub/include/hipcub/backend/rocprim/warp/warp_store.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -200,12 +200,7 @@ class WarpStore template <> struct StoreInternal { - using WarpExchangeT = WarpExchange< - T, - ITEMS_PER_THREAD, - LOGICAL_WARP_THREADS, - ARCH - >; + using WarpExchangeT = WarpExchange; using TempStorage = typename WarpExchangeT::TempStorage; TempStorage& temp_storage; int linear_tid; diff --git a/projects/hipcub/hipcub/include/hipcub/config.hpp b/projects/hipcub/hipcub/include/hipcub/config.hpp index 18ad351a093b..e5ed5a94e07c 100644 --- a/projects/hipcub/hipcub/include/hipcub/config.hpp +++ b/projects/hipcub/hipcub/include/hipcub/config.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2019-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2019-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -35,6 +35,14 @@ // Version #include "hipcub_version.hpp" // IWYU pragma: export +// Manage std implementation +#include "libcxx.hpp" // IWYU pragma: export + +// For _CCCL_IMPLICIT_SYSTEM_HEADER +#if _HIPCUB_HAS_DEVICE_SYSTEM_STD + #include _HIPCUB_LIBCXX_INCLUDE(__cccl_config) // IWYU pragma: export +#endif + #define HIPCUB_NAMESPACE hipcub // Inline namespace (e.g. HIPCUB_300400_NS where 300400 is the hipCUB version) is used to @@ -112,6 +120,8 @@ END_HIPCUB_NAMESPACE #define HIPCUB_RUNTIME_FUNCTION CUB_RUNTIME_FUNCTION #include + #include + #include #define HIPCUB_WARP_THREADS CUB_PTX_WARP_THREADS #define HIPCUB_DEVICE_WARP_THREADS CUB_PTX_WARP_THREADS #define HIPCUB_HOST_WARP_THREADS CUB_PTX_WARP_THREADS @@ -200,19 +210,6 @@ END_HIPCUB_NAMESPACE #define HipcubLog(msg) ::hipcub::Log(msg, __FILE__, __LINE__) #endif -#if __cpp_if_constexpr - #define HIPCUB_IF_CONSTEXPR constexpr -#else - #if defined(_MSC_VER) && !defined(__clang__) - // MSVC (and not Clang pretending to be MSVC) unconditionally exposes if constexpr (even in C++14 mode), - // moreover it triggers warning C4127 (conditional expression is constant) when not using it. nvcc will - // be calling cl.exe for host-side codegen. - #define HIPCUB_IF_CONSTEXPR constexpr - #else - #define HIPCUB_IF_CONSTEXPR - #endif -#endif - #ifdef DOXYGEN_SHOULD_SKIP_THIS // Documentation only /// \def HIPCUB_DEBUG_SYNC @@ -242,4 +239,13 @@ END_HIPCUB_NAMESPACE #endif #endif // HIPCUB_ROCPRIM_API +// This API needs to be deprecated once libhipcxx is available. +#if !defined(_CCCL_PRAGMA_UNROLL_FULL) + #define _CCCL_PRAGMA_UNROLL_FULL() _Pragma("unroll") +#endif // !defined(_CCCL_PRAGMA_UNROLL_FULL) + +#if !defined(_CCCL_PRAGMA_NOUNROLL) + #define _CCCL_PRAGMA_NOUNROLL() _Pragma("nounroll") +#endif // !defined(_CCCL_PRAGMA_NOUNROLL) + #endif // HIPCUB_CONFIG_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/device/device_spmv.hpp b/projects/hipcub/hipcub/include/hipcub/device/device_spmv.hpp deleted file mode 100644 index b32fc2811d30..000000000000 --- a/projects/hipcub/hipcub/include/hipcub/device/device_spmv.hpp +++ /dev/null @@ -1,39 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2020-2025, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ - -#ifndef HIPCUB_DEVICE_DEVICE_SPMV_HPP_ -#define HIPCUB_DEVICE_DEVICE_SPMV_HPP_ - -#ifdef __HIP_PLATFORM_AMD__ - #include "../backend/rocprim/device/device_spmv.hpp" // IWYU pragma: export -#elif defined(__HIP_PLATFORM_NVIDIA__) - #include "../backend/cub/device/device_spmv.hpp" // IWYU pragma: export -#endif - -#endif // HIPCUB_DEVICE_DEVICE_SELECT_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/grid/grid_barrier.hpp b/projects/hipcub/hipcub/include/hipcub/grid/grid_barrier.hpp deleted file mode 100644 index 8c8863be6474..000000000000 --- a/projects/hipcub/hipcub/include/hipcub/grid/grid_barrier.hpp +++ /dev/null @@ -1,40 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2011, Duane Merrill. All rights reserved. - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ - -#ifndef HIPCUB_GRID_GRID_BARRIER_HPP_ -#define HIPCUB_GRID_GRID_BARRIER_HPP_ - -#ifdef __HIP_PLATFORM_AMD__ - #include "../backend/rocprim/grid/grid_barrier.hpp" // IWYU pragma: export -#elif defined(__HIP_PLATFORM_NVIDIA__) - #include "../backend/cub/grid/grid_barrier.hpp" // IWYU pragma: export - #include "../config.hpp" -#endif - -#endif // HIPCUB_GRID_GRID_BARRIER_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/hipcub_version.hpp.in b/projects/hipcub/hipcub/include/hipcub/hipcub_version.hpp.in index 86790382e504..5e72747ee94d 100644 --- a/projects/hipcub/hipcub/include/hipcub/hipcub_version.hpp.in +++ b/projects/hipcub/hipcub/include/hipcub/hipcub_version.hpp.in @@ -1,4 +1,4 @@ -// Copyright (c) 2018-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2018-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal diff --git a/projects/hipcub/hipcub/include/hipcub/iterator/constant_input_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/iterator/constant_input_iterator.hpp deleted file mode 100644 index 7fc28707274d..000000000000 --- a/projects/hipcub/hipcub/include/hipcub/iterator/constant_input_iterator.hpp +++ /dev/null @@ -1,40 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ - -#ifndef HIPCUB_CONSTANT_INPUT_ITERATOR_HPP_ -#define HIPCUB_CONSTANT_INPUT_ITERATOR_HPP_ - -#ifdef __HIP_PLATFORM_AMD__ - #include "../backend/rocprim/iterator/constant_input_iterator.hpp" // IWYU pragma: export -#elif defined(__HIP_PLATFORM_NVIDIA__) - #include "../config.hpp" - #include // IWYU pragma: export -#endif - -#endif // HIPCUB_ITERATOR_DISCARD_OUTPUT__HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/iterator/counting_input_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/iterator/counting_input_iterator.hpp deleted file mode 100644 index 723af464aae3..000000000000 --- a/projects/hipcub/hipcub/include/hipcub/iterator/counting_input_iterator.hpp +++ /dev/null @@ -1,41 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ - -#ifndef HIPCUB_COUNTING_INPUT_ITERATOR_HPP_ -#define HIPCUB_COUNTING_INPUT_ITERATOR_HPP_ - -#ifdef __HIP_PLATFORM_AMD__ - #include "../backend/rocprim/iterator/counting_input_iterator.hpp" // IWYU pragma: export -#elif defined(__HIP_PLATFORM_NVIDIA__) - #include "../config.hpp" - #include // IWYU pragma: export -#endif - -#endif // HIPCUB_ITERATOR_DISCARD_OUTPUT__HPP_ - diff --git a/projects/hipcub/hipcub/include/hipcub/iterator/discard_output_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/iterator/discard_output_iterator.hpp deleted file mode 100644 index a2c8d6bedd6c..000000000000 --- a/projects/hipcub/hipcub/include/hipcub/iterator/discard_output_iterator.hpp +++ /dev/null @@ -1,40 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2020-2025, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ - -#ifndef HIPCUB_ITERATOR_DISCARD_OUTPUT_HPP_ -#define HIPCUB_ITERATOR_DISCARD_OUTPUT_HPP_ - -#ifdef __HIP_PLATFORM_AMD__ - #include "../backend/rocprim/iterator/discard_output_iterator.hpp" // IWYU pragma: export -#elif defined(__HIP_PLATFORM_NVIDIA__) - #include "../config.hpp" - #include // IWYU pragma: export -#endif - -#endif // HIPCUB_ITERATOR_DISCARD_OUTPUT__HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/iterator/transform_input_iterator.hpp b/projects/hipcub/hipcub/include/hipcub/iterator/transform_input_iterator.hpp deleted file mode 100644 index b2ce4b9f20fa..000000000000 --- a/projects/hipcub/hipcub/include/hipcub/iterator/transform_input_iterator.hpp +++ /dev/null @@ -1,40 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2020-2025, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ - -#ifndef HIPCUB_ITERATOR_TRANSFORM_INPUT_ITERATOR_HPP_ -#define HIPCUB_ITERATOR_TRANSFORM_INPUT_ITERATOR_HPP_ - -#ifdef __HIP_PLATFORM_AMD__ - #include "../backend/rocprim/iterator/transform_input_iterator.hpp" // IWYU pragma: export -#elif defined(__HIP_PLATFORM_NVIDIA__) - #include "../config.hpp" - #include // IWYU pragma: export -#endif - -#endif // HIPCUB_ITERATOR_TRANSFORM_INPUT_ITERATOR_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/libcxx.hpp b/projects/hipcub/hipcub/include/hipcub/libcxx.hpp new file mode 100644 index 000000000000..7ab49d3969fd --- /dev/null +++ b/projects/hipcub/hipcub/include/hipcub/libcxx.hpp @@ -0,0 +1,103 @@ +// MIT License +// +// Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#ifndef HIPCUB_LIBCXX_HPP_ +#define HIPCUB_LIBCXX_HPP_ + +#pragma once + +// This is a utility file that helps managing which +// 'std' implementation we're using. The provided +// macros are for internal use only and may change +// in future versions. +// +// Example usage: +// #include _HIPCUB_STD_INCLUDE(optional) +// using optional_int = _HIPCUB_STD::optional; + +// Minimum version that we depend on. +#define _HIPCUB_REQUIRED_LIBCXX_VERSION_MAJOR 3 +#define _HIPCUB_REQUIRED_LIBCXX_VERSION_MINOR 0 +#define _HIPCUB_REQUIRED_LIBCXX_VERSION_PATCH 0 + +#define _HIPCUB_REQUIRED_LIBCXX_VERSION \ + _HIPCUB_REQUIRED_LIBCXX_VERSION_MAJOR * 1000000 + _HIPCUB_REQUIRED_LIBCXX_VERSION_MINOR * 1000 \ + + _HIPCUB_REQUIRED_LIBCXX_VERSION_PATCH + +#ifdef __has_include + #define HIPCUB_HAS_INCLUDE(_X) __has_include(_X) +#else + #define HIPCUB_HAS_INCLUDE(_X) 0 +#endif + +#define _HIPCUB_STRINGIFY_IMPL(x) #x +#define _HIPCUB_STRINGIFY(x) _HIPCUB_STRINGIFY_IMPL(x) + +// clang-format off + +// If the '::cuda::std' namespace from 'libcudacxx' or 'libhipcxx' is available. +#if HIPCUB_HAS_INCLUDE() + #include + // If version matches and '_CUDA_VSTD' is available. + #if defined(_LIBCUDACXX_CUDA_API_VERSION) && (_LIBCUDACXX_CUDA_API_VERSION >= _HIPCUB_REQUIRED_LIBCXX_VERSION) && defined(_CUDA_VSTD) + #define _HIPCUB_LIBCXX_INCLUDE(LIB) _HIPCUB_STRINGIFY(cuda/LIB) + #define _HIPCUB_STD_INCLUDE(LIB) _HIPCUB_STRINGIFY(cuda/std/LIB) + #define _HIPCUB_LIBCXX ::cuda + #define _HIPCUB_STD _CUDA_VSTD + #define _HIPCUB_HAS_DEVICE_SYSTEM_STD 1 + #define _HIPCUB_STD_NAMESPACE_BEGIN _LIBCUDACXX_BEGIN_NAMESPACE_STD + #define _HIPCUB_STD_NAMESPACE_END _LIBCUDACXX_END_NAMESPACE_STD + #endif +#endif +// Otherwise, if the '::hip::std' namespace from 'libhipcxx' is available. +#if !defined(_HIPCUB_HAS_DEVICE_SYSTEM_STD) && HIPCUB_HAS_INCLUDE() + #include + // If version matches and '_CUDA_VSTD' is available. + #if defined(_LIBCUDACXX_CUDA_API_VERSION) && (_LIBCUDACXX_CUDA_API_VERSION >= _HIPCUB_REQUIRED_LIBCXX_VERSION) && defined(_CUDA_VSTD) + #define _HIPCUB_LIBCXX_INCLUDE(LIB) _HIPCUB_STRINGIFY(hip/LIB) + #define _HIPCUB_STD_INCLUDE(LIB) _HIPCUB_STRINGIFY(hip/std/LIB) + // In 'libhipcxx' the '::hip' namespace is synonymous with '::cuda'. + #define _HIPCUB_LIBCXX ::hip + // In 'libhipcxx' the macro '_CUDA_VSTD' is also defined. + #define _HIPCUB_STD _CUDA_VSTD + #define _HIPCUB_HAS_DEVICE_SYSTEM_STD 1 + #define _HIPCUB_STD_NAMESPACE_BEGIN _LIBCUDACXX_BEGIN_NAMESPACE_STD + #define _HIPCUB_STD_NAMESPACE_END _LIBCUDACXX_END_NAMESPACE_STD + #endif +#endif + +// If 'libcudacxx' or 'libhipcxx' is not found, use fallback. +#ifndef _HIPCUB_HAS_DEVICE_SYSTEM_STD + #define _HIPCUB_LIBCXX_INCLUDE(LIB) _HIPCUB_STRINGIFY(LIB) + #define _HIPCUB_STD_INCLUDE(LIB) _HIPCUB_STRINGIFY(LIB) + #define _HIPCUB_LIBCXX + #define _HIPCUB_STD ::std + #define _HIPCUB_HAS_DEVICE_SYSTEM_STD 0 + #define _HIPCUB_STD_NAMESPACE_BEGIN \ + namespace std \ + { + #define _HIPCUB_STD_NAMESPACE_END } +#endif + +// clang-format on + +#endif // HIPCUB_LIBCXX_HPP_ diff --git a/projects/hipcub/hipcub/include/hipcub/thread/thread_store.hpp b/projects/hipcub/hipcub/include/hipcub/thread/thread_store.hpp index c982be81ec7c..6dbd6ed44346 100644 --- a/projects/hipcub/hipcub/include/hipcub/thread/thread_store.hpp +++ b/projects/hipcub/hipcub/include/hipcub/thread/thread_store.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2010-2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2021-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2021-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -38,8 +38,8 @@ #include "../backend/rocprim/thread/thread_store.hpp" // IWYU pragma: export #elif defined(__HIP_PLATFORM_NVIDIA__) - #include "../config.hpp" - #include // IWYU pragma: export + #include "../backend/cub/thread/thread_store.hpp" // IWYU pragma: export + #endif #endif diff --git a/projects/hipcub/hipcub/include/hipcub/util_type.hpp b/projects/hipcub/hipcub/include/hipcub/util_type.hpp index 9f755ce45d62..6dd78870928c 100644 --- a/projects/hipcub/hipcub/include/hipcub/util_type.hpp +++ b/projects/hipcub/hipcub/include/hipcub/util_type.hpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2020-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2020-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -33,7 +33,7 @@ #ifdef __HIP_PLATFORM_AMD__ #include "backend/rocprim/util_type.hpp" // IWYU pragma: export #elif defined(__HIP_PLATFORM_NVIDIA__) - #include // IWYU pragma: export + #include "backend/cub/util_type.hpp" // IWYU pragma: export #endif diff --git a/projects/hipcub/rtest.xml b/projects/hipcub/rtest.xml index af8b5917109a..7156f0dcbd34 100644 --- a/projects/hipcub/rtest.xml +++ b/projects/hipcub/rtest.xml @@ -2,10 +2,10 @@ - + - + {CTEST_FILTER} {CTEST_REGEX} diff --git a/projects/hipcub/test/CMakeLists.txt b/projects/hipcub/test/CMakeLists.txt index 11771eae643a..70c28b2b197c 100644 --- a/projects/hipcub/test/CMakeLists.txt +++ b/projects/hipcub/test/CMakeLists.txt @@ -71,7 +71,16 @@ endfunction() # We'll use this small program to detect available GPUs and build the resource JSON file. set(GEN_RES_SPEC_PATH ${CMAKE_SOURCE_DIR}/test/generate_resource_spec.cpp) add_executable(generate_resource_spec ${GEN_RES_SPEC_PATH}) -set_target_properties(generate_resource_spec PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}") +set_target_properties(generate_resource_spec + PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}" +) + +if(HIP_COMPILER STREQUAL "nvcc") + set_source_files_properties(${GEN_RES_SPEC_PATH} + PROPERTIES LANGUAGE CUDA) +endif() + +target_compile_options(generate_resource_spec PRIVATE -Wno-unused-command-line-argument) target_link_libraries(generate_resource_spec PRIVATE hip::host) # This test may still get the offload-compress flag passed. Since it does not include any kernel # code it will give an unused-command-line-argument warning. diff --git a/projects/hipcub/test/extra/CMakeLists.txt b/projects/hipcub/test/extra/CMakeLists.txt index 8ab3eb577551..801ba1a5c0bd 100644 --- a/projects/hipcub/test/extra/CMakeLists.txt +++ b/projects/hipcub/test/extra/CMakeLists.txt @@ -1,6 +1,6 @@ # MIT License # -# Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -42,7 +42,7 @@ include(VerifyCompiler) # CUB (only for CUDA platform) if(HIP_COMPILER STREQUAL "nvcc") - set(CCCL_MINIMUM_VERSION 2.8.2) + set(CCCL_MINIMUM_VERSION 3.0.0) if(NOT DOWNLOAD_CUB) find_package(CCCL ${CCCL_MINIMUM_VERSION} CONFIG) endif() diff --git a/projects/hipcub/test/generate_resource_spec.cpp b/projects/hipcub/test/generate_resource_spec.cpp index 3fae3cc1618f..77569124291e 100644 --- a/projects/hipcub/test/generate_resource_spec.cpp +++ b/projects/hipcub/test/generate_resource_spec.cpp @@ -10,7 +10,7 @@ // ./enum_device // // Sample output: -// { +// { // "version": { // "major": 1, // "minor": 0 @@ -113,7 +113,7 @@ int main(int argc, char* argv[]) // Add one object for each gfxID. // Each gfxID-keyed object will contain an array of device IDs. - unsigned int key_index = 0; + size_t key_index = 0; for(auto& name_it : names_to_ids) { out_file << " \"" << name_it.first << "\": [" << std::endl; @@ -124,7 +124,7 @@ int main(int argc, char* argv[]) // to have a consistent output on each run so that the resource // spec file stays the same. std::sort(name_it.second.begin(), name_it.second.end()); - unsigned int id_index = 0; + size_t id_index = 0; for(const auto& id_it : name_it.second) { out_file << " {" << std::endl; diff --git a/projects/hipcub/test/hipcub/CMakeLists.txt b/projects/hipcub/test/hipcub/CMakeLists.txt index f62d3d85d224..d69d1d855c5d 100644 --- a/projects/hipcub/test/hipcub/CMakeLists.txt +++ b/projects/hipcub/test/hipcub/CMakeLists.txt @@ -1,5 +1,5 @@ # MIT License # -# Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -257,12 +257,10 @@ add_hipcub_test_parallel("hipcub.DeviceSegmentedRadixSort" test_hipcub_device_se add_hipcub_test("hipcub.DeviceSegmentedReduce" test_hipcub_device_segmented_reduce.cpp) add_hipcub_test_parallel("hipcub.DeviceSegmentedSort" test_hipcub_device_segmented_sort.cpp.in) add_hipcub_test("hipcub.DeviceSelect" test_hipcub_device_select.cpp) -add_hipcub_test("hipcub.DeviceSpmv" test_hipcub_device_spmv.cpp) add_hipcub_test("hipcub.DeviceTransform" test_hipcub_device_transform.cpp) add_hipcub_test("hipcub.DevicePartition" test_hipcub_device_partition.cpp) add_hipcub_test("hipcub.Grid" test_hipcub_grid.cpp) add_hipcub_test("hipcub.UtilPtx" test_hipcub_util_ptx.cpp) -add_hipcub_test("hipcub.UtilDevice" test_hipcub_util_device.cpp) add_hipcub_test("hipcub.Vector" test_hipcub_vector.cpp) add_hipcub_test("hipcub.WarpExchange" test_hipcub_warp_exchange.cpp) add_hipcub_test("hipcub.WarpLoad" test_hipcub_warp_load.cpp) diff --git a/projects/hipcub/test/hipcub/bfloat16.hpp b/projects/hipcub/test/hipcub/bfloat16.hpp index cc63e155b534..8aed6fc7c228 100644 --- a/projects/hipcub/test/hipcub/bfloat16.hpp +++ b/projects/hipcub/test/hipcub/bfloat16.hpp @@ -1,5 +1,6 @@ /****************************************************************************** * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -39,7 +40,11 @@ #include #if defined(__HIP_PLATFORM_NVIDIA__) -#include + #include + #include + #include +#else + #include #endif #ifdef __GNUC__ @@ -48,6 +53,33 @@ #pragma GCC diagnostic ignored "-Wstrict-aliasing" #endif +struct bfloat16_t; + +#if defined(__HIP_PLATFORM_NVIDIA__) + #include + #include + +using hip_bfloat16 = bfloat16_t; +namespace cuda +{ +namespace std +{ + +template<> +struct is_floating_point : true_type +{}; + +template<> +class numeric_limits +{ +public: + static constexpr bool is_specialized = true; +}; + +} // namespace std +} // namespace cuda + +#endif // __HIP_PLATFORM_NVIDIA__ /****************************************************************************** * bfloat16_t @@ -86,18 +118,20 @@ struct bfloat16_t *this = bfloat16_t(float(a)); } - /// Constructor from std::size_t - __host__ __device__ __forceinline__ bfloat16_t(std::size_t a) + /// Constructor from size_t + __host__ __device__ __forceinline__ + bfloat16_t(size_t a) { *this = bfloat16_t(float(a)); } /// Constructor from unsigned long long int template::value - && (!std::is_same::value)>::type> - __host__ __device__ __forceinline__ bfloat16_t(T a) + typename + = typename std::enable_if + && (!std::is_same_v)>::type> + __host__ __device__ __forceinline__ + bfloat16_t(T a) { *this = bfloat16_t(float(a)); } @@ -263,40 +297,22 @@ inline std::ostream& operator<<(std::ostream &out, const bfloat16_t &x) #if defined(__HIP_PLATFORM_NVIDIA__) - /// Insert formatted \p __nv_bfloat16 into the output stream - inline std::ostream& operator<<(std::ostream &out, const __nv_bfloat16 &x) - { - return out << bfloat16_t(x); - } +/// Insert formatted \p __nv_bfloat16 into the output stream +inline std::ostream& operator<<(std::ostream& out, const __nv_bfloat16& x) +{ + return out << bfloat16_t(x); +} #endif - - - /****************************************************************************** * Traits overloads ******************************************************************************/ -template <> -struct hipcub::FpLimits -{ - static __host__ __device__ __forceinline__ bfloat16_t Max() { return bfloat16_t::max(); } - - static __host__ __device__ __forceinline__ bfloat16_t Lowest() { return bfloat16_t::lowest(); } -}; - -#if defined(__HIP_PLATFORM_NVIDIA__) -_CCCL_SUPPRESS_DEPRECATED_PUSH -#else -HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH -#endif -template <> struct hipcub::NumericTraits : hipcub::BaseTraits {}; -#if defined(__HIP_PLATFORM_NVIDIA__) -_CCCL_SUPPRESS_DEPRECATED_POP -#else -HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP -#endif +template<> +struct hipcub::NumericTraits + : hipcub::BaseTraits +{}; #ifdef __GNUC__ #pragma GCC diagnostic pop diff --git a/projects/hipcub/test/hipcub/common_test_header.hpp b/projects/hipcub/test/hipcub/common_test_header.hpp index 737c29142cda..13321827b8fa 100755 --- a/projects/hipcub/test/hipcub/common_test_header.hpp +++ b/projects/hipcub/test/hipcub/common_test_header.hpp @@ -22,11 +22,9 @@ #include #include -#include #include #include #include -#include #include #include #include @@ -49,19 +47,29 @@ #include #endif +#include + +#include _HIPCUB_LIBCXX_INCLUDE(cmath) +#include _HIPCUB_STD_INCLUDE(limits) + // test_utils.hpp should only be included by this header. // The following definition is used as guard in test_utils.hpp // Including test_utils.hpp by itself will cause a compile error. -#define TEST_UTILS_INCLUDE_GAURD +#define TEST_UTILS_INCLUDE_GUARD #include "test_utils.hpp" -#if defined(__SANITIZE_ADDRESS__) || (defined(__has_feature) && __has_feature(address_sanitizer)) - #define GTEST_SKIP_ASAN() \ - do \ - { \ - GTEST_SKIP() << "Skipping test under ASan"; \ - } \ - while(0) +#if defined(__clang__) + #if defined(__SANITIZE_ADDRESS__) \ + || (defined(__has_feature) && __has_feature(address_sanitizer)) + #define GTEST_SKIP_ASAN() \ + do \ + { \ + GTEST_SKIP() << "Skipping test under ASan"; \ + } \ + while(0) + #else + #define GTEST_SKIP_ASAN() + #endif #else #define GTEST_SKIP_ASAN() #endif diff --git a/projects/hipcub/test/hipcub/experimental/sparse_matrix.hpp b/projects/hipcub/test/hipcub/experimental/sparse_matrix.hpp deleted file mode 100644 index 941c21cae52f..000000000000 --- a/projects/hipcub/test/hipcub/experimental/sparse_matrix.hpp +++ /dev/null @@ -1,1240 +0,0 @@ -/****************************************************************************** - * Copyright (c) 2011, Duane Merrill. All rights reserved. - * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2024-2025, Advanced Micro Devices, Inc. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the NVIDIA CORPORATION nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL NVIDIA CORPORATION BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - ******************************************************************************/ - -/****************************************************************************** - * Matrix data structures and parsing logic - ******************************************************************************/ - -#pragma once - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef CUB_MKL - #include - #include -#endif - -using namespace std; - -/****************************************************************************** - * COO matrix type - ******************************************************************************/ - -struct GraphStats -{ - int num_rows; - int num_cols; - int num_nonzeros; - - double diag_dist_mean; // mean - double diag_dist_std_dev; // sample std dev - double pearson_r; // coefficient of variation - - double row_length_mean; // mean - double row_length_std_dev; // sample std_dev - double row_length_variation; // coefficient of variation - double row_length_skewness; // skewness - - void Display(bool show_labels = true) - { - if (show_labels) - printf("\n" - "\t num_rows: %d\n" - "\t num_cols: %d\n" - "\t num_nonzeros: %d\n" - "\t diag_dist_mean: %.2f\n" - "\t diag_dist_std_dev: %.2f\n" - "\t pearson_r: %f\n" - "\t row_length_mean: %.5f\n" - "\t row_length_std_dev: %.5f\n" - "\t row_length_variation: %.5f\n" - "\t row_length_skewness: %.5f\n", - num_rows, - num_cols, - num_nonzeros, - diag_dist_mean, - diag_dist_std_dev, - pearson_r, - row_length_mean, - row_length_std_dev, - row_length_variation, - row_length_skewness); - else - printf( - "%d, " - "%d, " - "%d, " - "%.2f, " - "%.2f, " - "%f, " - "%.5f, " - "%.5f, " - "%.5f, " - "%.5f, ", - num_rows, - num_cols, - num_nonzeros, - diag_dist_mean, - diag_dist_std_dev, - pearson_r, - row_length_mean, - row_length_std_dev, - row_length_variation, - row_length_skewness); - } -}; - - - -/****************************************************************************** - * COO matrix type - ******************************************************************************/ - - -/** - * COO matrix type. A COO matrix is just a vector of edge tuples. Tuples are sorted - * first by row, then by column. - */ -template -struct CooMatrix -{ - //--------------------------------------------------------------------- - // Type definitions and constants - //--------------------------------------------------------------------- - - // COO edge tuple - struct CooTuple - { - OffsetT row; - OffsetT col; - ValueT val; - - CooTuple() : row(OffsetT()), col(OffsetT()), val(ValueT()) {} - CooTuple(OffsetT row, OffsetT col) : row(row), col(col) {} - CooTuple(OffsetT row, OffsetT col, ValueT val) : row(row), col(col), val(val) {} - - /** - * Comparator for sorting COO sparse format num_nonzeros - */ - bool operator<(const CooTuple &other) const - { - if ((row < other.row) || ((row == other.row) && (col < other.col))) - { - return true; - } - - return false; - } - }; - - - //--------------------------------------------------------------------- - // Data members - //--------------------------------------------------------------------- - - // Fields - int num_rows; - int num_cols; - int num_nonzeros; - CooTuple* coo_tuples; - - //--------------------------------------------------------------------- - // Methods - //--------------------------------------------------------------------- - - // Constructor - CooMatrix() : num_rows(0), num_cols(0), num_nonzeros(0), coo_tuples(nullptr) {} - - /** - * Clear - */ - void Clear() - { - if (coo_tuples) delete[] coo_tuples; - coo_tuples = nullptr; - } - - - // Destructor - ~CooMatrix() - { - Clear(); - } - - - // Display matrix to stdout - void Display() - { - cout << "COO Matrix (" << num_rows << " rows, " << num_cols << " columns, " << num_nonzeros << " non-zeros):\n"; - cout << "Ordinal, Row, Column, Value\n"; - for (int i = 0; i < num_nonzeros; i++) - { - cout << '\t' << i << ',' << coo_tuples[i].row << ',' << coo_tuples[i].col << ',' << coo_tuples[i].val << "\n"; - } - } - - - /** - * Builds a symmetric COO sparse from an asymmetric CSR matrix. - */ - template - void InitCsrSymmetric(CsrMatrixT &csr_matrix) - { - if (coo_tuples) - { - fprintf(stderr, "Matrix already constructed\n"); - exit(1); - } - - num_rows = csr_matrix.num_cols; - num_cols = csr_matrix.num_rows; - num_nonzeros = csr_matrix.num_nonzeros * 2; - coo_tuples = new CooTuple[num_nonzeros]; - - for (OffsetT row = 0; row < csr_matrix.num_rows; ++row) - { - for (OffsetT nonzero = csr_matrix.row_offsets[row]; nonzero < csr_matrix.row_offsets[row + 1]; ++nonzero) - { - coo_tuples[nonzero].row = row; - coo_tuples[nonzero].col = csr_matrix.column_indices[nonzero]; - coo_tuples[nonzero].val = csr_matrix.values[nonzero]; - - coo_tuples[csr_matrix.num_nonzeros + nonzero].row = coo_tuples[nonzero].col; - coo_tuples[csr_matrix.num_nonzeros + nonzero].col = coo_tuples[nonzero].row; - coo_tuples[csr_matrix.num_nonzeros + nonzero].val = csr_matrix.values[nonzero]; - - } - } - - // Sort by rows, then columns - std::stable_sort(coo_tuples, coo_tuples + num_nonzeros); - } - - /** - * Builds a COO sparse from a relabeled CSR matrix. - */ - template - void InitCsrRelabel(CsrMatrixT &csr_matrix, const OffsetT* relabel_indices) - { - if (coo_tuples) - { - fprintf(stderr, "Matrix already constructed\n"); - exit(1); - } - - num_rows = csr_matrix.num_rows; - num_cols = csr_matrix.num_cols; - num_nonzeros = csr_matrix.num_nonzeros; - coo_tuples = new CooTuple[num_nonzeros]; - - for (OffsetT row = 0; row < num_rows; ++row) - { - for (OffsetT nonzero = csr_matrix.row_offsets[row]; nonzero < csr_matrix.row_offsets[row + 1]; ++nonzero) - { - coo_tuples[nonzero].row = relabel_indices[row]; - coo_tuples[nonzero].col = relabel_indices[csr_matrix.column_indices[nonzero]]; - coo_tuples[nonzero].val = csr_matrix.values[nonzero]; - } - } - - // Sort by rows, then columns - std::stable_sort(coo_tuples, coo_tuples + num_nonzeros); - } - - - - /** - * Builds a METIS COO sparse from the given file. - */ - void InitMetis(const string& /*metis_filename*/) const - { - if (coo_tuples) - { - fprintf(stderr, "Matrix already constructed\n"); - exit(1); - } - - // TODO - } - - - /** - * Builds a MARKET COO sparse from the given file. - */ - void InitMarket( - const string& market_filename, - ValueT default_value = 1.0, - bool verbose = false) - { - if (verbose) { - printf("Reading... "); fflush(stdout); - } - - if (coo_tuples) - { - fprintf(stderr, "Matrix already constructed\n"); - exit(1); - } - - std::ifstream ifs; - ifs.open(market_filename.c_str(), std::ifstream::in); - if (!ifs.good()) - { - fprintf(stderr, "Error opening file\n"); - exit(1); - } - - bool array = false; - bool symmetric = false; - bool skew = false; - int current_edge = -1; - char line[1024]; - - if (verbose) { - printf("Parsing... "); fflush(stdout); - } - - while (true) - { - ifs.getline(line, 1024); - if (!ifs.good()) - { - // Done - break; - } - - if (line[0] == '%') - { - // Comment - if (line[1] == '%') - { - // Banner - symmetric = (strstr(line, "symmetric") != nullptr); - skew = (strstr(line, "skew") != nullptr); - array = (strstr(line, "array") != nullptr); - - if (verbose) { - printf("(symmetric: %d, skew: %d, array: %d) ", symmetric, skew, array); fflush(stdout); - } - } - } - else if (current_edge == -1) - { - // Problem description - int nparsed = sscanf(line, "%d %d %d", &num_rows, &num_cols, &num_nonzeros); - if ((!array) && (nparsed == 3)) - { - if (symmetric) - num_nonzeros *= 2; - - // Allocate coo matrix - coo_tuples = new CooTuple[num_nonzeros]; - current_edge = 0; - - } - else if (array && (nparsed == 2)) - { - // Allocate coo matrix - num_nonzeros = num_rows * num_cols; - coo_tuples = new CooTuple[num_nonzeros]; - current_edge = 0; - } - else - { - fprintf(stderr, "Error parsing MARKET matrix: invalid problem description: %s\n", line); - exit(1); - } - - } - else - { - // Edge - if (current_edge >= num_nonzeros) - { - fprintf(stderr, "Error parsing MARKET matrix: encountered more than %d num_nonzeros\n", num_nonzeros); - exit(1); - } - - int row, col; - double val; - - if (array) - { - if (sscanf(line, "%lf", &val) != 1) - { - fprintf(stderr, "Error parsing MARKET matrix: badly formed current_edge: '%s' at edge %d\n", line, current_edge); - exit(1); - } - col = (current_edge / num_rows); - row = (current_edge - (num_rows * col)); - - coo_tuples[current_edge] = CooTuple(row, col, val); // Convert indices to zero-based - } - else - { - // Parse nonzero (note: using strtol and strtod is 2x faster than sscanf or istream parsing) - char *l = line; - char* t = nullptr; - - // parse row - row = strtol(l, &t, 0); - if (t == l) - { - fprintf(stderr, "Error parsing MARKET matrix: badly formed row at edge %d\n", current_edge); - exit(1); - } - l = t; - - // parse col - col = strtol(l, &t, 0); - if (t == l) - { - fprintf(stderr, "Error parsing MARKET matrix: badly formed col at edge %d\n", current_edge); - exit(1); - } - l = t; - - // parse val - val = strtod(l, &t); - if (t == l) - { - val = default_value; - } -/* - int nparsed = sscanf(line, "%d %d %lf", &row, &col, &val); - if (nparsed == 2) - { - // No value specified - val = default_value; - - } - else if (nparsed != 3) - { - fprintf(stderr, "Error parsing MARKET matrix 1: badly formed current_edge: %d parsed at edge %d\n", nparsed, current_edge); - exit(1); - } -*/ - - coo_tuples[current_edge] = CooTuple(row - 1, col - 1, val); // Convert indices to zero-based - - } - - current_edge++; - - if (symmetric && (row != col)) - { - coo_tuples[current_edge].row = coo_tuples[current_edge - 1].col; - coo_tuples[current_edge].col = coo_tuples[current_edge - 1].row; - coo_tuples[current_edge].val = coo_tuples[current_edge - 1].val * (skew ? -1 : 1); - current_edge++; - } - } - } - - // Adjust nonzero count (nonzeros along the diagonal aren't reversed) - num_nonzeros = current_edge; - - if (verbose) { - printf("done. Ordering..."); fflush(stdout); - } - - // Sort by rows, then columns - std::stable_sort(coo_tuples, coo_tuples + num_nonzeros); - - if (verbose) { - printf("done. "); fflush(stdout); - } - - ifs.close(); - } - - - /** - * Builds a dense matrix - */ - int InitDense(OffsetT num_rows, - OffsetT num_cols, - ValueT default_value = 1.0, - bool /*verbose*/ = false) - { - if (coo_tuples) - { - fprintf(stderr, "Matrix already constructed\n"); - exit(1); - } - - this->num_rows = num_rows; - this->num_cols = num_cols; - - num_nonzeros = num_rows * num_cols; - coo_tuples = new CooTuple[num_nonzeros]; - - for (OffsetT row = 0; row < num_rows; ++row) - { - for (OffsetT col = 0; col < num_cols; ++col) - { - coo_tuples[(row * num_cols) + col] = CooTuple(row, col, default_value); - } - } - - // Sort by rows, then columns - std::stable_sort(coo_tuples, coo_tuples + num_nonzeros); - - return 0; - } - - /** - * Builds a wheel COO sparse matrix having spokes spokes. - */ - int InitWheel(OffsetT spokes, ValueT default_value = 1.0, bool /*verbose*/ = false) - { - if (coo_tuples) - { - fprintf(stderr, "Matrix already constructed\n"); - exit(1); - } - - num_rows = spokes + 1; - num_cols = num_rows; - num_nonzeros = spokes * 2; - coo_tuples = new CooTuple[num_nonzeros]; - - // Add spoke num_nonzeros - int current_edge = 0; - for (OffsetT i = 0; i < spokes; i++) - { - coo_tuples[current_edge] = CooTuple(0, i + 1, default_value); - current_edge++; - } - - // Add rim - for (OffsetT i = 0; i < spokes; i++) - { - OffsetT dest = (i + 1) % spokes; - coo_tuples[current_edge] = CooTuple(i + 1, dest + 1, default_value); - current_edge++; - } - - // Sort by rows, then columns - std::stable_sort(coo_tuples, coo_tuples + num_nonzeros); - - return 0; - } - - - /** - * Builds a square 2D grid CSR matrix. Interior num_vertices have degree 5 when including - * a self-loop. - * - * Returns 0 on success, 1 on failure. - */ - int InitGrid2d(OffsetT width, bool self_loop, ValueT default_value = 1.0) - { - if (coo_tuples) - { - fprintf(stderr, "Matrix already constructed\n"); - exit(1); - } - - int interior_nodes = (width - 2) * (width - 2); - int edge_nodes = (width - 2) * 4; - int corner_nodes = 4; - num_rows = width * width; - num_cols = num_rows; - num_nonzeros = (interior_nodes * 4) + (edge_nodes * 3) + (corner_nodes * 2); - - if (self_loop) - num_nonzeros += num_rows; - - coo_tuples = new CooTuple[num_nonzeros]; - int current_edge = 0; - - for (OffsetT j = 0; j < width; j++) - { - for (OffsetT k = 0; k < width; k++) - { - OffsetT me = (j * width) + k; - - // West - OffsetT neighbor = (j * width) + (k - 1); - if (k - 1 >= 0) { - coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); - current_edge++; - } - - // East - neighbor = (j * width) + (k + 1); - if (k + 1 < width) { - coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); - current_edge++; - } - - // North - neighbor = ((j - 1) * width) + k; - if (j - 1 >= 0) { - coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); - current_edge++; - } - - // South - neighbor = ((j + 1) * width) + k; - if (j + 1 < width) { - coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); - current_edge++; - } - - if (self_loop) - { - neighbor = me; - coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); - current_edge++; - } - } - } - - // Sort by rows, then columns, update dims - std::stable_sort(coo_tuples, coo_tuples + num_nonzeros); - - return 0; - } - - - /** - * Builds a square 3D grid COO sparse matrix. Interior num_vertices have degree 7 when including - * a self-loop. Values are uninitialized, coo_tuples are sorted. - */ - int InitGrid3d(OffsetT width, bool self_loop, ValueT default_value = 1.0) - { - if (coo_tuples) - { - fprintf(stderr, "Matrix already constructed\n"); - return -1; - } - - OffsetT interior_nodes = (width - 2) * (width - 2) * (width - 2); - OffsetT face_nodes = (width - 2) * (width - 2) * 6; - OffsetT edge_nodes = (width - 2) * 12; - OffsetT corner_nodes = 8; - num_cols = width * width * width; - num_rows = num_cols; - num_nonzeros = (interior_nodes * 6) + (face_nodes * 5) + (edge_nodes * 4) + (corner_nodes * 3); - - if (self_loop) - num_nonzeros += num_rows; - - coo_tuples = new CooTuple[num_nonzeros]; - int current_edge = 0; - - for (OffsetT i = 0; i < width; i++) - { - for (OffsetT j = 0; j < width; j++) - { - for (OffsetT k = 0; k < width; k++) - { - - OffsetT me = (i * width * width) + (j * width) + k; - - // Up - OffsetT neighbor = (i * width * width) + (j * width) + (k - 1); - if (k - 1 >= 0) { - coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); - current_edge++; - } - - // Down - neighbor = (i * width * width) + (j * width) + (k + 1); - if (k + 1 < width) { - coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); - current_edge++; - } - - // West - neighbor = (i * width * width) + ((j - 1) * width) + k; - if (j - 1 >= 0) { - coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); - current_edge++; - } - - // East - neighbor = (i * width * width) + ((j + 1) * width) + k; - if (j + 1 < width) { - coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); - current_edge++; - } - - // North - neighbor = ((i - 1) * width * width) + (j * width) + k; - if (i - 1 >= 0) { - coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); - current_edge++; - } - - // South - neighbor = ((i + 1) * width * width) + (j * width) + k; - if (i + 1 < width) { - coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); - current_edge++; - } - - if (self_loop) - { - neighbor = me; - coo_tuples[current_edge] = CooTuple(me, neighbor, default_value); - current_edge++; - } - } - } - } - - // Sort by rows, then columns, update dims - std::stable_sort(coo_tuples, coo_tuples + num_nonzeros); - - return 0; - } -}; - - - -/****************************************************************************** - * COO matrix type - ******************************************************************************/ - - -/** - * CSR sparse format matrix - */ -template< - typename ValueT, - typename OffsetT> -struct CsrMatrix -{ - int num_rows; - int num_cols; - int num_nonzeros; - OffsetT* row_offsets; - OffsetT* column_indices; - ValueT* values; - bool numa_malloc; - - /** - * Constructor - */ - CsrMatrix() - : num_rows(0) - , num_cols(0) - , num_nonzeros(0) - , row_offsets(nullptr) - , column_indices(nullptr) - , values(nullptr) - { -#ifdef CUB_MKL - numa_malloc = ((numa_available() >= 0) && (numa_num_task_nodes() > 1)); -#else - numa_malloc = false; -#endif - } - - - /** - * Clear - */ - void Clear() - { -#ifdef CUB_MKL - if (numa_malloc) - { - numa_free(row_offsets, sizeof(OffsetT) * (num_rows + 1)); - numa_free(values, sizeof(ValueT) * num_nonzeros); - numa_free(column_indices, sizeof(OffsetT) * num_nonzeros); - } - else - { - if (row_offsets) mkl_free(row_offsets); - if (column_indices) mkl_free(column_indices); - if (values) mkl_free(values); - } - -#else - if (row_offsets) delete[] row_offsets; - if (column_indices) delete[] column_indices; - if (values) delete[] values; -#endif - - row_offsets = nullptr; - column_indices = nullptr; - values = nullptr; - } - - /** - * Destructor - */ - ~CsrMatrix() - { - Clear(); - } - - GraphStats Stats() const - { - GraphStats stats; - stats.num_rows = num_rows; - stats.num_cols = num_cols; - stats.num_nonzeros = num_nonzeros; - - // - // Compute diag-distance statistics - // - - OffsetT samples = 0; - double mean = 0.0; - double ss_tot = 0.0; - - for (OffsetT row = 0; row < num_rows; ++row) - { - OffsetT nz_idx_start = row_offsets[row]; - OffsetT nz_idx_end = row_offsets[row + 1]; - - for (int nz_idx = nz_idx_start; nz_idx < nz_idx_end; ++nz_idx) - { - OffsetT col = column_indices[nz_idx]; - double x = (col > row) ? col - row : row - col; - - samples++; - double delta = x - mean; - mean = mean + (delta / samples); - ss_tot += delta * (x - mean); - } - } - stats.diag_dist_mean = mean; - double variance = ss_tot / samples; - stats.diag_dist_std_dev = sqrt(variance); - - - // - // Compute deming statistics - // - - samples = 0; - double mean_x = 0.0; - double mean_y = 0.0; - double ss_x = 0.0; - double ss_y = 0.0; - - for (OffsetT row = 0; row < num_rows; ++row) - { - OffsetT nz_idx_start = row_offsets[row]; - OffsetT nz_idx_end = row_offsets[row + 1]; - - for (int nz_idx = nz_idx_start; nz_idx < nz_idx_end; ++nz_idx) - { - OffsetT col = column_indices[nz_idx]; - - samples++; - double x = col; - double y = row; - double delta; - - delta = x - mean_x; - mean_x = mean_x + (delta / samples); - ss_x += delta * (x - mean_x); - - delta = y - mean_y; - mean_y = mean_y + (delta / samples); - ss_y += delta * (y - mean_y); - } - } - - samples = 0; - double s_xy = 0.0; - double s_xxy = 0.0; - double s_xyy = 0.0; - for (OffsetT row = 0; row < num_rows; ++row) - { - OffsetT nz_idx_start = row_offsets[row]; - OffsetT nz_idx_end = row_offsets[row + 1]; - - for (int nz_idx = nz_idx_start; nz_idx < nz_idx_end; ++nz_idx) - { - OffsetT col = column_indices[nz_idx]; - - samples++; - double x = col; - double y = row; - - double xy = (x - mean_x) * (y - mean_y); - double xxy = (x - mean_x) * (x - mean_x) * (y - mean_y); - double xyy = (x - mean_x) * (y - mean_y) * (y - mean_y); - double delta; - - delta = xy - s_xy; - s_xy = s_xy + (delta / samples); - - delta = xxy - s_xxy; - s_xxy = s_xxy + (delta / samples); - - delta = xyy - s_xyy; - s_xyy = s_xyy + (delta / samples); - } - } - - // double s_xx = ss_x / num_nonzeros; - // double s_yy = ss_y / num_nonzeros; - - stats.pearson_r = (num_nonzeros * s_xy) / (sqrt(ss_x) * sqrt(ss_y)); - - - // - // Compute row-length statistics - // - - // Sample mean - stats.row_length_mean = double(num_nonzeros) / num_rows; - variance = 0.0; - stats.row_length_skewness = 0.0; - for (OffsetT row = 0; row < num_rows; ++row) - { - OffsetT length = row_offsets[row + 1] - row_offsets[row]; - double delta = double(length) - stats.row_length_mean; - variance += (delta * delta); - stats.row_length_skewness += (delta * delta * delta); - } - variance /= num_rows; - stats.row_length_std_dev = sqrt(variance); - stats.row_length_skewness = (stats.row_length_skewness / num_rows) / pow(stats.row_length_std_dev, 3.0); - stats.row_length_variation = stats.row_length_std_dev / stats.row_length_mean; - - return stats; - } - - /** - * Build CSR matrix from sorted COO matrix - */ - void FromCoo(const CooMatrix &coo_matrix) - { - num_rows = coo_matrix.num_rows; - num_cols = coo_matrix.num_cols; - num_nonzeros = coo_matrix.num_nonzeros; - -#ifdef CUB_MKL - - if (numa_malloc) - { - numa_set_strict(1); -// numa_set_bind_policy(1); - -// values = (ValueT*) numa_alloc_interleaved(sizeof(ValueT) * num_nonzeros); -// row_offsets = (OffsetT*) numa_alloc_interleaved(sizeof(OffsetT) * (num_rows + 1)); -// column_indices = (OffsetT*) numa_alloc_interleaved(sizeof(OffsetT) * num_nonzeros); - - row_offsets = (OffsetT*) numa_alloc_onnode(sizeof(OffsetT) * (num_rows + 1), 0); - column_indices = (OffsetT*) numa_alloc_onnode(sizeof(OffsetT) * num_nonzeros, 0); - values = (ValueT*) numa_alloc_onnode(sizeof(ValueT) * num_nonzeros, 1); - } - else - { - values = (ValueT*) mkl_malloc(sizeof(ValueT) * num_nonzeros, 4096); - row_offsets = (OffsetT*) mkl_malloc(sizeof(OffsetT) * (num_rows + 1), 4096); - column_indices = (OffsetT*) mkl_malloc(sizeof(OffsetT) * num_nonzeros, 4096); - - } - -#else - row_offsets = new OffsetT[num_rows + 1]; - column_indices = new OffsetT[num_nonzeros]; - values = new ValueT[num_nonzeros]; -#endif - - OffsetT prev_row = -1; - for (OffsetT current_edge = 0; current_edge < num_nonzeros; current_edge++) - { - OffsetT current_row = coo_matrix.coo_tuples[current_edge].row; - - // Fill in rows up to and including the current row - for (OffsetT row = prev_row + 1; row <= current_row; row++) - { - row_offsets[row] = current_edge; - } - prev_row = current_row; - - column_indices[current_edge] = coo_matrix.coo_tuples[current_edge].col; - values[current_edge] = coo_matrix.coo_tuples[current_edge].val; - } - - // Fill out any trailing edgeless vertices (and the end-of-list element) - for (OffsetT row = prev_row + 1; row <= num_rows; row++) - { - row_offsets[row] = num_nonzeros; - } - } - - - /** - * Display log-histogram to stdout - */ - void DisplayHistogram() - { - // Initialize - int log_counts[9]; - for (int i = 0; i < 9; i++) - { - log_counts[i] = 0; - } - - // Scan - int max_log_length = -1; - for (OffsetT row = 0; row < num_rows; row++) - { - OffsetT length = row_offsets[row + 1] - row_offsets[row]; - - int log_length = -1; - while (length > 0) - { - length /= 10; - log_length++; - } - if (log_length > max_log_length) - { - max_log_length = log_length; - } - - log_counts[log_length + 1]++; - } - printf("CSR matrix (%d rows, %d columns, %d non-zeros):\n", (int) num_rows, (int) num_cols, (int) num_nonzeros); - for (int i = -1; i < max_log_length + 1; i++) - { - printf("\tDegree 1e%d: \t%d (%.2f%%)\n", i, log_counts[i + 1], (float) log_counts[i + 1] * 100.0 / num_cols); - } - fflush(stdout); - } - - - /** - * Display matrix to stdout - */ - void Display() - { - printf("Input Matrix:\n"); - for (OffsetT row = 0; row < num_rows; row++) - { - printf("%d [@%d, #%d]: ", row, row_offsets[row], row_offsets[row + 1] - row_offsets[row]); - for (OffsetT current_edge = row_offsets[row]; current_edge < row_offsets[row + 1]; current_edge++) - { - printf("%d (%f), ", column_indices[current_edge], values[current_edge]); - } - printf("\n"); - } - fflush(stdout); - } - - -}; - - - -/****************************************************************************** - * Matrix transformations - ******************************************************************************/ - -// Comparator for ordering rows by degree (lowest first), then by row-id (lowest first) -template -struct OrderByLow -{ - OffsetT* row_degrees; - OrderByLow(OffsetT* row_degrees) : row_degrees(row_degrees) {} - - bool operator()(const OffsetT &a, const OffsetT &b) const - { - if (row_degrees[a] < row_degrees[b]) - return true; - else if (row_degrees[a] > row_degrees[b]) - return false; - else - return (a < b); - } -}; - -// Comparator for ordering rows by degree (highest first), then by row-id (lowest first) -template -struct OrderByHigh -{ - OffsetT* row_degrees; - OrderByHigh(OffsetT* row_degrees) : row_degrees(row_degrees) {} - - bool operator()(const OffsetT &a, const OffsetT &b) const - { - if (row_degrees[a] > row_degrees[b]) - return true; - else if (row_degrees[a] < row_degrees[b]) - return false; - else - return (a < b); - } -}; - - - -/** - * Reverse Cuthill-McKee - */ -template -void RcmRelabel( - CsrMatrix& matrix, - OffsetT* relabel_indices) -{ - // Initialize row degrees - OffsetT* row_degrees_in = new OffsetT[matrix.num_rows]; - OffsetT* row_degrees_out = new OffsetT[matrix.num_rows]; - for (OffsetT row = 0; row < matrix.num_rows; ++row) - { - row_degrees_in[row] = 0; - row_degrees_out[row] = matrix.row_offsets[row + 1] - matrix.row_offsets[row]; - } - for (OffsetT nonzero = 0; nonzero < matrix.num_nonzeros; ++nonzero) - { - row_degrees_in[matrix.column_indices[nonzero]]++; - } - - // Initialize unlabeled set - using UnlabeledSet = std::set>; - typename UnlabeledSet::key_compare unlabeled_comp(row_degrees_in); - UnlabeledSet unlabeled(unlabeled_comp); - for (OffsetT row = 0; row < matrix.num_rows; ++row) - { - relabel_indices[row] = -1; - unlabeled.insert(row); - } - - // Initialize queue set - std::deque q; - - // Process unlabeled vertices (traverse connected components) - OffsetT relabel_idx = 0; - while (!unlabeled.empty()) - { - // Seed the unvisited frontier queue with the unlabeled vertex of lowest-degree - OffsetT vertex = *unlabeled.begin(); - q.push_back(vertex); - - while (!q.empty()) - { - vertex = q.front(); - q.pop_front(); - - if (relabel_indices[vertex] == -1) - { - // Update this vertex - unlabeled.erase(vertex); - relabel_indices[vertex] = relabel_idx; - relabel_idx++; - - // Sort neighbors by degree - OrderByLow neighbor_comp(row_degrees_in); - std::sort( - matrix.column_indices + matrix.row_offsets[vertex], - matrix.column_indices + matrix.row_offsets[vertex + 1], - neighbor_comp); - - // Inspect neighbors, adding to the out frontier if unlabeled - for (OffsetT neighbor_idx = matrix.row_offsets[vertex]; - neighbor_idx < matrix.row_offsets[vertex + 1]; - ++neighbor_idx) - { - OffsetT neighbor = matrix.column_indices[neighbor_idx]; - q.push_back(neighbor); - } - } - } - } - -/* - // Reverse labels - for (int row = 0; row < matrix.num_rows; ++row) - { - relabel_indices[row] = matrix.num_rows - relabel_indices[row] - 1; - } -*/ - - // Cleanup - if (row_degrees_in) delete[] row_degrees_in; - if (row_degrees_out) delete[] row_degrees_out; -} - - -/** - * Reverse Cuthill-McKee - */ -template -void RcmRelabel( - CsrMatrix& matrix, - bool verbose = false) -{ - // Do not process if not square - if (matrix.num_cols != matrix.num_rows) - { - if (verbose) { - printf("RCM transformation ignored (not square)\n"); fflush(stdout); - } - return; - } - - // Initialize relabel indices - OffsetT* relabel_indices = new OffsetT[matrix.num_rows]; - - if (verbose) { - printf("RCM relabeling... "); fflush(stdout); - } - - RcmRelabel(matrix, relabel_indices); - - if (verbose) { - printf("done. Reconstituting... "); fflush(stdout); - } - - // Create a COO matrix from the relabel indices - CooMatrix coo_matrix; - coo_matrix.InitCsrRelabel(matrix, relabel_indices); - - // Reconstitute the CSR matrix from the sorted COO tuples - if (relabel_indices) delete[] relabel_indices; - matrix.Clear(); - matrix.FromCoo(coo_matrix); - - if (verbose) { - printf("done. "); fflush(stdout); - } -} diff --git a/projects/hipcub/test/hipcub/half.hpp b/projects/hipcub/test/hipcub/half.hpp index f1fa31eebcd0..82b78a6096f5 100644 --- a/projects/hipcub/test/hipcub/half.hpp +++ b/projects/hipcub/test/hipcub/half.hpp @@ -1,6 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2023, NVIDIA CORPORATION. All rights reserved. + * Modifications Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -38,7 +39,11 @@ #include #if defined(__HIP_PLATFORM_NVIDIA__) + #include + #include #include +#else + #include #endif #include @@ -50,6 +55,29 @@ #pragma GCC diagnostic ignored "-Wstrict-aliasing" #endif +struct half_t; + +#if defined(__HIP_PLATFORM_NVIDIA__) + +namespace cuda +{ +namespace std +{ +template<> +struct is_floating_point : true_type +{}; + +template<> +class numeric_limits +{ +public: + static constexpr bool is_specialized = true; +}; + +} // namespace std +} // namespace cuda + +#endif // __HIP_PLATFORM_NVIDIA__ /****************************************************************************** * half_t @@ -75,18 +103,20 @@ struct half_t *this = half_t(float(a)); } - /// Constructor from std::size_t - __host__ __device__ __forceinline__ half_t(std::size_t a) + /// Constructor from size_t + __host__ __device__ __forceinline__ + half_t(size_t a) { *this = half_t(float(a)); } /// Constructor from unsigned long long int template::value - && (!std::is_same::value)>::type> - __host__ __device__ __forceinline__ half_t(T a) + typename + = typename std::enable_if + && (!std::is_same_v)>::type> + __host__ __device__ __forceinline__ + half_t(T a) { *this = half_t(float(a)); } @@ -198,7 +228,7 @@ struct half_t f = (0xff << 23) | (sign << 31); // inf } } - static_assert(sizeof(float) == sizeof(std::uint32_t), "4-byte size check"); + static_assert(sizeof(float) == sizeof(uint32_t), "4-byte size check"); float ret{}; std::memcpy(&ret, &f, sizeof(float)); return ret; @@ -331,25 +361,10 @@ inline std::ostream& operator<<(std::ostream &out, const half_t &x) * Traits overloads ******************************************************************************/ -template <> -struct hipcub::FpLimits -{ - static __host__ __device__ __forceinline__ half_t Max() { return half_t::max(); } - - static __host__ __device__ __forceinline__ half_t Lowest() { return half_t::lowest(); } -}; - -#if defined(__HIP_PLATFORM_NVIDIA__) -_CCCL_SUPPRESS_DEPRECATED_PUSH -#else -HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH -#endif -template <> struct hipcub::NumericTraits : hipcub::BaseTraits {}; -#if defined(__HIP_PLATFORM_NVIDIA__) -_CCCL_SUPPRESS_DEPRECATED_POP -#else -HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP -#endif +template<> +struct hipcub::NumericTraits + : hipcub::BaseTraits +{}; #ifdef __GNUC__ #pragma GCC diagnostic pop diff --git a/projects/hipcub/test/hipcub/single_index_iterator.hpp b/projects/hipcub/test/hipcub/single_index_iterator.hpp index 8ba95e600d2f..b33c73c54956 100644 --- a/projects/hipcub/test/hipcub/single_index_iterator.hpp +++ b/projects/hipcub/test/hipcub/single_index_iterator.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -50,6 +50,13 @@ class single_index_iterator return *this; } + // Implicit conversion for read access + HIPCUB_HOST_DEVICE + inline operator T() const + { + return *value_; + } + private: T* const value_; const bool keep_; diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_adjacent_difference.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_adjacent_difference.cpp index 43ce5be89a46..65a43392fe7f 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_adjacent_difference.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_adjacent_difference.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -23,11 +23,10 @@ #include "common_test_header.hpp" // required rocprim headers -#include #include #include #include -#include +#include template< class T, @@ -72,25 +71,25 @@ struct custom_op2 }; using ParamsSubtract - = ::testing::Types, + = ::testing::Types, params_subtract, params_subtract, params_subtract, params_subtract, params_subtract, - params_subtract, + params_subtract, params_subtract, params_subtract, - params_subtract, + params_subtract, params_subtract, params_subtract, - params_subtract, + params_subtract, params_subtract, params_subtract, - params_subtract, + params_subtract, params_subtract, params_subtract>; @@ -254,7 +253,7 @@ TYPED_TEST(HipcubBlockAdjacentDifferenceSubtract, SubtractLeft) using output_type = typename TestFixture::params_subtract::output; - using stored_type = std::conditional_t::value, int, output_type>; + using stored_type = std::conditional_t, int, output_type>; constexpr size_t block_size = TestFixture::params_subtract::block_size; constexpr size_t items_per_thread = TestFixture::params_subtract::items_per_thread; @@ -337,8 +336,8 @@ TYPED_TEST(HipcubBlockAdjacentDifferenceSubtract, SubtractLeft) ASSERT_NO_FATAL_FAILURE( test_utils::assert_near(output, expected, - std::max(test_utils::precision::value, - test_utils::precision::value))); + _HIPCUB_STD::max(test_utils::precision::value, + test_utils::precision::value))); HIP_CHECK(hipFree(d_input)); HIP_CHECK(hipFree(d_output)); @@ -356,7 +355,7 @@ TYPED_TEST(HipcubBlockAdjacentDifferenceSubtract, SubtractLeftPartialTile) using output_type = typename TestFixture::params_subtract::output; - using stored_type = std::conditional_t::value, int, output_type>; + using stored_type = std::conditional_t, int, output_type>; constexpr size_t block_size = TestFixture::params_subtract::block_size; constexpr size_t items_per_thread = TestFixture::params_subtract::items_per_thread; @@ -458,8 +457,8 @@ TYPED_TEST(HipcubBlockAdjacentDifferenceSubtract, SubtractLeftPartialTile) ASSERT_NO_FATAL_FAILURE( test_utils::assert_near(output, expected, - std::max(test_utils::precision::value, - test_utils::precision::value))); + _HIPCUB_STD::max(test_utils::precision::value, + test_utils::precision::value))); HIP_CHECK(hipFree(d_input)); HIP_CHECK(hipFree(d_tile_sizes)); @@ -478,7 +477,7 @@ TYPED_TEST(HipcubBlockAdjacentDifferenceSubtract, SubtractRight) using output_type = typename TestFixture::params_subtract::output; - using stored_type = std::conditional_t::value, int, output_type>; + using stored_type = std::conditional_t, int, output_type>; constexpr size_t block_size = TestFixture::params_subtract::block_size; constexpr size_t items_per_thread = TestFixture::params_subtract::items_per_thread; @@ -561,8 +560,8 @@ TYPED_TEST(HipcubBlockAdjacentDifferenceSubtract, SubtractRight) ASSERT_NO_FATAL_FAILURE( test_utils::assert_near(output, expected, - std::max(test_utils::precision::value, - test_utils::precision::value))); + _HIPCUB_STD::max(test_utils::precision::value, + test_utils::precision::value))); HIP_CHECK(hipFree(d_input)); HIP_CHECK(hipFree(d_output)); @@ -580,7 +579,7 @@ TYPED_TEST(HipcubBlockAdjacentDifferenceSubtract, SubtractRightPartialTile) using output_type = typename TestFixture::params_subtract::output; - using stored_type = std::conditional_t::value, int, output_type>; + using stored_type = std::conditional_t, int, output_type>; constexpr size_t block_size = TestFixture::params_subtract::block_size; constexpr size_t items_per_thread = TestFixture::params_subtract::items_per_thread; @@ -682,8 +681,8 @@ TYPED_TEST(HipcubBlockAdjacentDifferenceSubtract, SubtractRightPartialTile) // clang-format off ASSERT_NO_FATAL_FAILURE(test_utils::assert_near(output, expected, is_add_op::value - ? std::max(test_utils::precision::value, test_utils::precision::value) - : std::is_same::value + ? _HIPCUB_STD::max(test_utils::precision::value, test_utils::precision::value) + : std::is_same_v ? 0 : test_utils::precision::value)); // clang-format on diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_discontinuity.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_discontinuity.cpp index 0bed699bdefb..61e44a99023e 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_discontinuity.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_discontinuity.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -26,7 +26,6 @@ #include #include #include -#include template struct params @@ -73,8 +72,8 @@ bool apply(FlagOp flag_op, const T& a, const T& b, unsigned int) using Params = ::testing::Types< // Power of 2 BlockSize - params, - params, + params, + params, params, params, params, @@ -87,18 +86,18 @@ using Params = ::testing::Types< params, params, params, - params, - params, + params, + params, // Power of 2 BlockSize and ItemsPerThread > 1 params>, params, params>, - params, + params, // Non-power of 2 BlockSize and ItemsPerThread > 1 params>, - params, + params, params, params, params, @@ -146,10 +145,10 @@ TYPED_TEST(HipcubBlockDiscontinuity, FlagHeads) using type = typename TestFixture::params::type; // std::vector is a special case that will cause an error in hipMemcpy - using stored_flag_type = typename std::conditional< - std::is_same::value, - int, - typename TestFixture::params::flag_type>::type; + using stored_flag_type = + typename std::conditional, + int, + typename TestFixture::params::flag_type>::type; using flag_type = typename TestFixture::params::flag_type; using flag_op_type = typename TestFixture::params::flag_op_type; constexpr size_t block_size = TestFixture::params::block_size; @@ -284,10 +283,10 @@ TYPED_TEST(HipcubBlockDiscontinuity, FlagTails) using type = typename TestFixture::params::type; // std::vector is a special case that will cause an error in hipMemcpy - using stored_flag_type = typename std::conditional< - std::is_same::value, - int, - typename TestFixture::params::flag_type>::type; + using stored_flag_type = + typename std::conditional, + int, + typename TestFixture::params::flag_type>::type; using flag_type = typename TestFixture::params::flag_type; using flag_op_type = typename TestFixture::params::flag_op_type; constexpr size_t block_size = TestFixture::params::block_size; @@ -451,10 +450,10 @@ TYPED_TEST(HipcubBlockDiscontinuity, FlagHeadsAndTails) using type = typename TestFixture::params::type; // std::vector is a special case that will cause an error in hipMemcpy - using stored_flag_type = typename std::conditional< - std::is_same::value, - int, - typename TestFixture::params::flag_type>::type; + using stored_flag_type = + typename std::conditional, + int, + typename TestFixture::params::flag_type>::type; using flag_type = typename TestFixture::params::flag_type; using flag_op_type = typename TestFixture::params::flag_op_type; constexpr size_t block_size = TestFixture::params::block_size; diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_exchange.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_exchange.cpp index afa6f4c6265d..4db4d9ea1370 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_exchange.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_exchange.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -908,7 +908,7 @@ TYPED_TEST(HipcubBlockExchangeTests, ScatterToStripedGuarded) const size_t i0 = offset + ti * items_per_thread + ii; const size_t i1 = offset + host_ranks[i0] % block_size * items_per_thread + host_ranks[i0] / block_size; - if(i1 >= 0 && i1 < size) + if(i1 < size) host_expected[i1] = host_input[i0]; } } @@ -1025,7 +1025,7 @@ TYPED_TEST(HipcubBlockExchangeTests, ScatterToStripedFlagged) const size_t i0 = offset + ti * items_per_thread + ii; const size_t i1 = offset + host_ranks[i0] % block_size * items_per_thread + host_ranks[i0] / block_size; - if(i1 >= 0 && i1 < size) + if(i1 < size) host_expected[i1] = host_input[i0]; host_flags[i0] = (ti == block_size - 1) && (ii == items_per_thread - 1) ? false : true; @@ -1818,7 +1818,7 @@ TYPED_TEST(HipcubBlockExchangeTests, ScatterToStripedGuardedNoOutputParam) const size_t i0 = offset + ti * items_per_thread + ii; const size_t i1 = offset + host_ranks[i0] % block_size * items_per_thread + host_ranks[i0] / block_size; - if(i1 >= 0 && i1 < size) + if(i1 < size) host_expected[i1] = host_input[i0]; } } @@ -1930,7 +1930,7 @@ TYPED_TEST(HipcubBlockExchangeTests, ScatterToStripedFlaggedNoOutputParam) const size_t i0 = offset + ti * items_per_thread + ii; const size_t i1 = offset + host_ranks[i0] % block_size * items_per_thread + host_ranks[i0] / block_size; - if(i1 >= 0 && i1 < size) + if(i1 < size) host_expected[i1] = host_input[i0]; host_flags[i0] = (ti == block_size - 1) && (ii == items_per_thread - 1) ? false : true; diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_histogram.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_histogram.cpp index 0691aabde326..78f55bdd6264 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_histogram.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_histogram.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -117,7 +117,7 @@ void histogram_kernel(T* device_output, T* device_output_bin) bhistogram_t(temp_storage).Histogram(in_out, hist); __syncthreads(); - #pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for (unsigned int offset = 0; offset < BinSize; offset += BlockSize) { if(offset + hipThreadIdx_x < BinSize) diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_load_store.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_load_store.cpp index b9ff9334b3f8..04fbb2b522d4 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_load_store.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_load_store.cpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2020, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -34,7 +34,6 @@ // kernel definitions #include "test_hipcub_block_load_store.kernels.hpp" -#include // Start stamping out tests struct HipcubBlockLoadStoreTests; diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_load_store.hpp b/projects/hipcub/test/hipcub/test_hipcub_block_load_store.hpp index bf5fa0e2e1cb..bd39aeb70d9c 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_load_store.hpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_load_store.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -371,24 +371,19 @@ typed_test_def(HipcubBlockLoadStoreTests, name_suffix, LoadStoreDiscardIterator) input.size() * sizeof(typename decltype(input)::value_type), hipMemcpyHostToDevice)); - // Test with discard output iterator - // using OffsetT = typename std::iterator_traits::difference_type; - HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH - // TODO: Here in block load an store, it's not possible to use rocprim::discard_iterator - hipcub::DiscardOutputIterator discard_itr; + // Running kernel for discard case + Type* dummy; + HIP_CHECK(hipMalloc(&dummy, guarded_elements * sizeof(Type))); - // Running kernel - load_store_guarded_kernel, + load_store_guarded_kernel - <<>>(device_input, - discard_itr, - discard_itr, - guarded_elements); - HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP + <<>>(device_input, dummy, dummy, guarded_elements); + HIP_CHECK(hipFree(dummy)); + // Running kernel load_store_guarded_kernel::value_type; + using InputT = hipcub::detail::it_value_t; // The output value type using OutputT = typename std::conditional< - (std::is_same::value_type, - void>::value), // OutputT = (if output iterator's value type is void) ? - typename std::iterator_traits::value_type, // ... then the input iterator's + (std::is_same_v, + void>), // OutputT = (if output iterator's value type is void) ? + hipcub::detail::it_value_t, // ... then the input iterator's // value type, - typename std::iterator_traits::value_type>:: + hipcub::detail::it_value_t>:: type; // ... else the output iterator's value type // Threadblock load/store abstraction types @@ -191,7 +191,7 @@ __launch_bounds__(BlockSize) __global__ __syncthreads(); // reset data -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(unsigned int item = 0; item < ItemsPerThread; ++item) data[item] = OutputT(); diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_merge_sort.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_merge_sort.cpp index 6eda70ee2534..0258a33880e3 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_merge_sort.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_merge_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -123,10 +123,11 @@ TYPED_TEST(HipcubBlockMergeSort, SortKeys) // Generate data std::vector keys_output; - keys_output = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value); + keys_output + = test_utils::get_random_data(size, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value); // Calculate expected results on host std::vector expected(keys_output); @@ -182,19 +183,45 @@ void sort_key_with_valid_items_kernel(T* device_input, T default_val) { constexpr size_t items_per_block = items_per_thread * block_size; - const size_t offset = (blockIdx.x * items_per_block) + (threadIdx.x * items_per_thread); + + const int block_offset = static_cast(blockIdx.x * items_per_block); + const int thread_offset = static_cast(threadIdx.x * items_per_thread); T input[items_per_thread]; + // Define per-thread valid range within the block + const int thread_start = thread_offset; + const int thread_end = thread_start + static_cast(items_per_thread); + + // Count valid items this thread actually owns + const int local_valid = (thread_start >= valid_items) + ? 0 + : ((thread_end <= valid_items) ? static_cast(items_per_thread) + : (valid_items - thread_start)); + + // Load valid items and fill the rest with default_val for(size_t i = 0; i < items_per_thread; i++) - input[i] = device_input[offset + i]; + { + const int idx = block_offset + thread_offset + static_cast(i); + if(static_cast(i) < local_valid) + input[i] = device_input[idx]; + else + input[i] = default_val; + } - hipcub::BlockMergeSort bsort; + using BlockSort = hipcub::BlockMergeSort; + __shared__ + typename BlockSort::TempStorage temp_storage; + BlockSort bsort(temp_storage); - bsort.Sort(input, compare_op, valid_items, default_val); + // Sort the whole block since all invalid items are already default_val + bsort.Sort(input, compare_op); for(size_t i = 0; i < items_per_thread; i++) - device_input[offset + i] = input[i]; + { + const int idx = block_offset + thread_offset + static_cast(i); + device_input[idx] = input[i]; + } } TYPED_TEST(HipcubBlockMergeSort, SortKeysWithValidItems) @@ -217,8 +244,8 @@ TYPED_TEST(HipcubBlockMergeSort, SortKeysWithValidItems) constexpr size_t size = grid_size * items_per_block; // minus|plus two to prevent overflow weirdness - const T mini = std::numeric_limits::min() + static_cast(2); - const T maxi = std::numeric_limits::max() - static_cast(2); + const T mini = _HIPCUB_STD::numeric_limits::min() + static_cast(2); + const T maxi = _HIPCUB_STD::numeric_limits::max() - static_cast(2); const T default_val = static_cast(compare_op(mini, maxi) ? maxi : mini); const int valid_items_arr[8] = {items_per_block / 2, @@ -354,17 +381,18 @@ TYPED_TEST(HipcubBlockMergeSort, SortKeysValues) // Generate data std::vector keys_output; - keys_output = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value); + keys_output + = test_utils::get_random_data(size, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value); std::vector values_output; - values_output - = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value + seed_value_addition); + values_output = test_utils::get_random_data( + size, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value + seed_value_addition); using key_value = std::pair; @@ -593,17 +621,18 @@ TYPED_TEST(HipcubBlockMergeSort, StableSortKeysValues) // Generate data std::vector keys_output; - keys_output = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value); + keys_output + = test_utils::get_random_data(size, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value); std::vector values_output; - values_output - = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value + seed_value_addition); + values_output = test_utils::get_random_data( + size, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value + seed_value_addition); // Set some keys to be the same, but have different values to test stability for(size_t i = 0; i < 10; i++) @@ -694,23 +723,45 @@ void stable_sort_key_with_valid_items_kernel(T* device_input, T default_val) { constexpr size_t items_per_block = items_per_thread * block_size; - const size_t offset = (blockIdx.x * items_per_block) + (threadIdx.x * items_per_thread); + const int block_offset = static_cast(blockIdx.x * items_per_block); + const int thread_offset = static_cast(threadIdx.x * items_per_thread); T input[items_per_thread]; + // Define per-thread valid range within the block + const int thread_start = thread_offset; + const int thread_end = thread_start + static_cast(items_per_thread); + + const int local_valid = (thread_start >= valid_items) + ? 0 + : ((thread_end <= valid_items) ? static_cast(items_per_thread) + : (valid_items - thread_start)); + + // Load valid items and fill invalid ones with default_val for(size_t i = 0; i < items_per_thread; i++) - input[i] = device_input[offset + i]; + { + const int idx = block_offset + thread_offset + static_cast(i); + + if(static_cast(i) < local_valid) + input[i] = device_input[idx]; + else + input[i] = default_val; + } - hipcub::BlockMergeSort bsort; + using BlockSort = hipcub::BlockMergeSort; + __shared__ + typename BlockSort::TempStorage temp_storage; + BlockSort bsort(temp_storage); - bsort.StableSort( - input, - [&](const T& lhs, const T& rhs) { return compare_op(lhs.elem, rhs.elem); }, - valid_items, - default_val); + // Stable-sort the whole block since all invalid items are masked + bsort.StableSort(input, + [&](const T& lhs, const T& rhs) { return compare_op(lhs.elem, rhs.elem); }); for(size_t i = 0; i < items_per_thread; i++) - device_input[offset + i] = input[i]; + { + const int idx = block_offset + thread_offset + static_cast(i); + device_input[idx] = input[i]; + } } TYPED_TEST(HipcubBlockMergeSort, StableSortKeysWithValidItems) @@ -734,8 +785,8 @@ TYPED_TEST(HipcubBlockMergeSort, StableSortKeysWithValidItems) constexpr size_t size = grid_size * items_per_block; // minus|plus two to prevent overflow weirdness - const T mini = std::numeric_limits::min() + static_cast(2); - const T maxi = std::numeric_limits::max() - static_cast(2); + const T mini = _HIPCUB_STD::numeric_limits::min() + static_cast(2); + const T maxi = _HIPCUB_STD::numeric_limits::max() - static_cast(2); const custom_type default_val = {static_cast(compare_op(mini, maxi) ? maxi : mini), 0}; const int valid_items_arr[8] = {items_per_block / 2, @@ -834,25 +885,52 @@ void stable_sort_key_value_with_valid_items_kernel(T* device_key_input, T default_val) { constexpr size_t items_per_block = items_per_thread * block_size; - const size_t offset = (blockIdx.x * items_per_block) + (threadIdx.x * items_per_thread); + + const int block_offset = static_cast(blockIdx.x * items_per_block); + const int thread_offset = static_cast(threadIdx.x * items_per_thread); T key_input[items_per_thread]; T value_input[items_per_thread]; + // Define per-thread valid range + const int thread_start = thread_offset; + const int thread_end = thread_start + static_cast(items_per_thread); + + const int local_valid = (thread_start >= valid_items) + ? 0 + : ((thread_end <= valid_items) ? static_cast(items_per_thread) + : (valid_items - thread_start)); + + // Load valid items and fill invalid ones with default_val for(size_t i = 0; i < items_per_thread; i++) { - key_input[i] = device_key_input[offset + i]; - value_input[i] = device_value_input[offset + i]; + const int idx = block_offset + thread_offset + static_cast(i); + + if(static_cast(i) < local_valid) + { + key_input[i] = device_key_input[idx]; + value_input[i] = device_value_input[idx]; + } + else + { + key_input[i] = default_val; + value_input[i] = device_value_input[idx]; + } } - hipcub::BlockMergeSort bsort; + using BlockSort = hipcub::BlockMergeSort; + __shared__ + typename BlockSort::TempStorage temp_storage; + BlockSort bsort(temp_storage); - bsort.StableSort(key_input, value_input, compare_op, valid_items, default_val); + // Sort entire block since all invalid items are masked + bsort.StableSort(key_input, value_input, compare_op); for(size_t i = 0; i < items_per_thread; i++) { - device_key_input[offset + i] = key_input[i]; - device_value_input[offset + i] = value_input[i]; + const int idx = block_offset + thread_offset + static_cast(i); + device_key_input[idx] = key_input[i]; + device_value_input[idx] = value_input[i]; } } @@ -882,8 +960,8 @@ TYPED_TEST(HipcubBlockMergeSort, StableSortKeysValuesWithValidItems) constexpr size_t size = grid_size * items_per_block; // minus|plus two to prevent overflow weirdness - const T mini = std::numeric_limits::min() + static_cast(2); - const T maxi = std::numeric_limits::max() - static_cast(2); + const T mini = _HIPCUB_STD::numeric_limits::min() + static_cast(2); + const T maxi = _HIPCUB_STD::numeric_limits::max() - static_cast(2); T default_val = static_cast(compare_op(mini, maxi) ? maxi : mini); const int valid_items_arr[8] = {items_per_block / 2, diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_radix_rank.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_radix_rank.cpp index 9e5ca6ccb38b..f8311dfb6beb 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_radix_rank.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_radix_rank.cpp @@ -341,7 +341,7 @@ void rank_kernel(const KeyType* keys_input, UnsignedBits(&unsigned_keys)[ItemsPerThread] = reinterpret_cast(keys); -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(unsigned int key = 0; key < ItemsPerThread; key++) { unsigned_keys[key] = KeyTraits::TwiddleIn(unsigned_keys[key]); @@ -531,18 +531,39 @@ void rank_with_prefix_sum_kernel(const KeyType* keys_input, int* prefix_sum_output, unsigned int start_bit) { +#if defined(__HIP_PLATFORM_NVIDIA__) + constexpr bool warp_striped = false; // CUB BlockRadixRankMatch expects blocked layout +#else constexpr bool warp_striped = Algorithm == RadixRankAlgorithm::RADIX_RANK_MATCH; +#endif using KeyTraits = hipcub::Traits; using UnsignedBits = typename KeyTraits::UnsignedBits; using DigitExtractor = hipcub::BFEDigitExtractor; - using RankType = std::conditional_t< - Algorithm == RadixRankAlgorithm::RADIX_RANK_MATCH, - hipcub::BlockRadixRankMatch, - hipcub::BlockRadixRank>; + + using RankType = +#if defined(__HIP_PLATFORM_NVIDIA__) + // For CUB + ULL + MATCH, fall back to basic BlockRadixRank + std::conditional_t< + Algorithm == RadixRankAlgorithm::RADIX_RANK_MATCH + && std::is_same_v, + hipcub::BlockRadixRank, + std::conditional_t< + Algorithm == RadixRankAlgorithm::RADIX_RANK_MATCH, + hipcub::BlockRadixRankMatch, + hipcub::BlockRadixRank>>; +#else + std::conditional_t< + Algorithm == RadixRankAlgorithm::RADIX_RANK_MATCH, + hipcub::BlockRadixRankMatch, + hipcub::BlockRadixRank>; +#endif using KeyExchangeType = hipcub::BlockExchange; using RankExchangeType = hipcub::BlockExchange; @@ -571,7 +592,7 @@ void rank_with_prefix_sum_kernel(const KeyType* keys_input, UnsignedBits(&unsigned_keys)[ItemsPerThread] = reinterpret_cast(keys); -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(unsigned int key = 0; key < ItemsPerThread; key++) { unsigned_keys[key] = KeyTraits::TwiddleIn(unsigned_keys[key]); @@ -595,13 +616,28 @@ void rank_with_prefix_sum_kernel(const KeyType* keys_input, hipcub::StoreDirectBlocked(lid, ranks_output + block_offset, ranks); - const size_t pfs_size = (1 << RadixBits); - const size_t pfs_offset = (blockIdx.x * pfs_size) + (threadIdx.x * bins_tracked_per_thread); + const size_t pfs_size = (1 << RadixBits); + const size_t pfs_offset = (blockIdx.x * pfs_size); for(size_t i = 0; i < bins_tracked_per_thread; i++) { - if((threadIdx.x * bins_tracked_per_thread) + i < pfs_size) - prefix_sum_output[pfs_offset + i] = prefix_sum_storage[i]; + const size_t local_bin = threadIdx.x * bins_tracked_per_thread + i; + if(local_bin >= pfs_size) + continue; + +#if defined(__HIP_PLATFORM_NVIDIA__) + if constexpr(std::is_same_v && Descending) + { + // Make CUB's layout match rocPRIM's: flip the global bin index + const size_t mirrored_bin = pfs_size - 1 - local_bin; + prefix_sum_output[pfs_offset + mirrored_bin] = prefix_sum_storage[i]; + } + else +#endif + { + // Normal (rocPRIM-compatible) layout + prefix_sum_output[pfs_offset + local_bin] = prefix_sum_storage[i]; + } } } @@ -640,7 +676,7 @@ void test_radix_rank_with_prefix_sum_output() constexpr unsigned end_bit = start_bit + radix_bits; constexpr size_t items_per_block = block_size * items_per_thread; - if constexpr(std::is_same::value) + if constexpr(std::is_same_v) { // Given block size not supported @@ -715,10 +751,10 @@ void test_radix_rank_with_prefix_sum_output() uint64_t bit_rep = c.out; bit_rep >>= start_bit; - bit_rep &= ((1 << radix_bits) - 1); + bit_rep &= ((1ull << radix_bits) - 1); if(descending) - bit_rep = (1 << radix_bits) - (1 + bit_rep); //flip it + bit_rep = (1ull << radix_bits) - (1 + bit_rep); //flip it ++histogram[bit_rep]; } @@ -783,14 +819,22 @@ void test_radix_rank_with_prefix_sum_output() { SCOPED_TRACE(testing::Message() << "with index= " << i); ASSERT_EQ(ranks_output[i], expected[i]); - - if(i < pfs_size) - ASSERT_EQ(prefix_sum_output[i], pfs_expected[i]); } HIP_CHECK(hipFree(d_keys_input)); HIP_CHECK(hipFree(d_ranks_output)); HIP_CHECK(hipFree(d_prefix_sum_output)); + + for(size_t block = 0; block < grid_size; ++block) + { + const size_t block_pfs_offset = block * pfs_items_per_block; + + for(size_t bin = 0; bin < pfs_items_per_block; ++bin) + { + const size_t idx = block_pfs_offset + bin; + ASSERT_EQ(prefix_sum_output[idx], pfs_expected[idx]); + } + } } } } diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_radix_sort.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_radix_sort.cpp index f5abe06154b2..7a156732f9a3 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_radix_sort.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_radix_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -60,7 +60,7 @@ class HipcubBlockRadixSort : public ::testing::Test using Params = ::testing::Types< // Power of 2 BlockSize -#if HIPCUB_IS_INT128_ENABLED +#if _CCCL_HAS_INT128() params<__int128_t, __int128_t, 64U, 1>, params<__uint128_t, __uint128_t, 64U, 1>, #endif @@ -448,11 +448,11 @@ TYPED_TEST(HipcubBlockRadixSort, SortKeys) } else { - keys_output - = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value); + keys_output = test_utils::get_random_data( + size, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value); } // Calculate expected results on host @@ -570,11 +570,11 @@ TYPED_TEST(HipcubBlockRadixSort, SortKeysValues) } else { - keys_output - = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value); + keys_output = test_utils::get_random_data( + size, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value); } std::vector values_output; @@ -588,11 +588,11 @@ TYPED_TEST(HipcubBlockRadixSort, SortKeysValues) } else { - values_output - = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value + seed_value_addition); + values_output = test_utils::get_random_data( + size, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value + seed_value_addition); } using key_value = std::pair; diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_reduce.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_reduce.cpp index ee1be7338dfc..11e078842c10 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_reduce.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_reduce.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -24,7 +24,6 @@ // hipcub API #include -#include // Params for tests template; - __shared__ typename breduce_t::TempStorage temp_storage; - value = breduce_t(temp_storage).Reduce(value, hipcub::Sum()); + __shared__ + typename breduce_t::TempStorage temp_storage; + value = breduce_t(temp_storage).Reduce(value, test_utils::plus{}); if(hipThreadIdx_x == 0) { device_output_reductions[hipBlockIdx_x] = value; @@ -337,8 +337,9 @@ void reduce_valid_kernel(T* device_output, const unsigned int index = (hipBlockIdx_x * BlockSize) + hipThreadIdx_x; T value = device_output[index]; using breduce_t = hipcub::BlockReduce; - __shared__ typename breduce_t::TempStorage temp_storage; - value = breduce_t(temp_storage).Reduce(value, hipcub::Sum(), valid_items); + __shared__ + typename breduce_t::TempStorage temp_storage; + value = breduce_t(temp_storage).Reduce(value, test_utils::plus{}, valid_items); if(hipThreadIdx_x == 0) { device_output_reductions[hipBlockIdx_x] = value; @@ -606,7 +607,7 @@ void reduce_array_kernel(T* device_output, T* device_output_reductions) T reduction; using breduce_t = hipcub::BlockReduce; __shared__ typename breduce_t::TempStorage temp_storage; - reduction = breduce_t(temp_storage).Reduce(in_out, hipcub::Sum()); + reduction = breduce_t(temp_storage).Reduce(in_out, test_utils::plus{}); if(hipThreadIdx_x == 0) { diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_run_length_decode.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_run_length_decode.cpp index 02250d546230..b969423be9f0 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_run_length_decode.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_run_length_decode.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -112,11 +112,11 @@ void block_run_length_decode_kernel(const ItemT* d_run_items, ItemT decoded_items[DecodedItemsPerThread]; block_run_length_decode.RunLengthDecode(decoded_items, decoded_window_offset); - hipcub::StoreDirectBlocked( - global_thread_idx, - d_decoded_items + decoded_window_offset, - decoded_items, - hipcub::Min{}(total_decoded_size - decoded_window_offset, decoded_items_per_block)); + hipcub::StoreDirectBlocked(global_thread_idx, + d_decoded_items + decoded_window_offset, + decoded_items, + test_utils::minimum{}(total_decoded_size - decoded_window_offset, + decoded_items_per_block)); decoded_window_offset += decoded_items_per_block; } @@ -141,13 +141,15 @@ TYPED_TEST(HipcubBlockRunLengthDecodeTest, TestDecode) SCOPED_TRACE(testing::Message() << "with seed= " << seed_value); const LengthT max_run_length = static_cast( - std::min(1000ll, static_cast(std::numeric_limits::max()))); + _HIPCUB_STD::min(1000ll, + static_cast(_HIPCUB_STD::numeric_limits::max()))); size_t num_runs = runs_per_thread * block_size; - auto run_items = test_utils::get_random_data(num_runs, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value); + auto run_items + = test_utils::get_random_data(num_runs, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value); auto run_lengths = test_utils::get_random_data(num_runs, static_cast(1), max_run_length, @@ -160,8 +162,8 @@ TYPED_TEST(HipcubBlockRunLengthDecodeTest, TestDecode) const auto empty_run_items = test_utils::get_random_data(num_trailing_empty_runs, - std::numeric_limits::min(), - std::numeric_limits::max(), + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), seed_value); // Not strictly required, but fixes a spurious GCC warning and good practice anyways run_items.reserve(run_items.size() + empty_run_items.size()); diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_scan.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_scan.cpp index edc0728aa05e..aef679bb5c42 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_scan.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_scan.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -96,7 +96,7 @@ void inclusive_scan_kernel(T* device_output) using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).InclusiveScan(value, value, hipcub::Sum()); + bscan_t(temp_storage).InclusiveScan(value, value, test_utils::plus{}); device_output[index] = value; } @@ -201,7 +201,7 @@ void inclusive_scan_initial_value_kernel(T* device_output, T initial_value) using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).InclusiveScan(input, output, initial_value, hipcub::Sum()); + bscan_t(temp_storage).InclusiveScan(input, output, initial_value, test_utils::plus{}); for(unsigned int i = 0; i < ItemsPerThread; ++i) { @@ -304,7 +304,7 @@ void inclusive_scan_reduce_kernel(T* device_output, T* device_output_reductions) T reduction; using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).InclusiveScan(value, value, hipcub::Sum(), reduction); + bscan_t(temp_storage).InclusiveScan(value, value, test_utils::plus{}, reduction); device_output[index] = value; if(hipThreadIdx_x == 0) { @@ -438,7 +438,7 @@ void inclusive_scan_reduce_initial_value_kernel(T* device_output, using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).InclusiveScan(input, output, initial_value, hipcub::Sum(), reduction); + bscan_t(temp_storage).InclusiveScan(input, output, initial_value, test_utils::plus{}, reduction); for(unsigned int i = 0; i < ItemsPerThread; ++i) { @@ -579,7 +579,7 @@ void inclusive_scan_prefix_callback_kernel(T* device_output, T* device_output_bp using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).InclusiveScan(value, value, hipcub::Sum(), prefix_callback); + bscan_t(temp_storage).InclusiveScan(value, value, test_utils::plus{}, prefix_callback); device_output[index] = value; if(hipThreadIdx_x == 0) @@ -698,7 +698,7 @@ void exclusive_scan_kernel(T* device_output, T init) T value = device_output[index]; using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).ExclusiveScan(value, value, init, hipcub::Sum()); + bscan_t(temp_storage).ExclusiveScan(value, value, init, test_utils::plus{}); device_output[index] = value; } @@ -795,7 +795,7 @@ void exclusive_scan_reduce_kernel(T* device_output, T* device_output_reductions, T reduction; using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).ExclusiveScan(value, value, init, hipcub::Sum(), reduction); + bscan_t(temp_storage).ExclusiveScan(value, value, init, test_utils::plus{}, reduction); device_output[index] = value; if(hipThreadIdx_x == 0) { @@ -932,7 +932,7 @@ void exclusive_scan_prefix_callback_kernel(T* device_output, T* device_output_bp using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).ExclusiveScan(value, value, hipcub::Sum(), prefix_callback); + bscan_t(temp_storage).ExclusiveScan(value, value, test_utils::plus{}, prefix_callback); device_output[index] = value; if(hipThreadIdx_x == 0) @@ -1907,7 +1907,7 @@ void inclusive_scan_array_kernel(T* device_output) using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).InclusiveScan(in_out, in_out, hipcub::Sum()); + bscan_t(temp_storage).InclusiveScan(in_out, in_out, test_utils::plus{}); // store for(unsigned int j = 0; j < ItemsPerThread; j++) @@ -2020,7 +2020,7 @@ void inclusive_scan_reduce_array_kernel(T* device_output, T* device_output_reduc using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; T reduction; - bscan_t(temp_storage).InclusiveScan(in_out, in_out, hipcub::Sum(), reduction); + bscan_t(temp_storage).InclusiveScan(in_out, in_out, test_utils::plus{}, reduction); // store for(unsigned int j = 0; j < ItemsPerThread; j++) @@ -2171,7 +2171,7 @@ void inclusive_scan_array_prefix_callback_kernel(T* device_output, using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).InclusiveScan(in_out, in_out, hipcub::Sum(), prefix_callback); + bscan_t(temp_storage).InclusiveScan(in_out, in_out, test_utils::plus{}, prefix_callback); // store for(unsigned int j = 0; j < ItemsPerThread; j++) @@ -2319,7 +2319,7 @@ void exclusive_scan_array_kernel(T* device_output, T init) using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).ExclusiveScan(in_out, in_out, init, hipcub::Sum()); + bscan_t(temp_storage).ExclusiveScan(in_out, in_out, init, test_utils::plus{}); // store for(unsigned int j = 0; j < ItemsPerThread; j++) @@ -2440,7 +2440,7 @@ void exclusive_scan_reduce_array_kernel(T* device_output, T* device_output_reduc using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; T reduction; - bscan_t(temp_storage).ExclusiveScan(in_out, in_out, init, hipcub::Sum(), reduction); + bscan_t(temp_storage).ExclusiveScan(in_out, in_out, init, test_utils::plus{}, reduction); // store for(unsigned int j = 0; j < ItemsPerThread; j++) @@ -2608,7 +2608,7 @@ void exclusive_scan_prefix_callback_array_kernel(T* device_output, using bscan_t = hipcub::BlockScan; __shared__ typename bscan_t::TempStorage temp_storage; - bscan_t(temp_storage).ExclusiveScan(in_out, in_out, hipcub::Sum(), prefix_callback); + bscan_t(temp_storage).ExclusiveScan(in_out, in_out, test_utils::plus{}, prefix_callback); // store for(unsigned int j = 0; j < ItemsPerThread; j++) diff --git a/projects/hipcub/test/hipcub/test_hipcub_block_shuffle.cpp b/projects/hipcub/test/hipcub/test_hipcub_block_shuffle.cpp index ea5b4376251b..b81581552a9f 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_block_shuffle.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_block_shuffle.cpp @@ -99,8 +99,8 @@ TYPED_TEST(HipcubBlockShuffleTests, BlockOffset) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; - int distance - = rand() % std::min(size_t(10), block_size / 2) - std::min(size_t(10), block_size / 2); + int distance = rand() % _HIPCUB_STD::min(size_t(10), block_size / 2) + - _HIPCUB_STD::min(size_t(10), block_size / 2); SCOPED_TRACE(testing::Message() << "with seed= " << seed_value << " & distance = " << distance); // Generate data @@ -183,7 +183,7 @@ TYPED_TEST(HipcubBlockShuffleTests, BlockRotate) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; - int distance = rand() % std::min(size_t(5), block_size / 2); + int distance = rand() % _HIPCUB_STD::min(size_t(5), block_size / 2); SCOPED_TRACE(testing::Message() << "with seed= " << seed_value << " & distance = " << distance); // Generate data diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_adjacent_difference.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_adjacent_difference.cpp index 155b748d041e..cb831ea02d3b 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_adjacent_difference.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_adjacent_difference.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -24,9 +24,6 @@ // hipcub API #include -#include -#include -#include #include "test_utils.hpp" #include "test_utils_data_generation.hpp" @@ -157,7 +154,7 @@ TYPED_TEST(HipcubDeviceAdjacentDifference, SubtractLeftCopy) static constexpr std::integral_constant copy_constant{}; using output_type = std::conditional_t; - static constexpr ::hipcub::Difference op; + static constexpr test_utils::minus op; hipStream_t stream = 0; if(TestFixture::params::use_graphs) @@ -460,7 +457,7 @@ TYPED_TEST(HipcubDeviceAdjacentDifferenceLargeTests, LargeIndicesAndOpOnce) static constexpr hipStream_t stream = 0; // default - for(std::size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) + for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { unsigned int seed_value = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; @@ -481,15 +478,15 @@ TYPED_TEST(HipcubDeviceAdjacentDifferenceLargeTests, LargeIndicesAndOpOnce) HIP_CHECK(hipMemset(d_counter, 0, sizeof(*d_counter))); OutputIterator output(d_incorrect_flag, d_counter); - const auto input = rocprim::counting_iterator(T{0}); + const auto input = test_utils::counting_iterator(T{0}); static constexpr auto left_tag = std::integral_constant{}; static constexpr auto copy_tag = std::integral_constant{}; FocusIndex op; // Allocate temporary storage - std::size_t temp_storage_size = 0; - void* d_temp_storage = nullptr; + size_t temp_storage_size = 0; + void* d_temp_storage = nullptr; HIP_CHECK(dispatch_adjacent_difference(left_tag, copy_tag, d_temp_storage, diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_copy.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_copy.cpp index e446d77af0b5..f9a3cf1d4f24 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_copy.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_copy.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -27,7 +27,6 @@ #include "test_utils_types.hpp" #include -#include #include #include @@ -268,12 +267,12 @@ TYPED_TEST(DeviceBatchCopyTests, SizeAndTypeVariation) h_buffer_num_elements.end(), 0, src_offsets.begin(), - hipcub::Sum{}); + test_utils::plus{}); test_utils::host_exclusive_scan(h_buffer_num_elements.begin(), h_buffer_num_elements.end(), 0, dst_offsets.begin(), - hipcub::Sum{}); + test_utils::plus{}); } // Generate the source and destination pointers. diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_for.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_for.cpp index 94fa2f8e59a0..5e291994a1f3 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_for.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_for.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2024-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -24,7 +24,6 @@ // required hipcub headers #include -#include #include #include @@ -692,7 +691,7 @@ TEST(HipcubDeviceForTests, ForCountingIterator) // Device pointers unsigned int* d_count; - const auto it = rocprim::counting_iterator{0}; + const auto it = test_utils::counting_iterator{0}; // Allocate memory HIP_CHECK(test_common_utils::hipMallocHelper(&d_count, sizeof(unsigned int))); @@ -745,7 +744,7 @@ TEST(HipcubDeviceForTests, ForCopyCountingIterator) // Device pointers unsigned int* d_count; - const auto it = rocprim::counting_iterator{0}; + const auto it = test_utils::counting_iterator{0}; // Allocate memory HIP_CHECK(test_common_utils::hipMallocHelper(&d_count, sizeof(unsigned int))); @@ -854,7 +853,7 @@ TEST(HipcubDeviceForTests, ForEachCopyNTempStore) } // ForEachInExtents only enables when the cccl mdspan extension is enabled -#if(defined(__HIP_PLATFORM_NVIDIA__) && defined(__cccl_lib_mdspan)) || defined(__HIP_PLATFORM_AMD__) +#if defined(__HIP_PLATFORM_NVIDIA__) || defined(__HIP_PLATFORM_AMD__) template struct HipcubTestParamsMerge @@ -897,19 +896,19 @@ struct HipcubDeviceForEachInExtentsTests : public ::testing::Test template using HipcubDeviceForEachInExtentsParamGenerator - = ::testing::Types>, - DeviceForEachInExtentsParams<::hipcub::extents>, - DeviceForEachInExtentsParams<::hipcub::extents>, - DeviceForEachInExtentsParams<::hipcub::extents>, - DeviceForEachInExtentsParams<::hipcub::extents>>; + = ::testing::Types>, + DeviceForEachInExtentsParams<::test_utils::extents>, + DeviceForEachInExtentsParams<::test_utils::extents>, + DeviceForEachInExtentsParams<::test_utils::extents>, + DeviceForEachInExtentsParams<::test_utils::extents>>; using HipcubDeviceForEachInExtentsTestsParams = typename HipcubTestParamsMergeAll< HipcubDeviceForEachInExtentsParamGenerator, - HipcubDeviceForEachInExtentsParamGenerator, - HipcubDeviceForEachInExtentsParamGenerator, - HipcubDeviceForEachInExtentsParamGenerator, - HipcubDeviceForEachInExtentsParamGenerator, - HipcubDeviceForEachInExtentsParamGenerator>::type; + HipcubDeviceForEachInExtentsParamGenerator, + HipcubDeviceForEachInExtentsParamGenerator, + HipcubDeviceForEachInExtentsParamGenerator, + HipcubDeviceForEachInExtentsParamGenerator<_HIPCUB_STD::int64_t>, + HipcubDeviceForEachInExtentsParamGenerator>::type; template -inline void fill_linear(std::vector& vector, const ::hipcub::extents& ext) +inline void fill_linear(std::vector& vector, + const ::test_utils::extents& ext) { size_t pos = 0; fill_linear_impl(vector, ext, pos); @@ -964,6 +964,22 @@ struct LinearStore } }; +template +struct ForEachInExtentsOp + +{ + using op_data_t = item_t[3]; + void* d_data; + + __device__ __host__ __forceinline__ + void operator()(int idx, int x, int y, int z) + { + auto& i = static_cast(d_data)[idx]; + // We use the "placement new" operator to copy the data from an initializer list. + new(&i) op_data_t{x, y, z}; + } +}; + TYPED_TEST_SUITE(HipcubDeviceForEachInExtentsTests, HipcubDeviceForEachInExtentsTestsParams); TEST(HipcubDeviceForEachInExtentsTests, ForEachInExtentsAPI) @@ -974,8 +990,8 @@ TEST(HipcubDeviceForEachInExtentsTests, ForEachInExtentsAPI) using item_t = int; using data_t = std::array; - using extents_type = hipcub::extents; - constexpr auto extents_size = hipcub::extents_size::value; + using extents_type = test_utils::extents; + constexpr auto extents_size = test_utils::extents_size::value; constexpr auto memory_size = extents_size * sizeof(data_t); constexpr extents_type ext{}; @@ -999,21 +1015,7 @@ TEST(HipcubDeviceForEachInExtentsTests, ForEachInExtentsAPI) HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, memory_size)); HIP_CHECK(hipMemset(d_input, 0, memory_size)); - struct Op - { - using op_data_t = item_t[3]; - void* d_data; - - __device__ __host__ __forceinline__ - void operator()(int idx, int x, int y, int z) - { - auto& i = static_cast(d_data)[idx]; - // We use the "placement new" operator to copy the data from an initializer list. - new(&i) op_data_t{x, y, z}; - } - }; - - HIP_CHECK(hipcub::DeviceFor::ForEachInExtents(ext, Op{d_input})); + HIP_CHECK(hipcub::DeviceFor::ForEachInExtents(ext, ForEachInExtentsOp{d_input})); HIP_CHECK(hipGetLastError()); HIP_CHECK(hipDeviceSynchronize()); @@ -1033,7 +1035,7 @@ TYPED_TEST(HipcubDeviceForEachInExtentsTests, ForEachInExtentsStatic) using item_t = index_type; using data_t = std::array; - constexpr auto extents_size = hipcub::extents_size::value; + constexpr auto extents_size = test_utils::extents_size::value; constexpr auto memory_size = extents_size * sizeof(data_t); constexpr auto rank = extents_type::rank(); using store_op_t = LinearStore; @@ -1065,16 +1067,16 @@ TYPED_TEST(HipcubDeviceForEachInExtentsTests, ForEachInExtentsStatic) HIP_CHECK(hipFree(d_input)); } -#endif // (defined(__HIP_PLATFORM_NVIDIA__) && defined(__cccl_lib_mdspan)) || defined(__HIP_PLATFORM_AMD__) +#endif // defined(__HIP_PLATFORM_NVIDIA__) || defined(__HIP_PLATFORM_AMD__) template class HipcubDeviceForBulkTests : public HipcubDeviceForTests {}; -using HipcubDeviceForBulkTestsParams = ::testing::Types, - DeviceForParams, - DeviceForParams, - DeviceForParams>; +using HipcubDeviceForBulkTestsParams = ::testing::Types, + DeviceForParams, + DeviceForParams<_HIPCUB_STD::int64_t>, + DeviceForParams>; TYPED_TEST_SUITE(HipcubDeviceForBulkTests, HipcubDeviceForBulkTestsParams); @@ -1087,7 +1089,7 @@ struct offset_count_device_t HIPCUB_DEVICE void operator()(OffsetT i) { - static_assert(std::is_same::value, "T and OffsetT must be the same type"); + static_assert(std::is_same_v, "T and OffsetT must be the same type"); atomicAdd(d_count + i, 1); } }; diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_histogram.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_histogram.cpp index 1433f69e56ac..48ed8092f9f3 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_histogram.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_histogram.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -30,8 +30,6 @@ // hipcub API #include -#include -#include // rows, columns, (row_stride - columns * Channels) std::vector> get_dims() @@ -73,9 +71,11 @@ inline auto get_random_samples(size_t size, U min, U max, unsigned int seed_valu return test_utils::get_random_data( size, static_cast( - std::max(min1 - d / 10, static_cast(std::numeric_limits::lowest()))), + _HIPCUB_STD::max(min1 - d / 10, + static_cast(_HIPCUB_STD::numeric_limits::lowest()))), static_cast( - std::min(max1 + d / 10, static_cast(std::numeric_limits::max()))), + _HIPCUB_STD::min(max1 + d / 10, + static_cast(_HIPCUB_STD::numeric_limits::max()))), seed_value); } @@ -89,8 +89,11 @@ inline auto get_random_samples(size_t size, U min, U max, unsigned int seed_valu return test_utils::get_random_data( size, static_cast( - std::max(min1 - d / 10, static_cast(std::numeric_limits::lowest()))), - static_cast(std::min(max1 + d / 10, static_cast(std::numeric_limits::max()))), + _HIPCUB_STD::max(min1 - d / 10, + static_cast(_HIPCUB_STD::numeric_limits::lowest()))), + static_cast( + _HIPCUB_STD::min(max1 + d / 10, + static_cast(_HIPCUB_STD::numeric_limits::max()))), seed_value); } @@ -195,7 +198,7 @@ TYPED_TEST(HipcubDeviceHistogramEven, Even) const size_t row_stride = columns + std::get<2>(dim); const size_t row_stride_bytes = row_stride * sizeof(sample_type); - const size_t size = std::max(1, rows * row_stride); + const size_t size = _HIPCUB_STD::max(1, rows * row_stride); for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { @@ -371,7 +374,7 @@ TYPED_TEST(HipcubDeviceHistogramEvenOverflow, EvenOverflow) const native_level_type n_lower_level = 0; const native_level_type n_upper_level - = static_cast(std::numeric_limits::max()); + = static_cast(_HIPCUB_STD::numeric_limits::max()); level_type lower_level = test_utils::convert_to_device(n_lower_level); level_type upper_level = test_utils::convert_to_device(n_upper_level); @@ -387,7 +390,7 @@ TYPED_TEST(HipcubDeviceHistogramEvenOverflow, EvenOverflow) SCOPED_TRACE(testing::Message() << "with seed= " << seed_value); // Generate data - auto d_input = rocprim::counting_iterator{0UL}; + auto d_input = test_utils::counting_iterator{0UL}; counter_type* d_histogram; HIP_CHECK(test_common_utils::hipMallocHelper(&d_histogram, bins * sizeof(counter_type))); @@ -470,7 +473,9 @@ using Params2 = ::testing::Types< params2, params2, params2, +#if defined(__HIP_PLATFORM_AMD__) params2, +#endif params2, params2, params2, @@ -522,7 +527,7 @@ TYPED_TEST(HipcubDeviceHistogramRange, Range) const size_t row_stride = columns + std::get<2>(dim); const size_t row_stride_bytes = row_stride * sizeof(sample_type); - const size_t size = std::max(1, rows * row_stride); + const size_t size = _HIPCUB_STD::max(1, rows * row_stride); for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { @@ -768,6 +773,14 @@ TYPED_TEST(HipcubDeviceHistogramMultiEven, MultiEven) upper_level[channel] = test_utils::convert_to_device(n_upper_level[channel]); } + // accuracy problems with bfloat and half + // nvidia cub also doesn't work + // TODO: check if nvidia works with only sample type bfloat/half + if(test_utils::is_half::value || test_utils::is_bfloat16::value) + { + GTEST_SKIP(); + } + hipStream_t stream = 0; // default if(TestFixture::params::use_graphs) { @@ -787,7 +800,7 @@ TYPED_TEST(HipcubDeviceHistogramMultiEven, MultiEven) const size_t row_stride = columns * channels + std::get<2>(dim); const size_t row_stride_bytes = row_stride * sizeof(sample_type); - const size_t size = std::max(1, rows * row_stride); + const size_t size = _HIPCUB_STD::max(1, rows * row_stride); for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { @@ -795,9 +808,9 @@ TYPED_TEST(HipcubDeviceHistogramMultiEven, MultiEven) SCOPED_TRACE(testing::Message() << "with seed= " << seed_value); std::vector channel_seeds = test_utils::get_random_data( - std::max(size, static_cast(channels)), - std::numeric_limits::min(), - std::numeric_limits::max(), + _HIPCUB_STD::max(size, static_cast(channels)), + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), seed_value + seed_value_addition // Make sure that we do not use the same or shifted sequence ); @@ -880,8 +893,9 @@ TYPED_TEST(HipcubDeviceHistogramMultiEven, MultiEven) } } } - rocprim::transform_iterator, sample_type> - d_input2(d_input, transform_op()); + test_utils::transform_iterator> d_input2( + d_input, + transform_op()); size_t temporary_storage_bytes = 0; if(rows == 1) { @@ -1026,7 +1040,9 @@ using Params4 = ::testing::Types< params4, params4, params4, +#if defined(__HIP_PLATFORM_AMD__) params4, +#endif params4, params4, params4, @@ -1089,7 +1105,7 @@ TYPED_TEST(HipcubDeviceHistogramMultiRange, MultiRange) const size_t row_stride = columns * channels + std::get<2>(dim); const size_t row_stride_bytes = row_stride * sizeof(sample_type); - const size_t size = std::max(1, rows * row_stride); + const size_t size = _HIPCUB_STD::max(1, rows * row_stride); for (size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) { @@ -1097,9 +1113,9 @@ TYPED_TEST(HipcubDeviceHistogramMultiRange, MultiRange) SCOPED_TRACE(testing::Message() << "with seed= " << seed_value); std::vector channel_seeds = test_utils::get_random_data( - std::max(size, static_cast(channels)), - std::numeric_limits::min(), - std::numeric_limits::max(), + _HIPCUB_STD::max(size, static_cast(channels)), + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), seed_value); // Generate data @@ -1210,8 +1226,9 @@ TYPED_TEST(HipcubDeviceHistogramMultiRange, MultiRange) } } } - rocprim::transform_iterator, sample_type> - d_input2(d_input, transform_op()); + test_utils::transform_iterator> d_input2( + d_input, + transform_op()); size_t temporary_storage_bytes = 0; if(rows == 1) { diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_memcpy.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_memcpy.cpp index b2b997ff8f10..342214e995c6 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_memcpy.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_memcpy.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -27,7 +27,6 @@ #include "test_utils_types.hpp" #include -#include #include #include @@ -272,12 +271,12 @@ TYPED_TEST(DeviceBatchMemcpyTests, SizeAndTypeVariation) h_buffer_num_elements.end(), 0, src_offsets.begin(), - hipcub::Sum{}); + test_utils::plus{}); test_utils::host_exclusive_scan(h_buffer_num_elements.begin(), h_buffer_num_elements.end(), 0, dst_offsets.begin(), - hipcub::Sum{}); + test_utils::plus{}); } // Generate the source and destination pointers. @@ -338,3 +337,306 @@ TYPED_TEST(DeviceBatchMemcpyTests, SizeAndTypeVariation) HIP_CHECK(hipFree(d_temp_storage)); } + +TEST(DeviceMemcpyBatched, ZeroBuffersNoOp) +{ + using T = uint8_t; + T** d_srcs = nullptr; + T** d_dsts = nullptr; + size_t* d_sizes = nullptr; + + size_t temp_bytes = 0; + HIP_CHECK(hipcub::DeviceMemcpy::Batched(nullptr, temp_bytes, d_srcs, d_dsts, d_sizes, 0)); + void* d_temp = nullptr; + if(temp_bytes) + { + HIP_CHECK(hipMalloc(&d_temp, temp_bytes)); + } + + // Should be a no-op without crashing + HIP_CHECK(hipcub::DeviceMemcpy::Batched(d_temp, + temp_bytes, + d_srcs, + d_dsts, + d_sizes, + 0, + hipStreamDefault)); + if(d_temp) + { + HIP_CHECK(hipFree(d_temp)); + } +} + +TEST(DeviceMemcpyBatched, ZeroSizeEntries) +{ + using T = uint8_t; + const int num_buffers = 5; + const std::vector h_sizes = {0, 1, 0, 7, 0}; + + size_t total = 0; + for(auto s : h_sizes) + { + total += s; + } + + T* d_input = nullptr; + T* d_output = nullptr; + HIP_CHECK(hipMalloc(&d_input, total)); + HIP_CHECK(hipMalloc(&d_output, total)); + + // Build src/dst arrays + std::vector h_srcs(num_buffers), h_dsts(num_buffers); + size_t offset = 0; + for(int i = 0; i < num_buffers; ++i) + { + h_srcs[i] = d_input + offset; + h_dsts[i] = d_output + offset; + offset += h_sizes[i]; + } + + // Fill input + std::vector h_in(total); + std::iota(h_in.begin(), h_in.end(), static_cast(3)); + HIP_CHECK(hipMemcpy(d_input, h_in.data(), total, hipMemcpyHostToDevice)); + + // Device arrays + T** d_srcs = nullptr; + T** d_dsts = nullptr; + size_t* d_sizes = nullptr; + HIP_CHECK(hipMalloc(&d_srcs, num_buffers * sizeof(T*))); + HIP_CHECK(hipMalloc(&d_dsts, num_buffers * sizeof(T*))); + HIP_CHECK(hipMalloc(&d_sizes, num_buffers * sizeof(size_t))); + HIP_CHECK(hipMemcpy(d_srcs, h_srcs.data(), num_buffers * sizeof(T*), hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(d_dsts, h_dsts.data(), num_buffers * sizeof(T*), hipMemcpyHostToDevice)); + HIP_CHECK( + hipMemcpy(d_sizes, h_sizes.data(), num_buffers * sizeof(size_t), hipMemcpyHostToDevice)); + + // Temp storage + size_t temp_bytes = 0; + HIP_CHECK( + hipcub::DeviceMemcpy::Batched(nullptr, temp_bytes, d_srcs, d_dsts, d_sizes, num_buffers)); + void* d_temp = nullptr; + HIP_CHECK(hipMalloc(&d_temp, temp_bytes)); + + HIP_CHECK( + hipcub::DeviceMemcpy::Batched(d_temp, temp_bytes, d_srcs, d_dsts, d_sizes, num_buffers)); + + // Verify + std::vector h_out(total); + HIP_CHECK(hipMemcpy(h_out.data(), d_output, total, hipMemcpyDeviceToHost)); + EXPECT_EQ(h_in, h_out); + + HIP_CHECK(hipFree(d_temp)); + HIP_CHECK(hipFree(d_sizes)); + HIP_CHECK(hipFree(d_dsts)); + HIP_CHECK(hipFree(d_srcs)); + HIP_CHECK(hipFree(d_output)); + HIP_CHECK(hipFree(d_input)); +} + +TEST(DeviceMemcpyBatched, NonDefaultStreamAndTempReuse) +{ + using T = uint32_t; + const int num_buffers = 8; + std::mt19937 rng(123); + std::uniform_int_distribution dist(1, 4096); + std::vector sizes(num_buffers); + size_t total = 0; + for(int i = 0; i < num_buffers; ++i) + { + sizes[i] = dist(rng) * sizeof(T); + total += sizes[i]; + } + + T* d_in = nullptr; + T* d_out = nullptr; + HIP_CHECK(hipMalloc(&d_in, total)); + HIP_CHECK(hipMalloc(&d_out, total)); + + // Fill input + std::vector h_in(total / sizeof(T)); + for(size_t i = 0; i < h_in.size(); ++i) + h_in[i] = static_cast(i ^ 0xDEADBEEF); + HIP_CHECK(hipMemcpy(d_in, h_in.data(), total, hipMemcpyHostToDevice)); + + // Build offset/size pairs + struct Chunk + { + size_t offset; + size_t size; + }; + std::vector chunks(num_buffers); + size_t acc = 0; + for(int i = 0; i < num_buffers; ++i) + { + chunks[i].offset = acc; + chunks[i].size = sizes[i]; + acc += sizes[i]; + } + std::shuffle(chunks.begin(), chunks.end(), rng); + + std::vector h_srcs(num_buffers), h_dsts(num_buffers); + std::vector h_sizes(num_buffers); + for(int i = 0; i < num_buffers; ++i) + { + h_srcs[i] = d_in + chunks[i].offset / sizeof(T); + h_dsts[i] = d_out + chunks[i].offset / sizeof(T); + h_sizes[i] = chunks[i].size; + } + + T** d_srcs = nullptr; + T** d_dsts = nullptr; + size_t* d_sizes = nullptr; + HIP_CHECK(hipMalloc(&d_srcs, num_buffers * sizeof(T*))); + HIP_CHECK(hipMalloc(&d_dsts, num_buffers * sizeof(T*))); + HIP_CHECK(hipMalloc(&d_sizes, num_buffers * sizeof(size_t))); + + // Setup stream and event + hipStream_t setup; + hipEvent_t ready; + HIP_CHECK(hipStreamCreate(&setup)); + HIP_CHECK(hipEventCreateWithFlags(&ready, hipEventDisableTiming)); + + HIP_CHECK(hipMemcpyAsync(d_srcs, + h_srcs.data(), + num_buffers * sizeof(T*), + hipMemcpyHostToDevice, + setup)); + HIP_CHECK(hipMemcpyAsync(d_dsts, + h_dsts.data(), + num_buffers * sizeof(T*), + hipMemcpyHostToDevice, + setup)); + HIP_CHECK(hipMemcpyAsync(d_sizes, + h_sizes.data(), + num_buffers * sizeof(size_t), + hipMemcpyHostToDevice, + setup)); + HIP_CHECK(hipEventRecord(ready, setup)); + + // Query temp storage + size_t temp_bytes = 0; + HIP_CHECK( + hipcub::DeviceMemcpy::Batched(nullptr, temp_bytes, d_srcs, d_dsts, d_sizes, num_buffers)); + + void* d_tempA = nullptr; + void* d_tempB = nullptr; + HIP_CHECK(hipMalloc(&d_tempA, temp_bytes)); + HIP_CHECK(hipMalloc(&d_tempB, temp_bytes)); + + hipStream_t streamA, streamB; + HIP_CHECK(hipStreamCreate(&streamA)); + HIP_CHECK(hipStreamCreate(&streamB)); + HIP_CHECK(hipStreamWaitEvent(streamA, ready, 0)); + HIP_CHECK(hipStreamWaitEvent(streamB, ready, 0)); + + // Launch batched memcpy + HIP_CHECK(hipcub::DeviceMemcpy::Batched(d_tempA, + temp_bytes, + d_srcs, + d_dsts, + d_sizes, + num_buffers, + streamA)); + HIP_CHECK(hipcub::DeviceMemcpy::Batched(d_tempB, + temp_bytes, + d_srcs, + d_dsts, + d_sizes, + num_buffers, + streamB)); + + HIP_CHECK(hipStreamSynchronize(streamA)); + HIP_CHECK(hipStreamSynchronize(streamB)); + + // Verify + std::vector h_out(h_in.size()); + HIP_CHECK(hipMemcpy(h_out.data(), d_out, total, hipMemcpyDeviceToHost)); + + for(size_t i = 0; i < h_in.size(); ++i) + { + EXPECT_EQ(h_in[i], h_out[i]) << "Mismatch at index " << i; + } + + // Cleanup + HIP_CHECK(hipEventDestroy(ready)); + HIP_CHECK(hipStreamDestroy(setup)); + HIP_CHECK(hipStreamDestroy(streamA)); + HIP_CHECK(hipStreamDestroy(streamB)); + HIP_CHECK(hipFree(d_tempA)); + HIP_CHECK(hipFree(d_tempB)); + HIP_CHECK(hipFree(d_sizes)); + HIP_CHECK(hipFree(d_dsts)); + HIP_CHECK(hipFree(d_srcs)); + HIP_CHECK(hipFree(d_out)); + HIP_CHECK(hipFree(d_in)); +} + +struct PackedPair +{ + uint16_t a; + uint16_t b; +}; // alignment-sensitive +TEST(DeviceMemcpyBatched, PackedStructAlignment) +{ + using T = PackedPair; + const int num_buffers = 4; + const size_t elems_per_buffer = 1024; + const size_t bytes_per_buffer = elems_per_buffer * sizeof(T); + const size_t total_bytes = num_buffers * bytes_per_buffer; + + T* d_in = nullptr; + T* d_out = nullptr; + HIP_CHECK(hipMalloc(&d_in, total_bytes)); + HIP_CHECK(hipMalloc(&d_out, total_bytes)); + + std::vector h_in(num_buffers * elems_per_buffer); + for(size_t i = 0; i < h_in.size(); ++i) + { + h_in[i] = T{static_cast(i), static_cast(~i)}; + } + HIP_CHECK(hipMemcpy(d_in, h_in.data(), total_bytes, hipMemcpyHostToDevice)); + + std::vector h_sizes(num_buffers, bytes_per_buffer); + std::vector h_srcs(num_buffers), h_dsts(num_buffers); + for(int i = 0; i < num_buffers; ++i) + { + h_srcs[i] = d_in + i * elems_per_buffer; + h_dsts[i] = d_out + i * elems_per_buffer; + } + + T** d_srcs = nullptr; + T** d_dsts = nullptr; + size_t* d_sizes = nullptr; + HIP_CHECK(hipMalloc(&d_srcs, num_buffers * sizeof(T*))); + HIP_CHECK(hipMalloc(&d_dsts, num_buffers * sizeof(T*))); + HIP_CHECK(hipMalloc(&d_sizes, num_buffers * sizeof(size_t))); + HIP_CHECK(hipMemcpy(d_srcs, h_srcs.data(), num_buffers * sizeof(T*), hipMemcpyHostToDevice)); + HIP_CHECK(hipMemcpy(d_dsts, h_dsts.data(), num_buffers * sizeof(T*), hipMemcpyHostToDevice)); + HIP_CHECK( + hipMemcpy(d_sizes, h_sizes.data(), num_buffers * sizeof(size_t), hipMemcpyHostToDevice)); + + size_t temp_bytes = 0; + HIP_CHECK( + hipcub::DeviceMemcpy::Batched(nullptr, temp_bytes, d_srcs, d_dsts, d_sizes, num_buffers)); + void* d_temp = nullptr; + HIP_CHECK(hipMalloc(&d_temp, temp_bytes)); + + HIP_CHECK( + hipcub::DeviceMemcpy::Batched(d_temp, temp_bytes, d_srcs, d_dsts, d_sizes, num_buffers)); + + std::vector h_out(h_in.size()); + HIP_CHECK(hipMemcpy(h_out.data(), d_out, total_bytes, hipMemcpyDeviceToHost)); + EXPECT_TRUE(std::equal(h_in.begin(), + h_in.end(), + h_out.begin(), + [](const PackedPair& x, const PackedPair& y) + { return x.a == y.a && x.b == y.b; })); + + HIP_CHECK(hipFree(d_temp)); + HIP_CHECK(hipFree(d_sizes)); + HIP_CHECK(hipFree(d_dsts)); + HIP_CHECK(hipFree(d_srcs)); + HIP_CHECK(hipFree(d_out)); + HIP_CHECK(hipFree(d_in)); +} diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_merge.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_merge.cpp index c04f35afa759..d07aa79d3537 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_merge.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_merge.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2025-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -24,7 +24,6 @@ // hipcub API #include -#include #include "identity_iterator.hpp" #include "test_utils_data_generation.hpp" @@ -469,11 +468,16 @@ std::vector> get_large_sizes() TEST(HipcubDeviceMerge, MergeLargeSizeIterators) { + +#if defined(_WIN32) + GTEST_SKIP() << "Windows AMD HIP cannot allocate >= 4 GiB buffers."; +#endif + int device_id = test_common_utils::obtain_device_from_ctest(); SCOPED_TRACE(testing::Message() << "with device_id = " << device_id); HIP_CHECK(hipSetDevice(device_id)); - using key_type = int; + using key_type = _HIPCUB_STD::int64_t; using compare_function = test_utils::less; hipStream_t stream = 0; // default @@ -496,9 +500,9 @@ TEST(HipcubDeviceMerge, MergeLargeSizeIterators) compare_function compare_op; // Generate data - const auto input1 = rocprim::counting_iterator(key_type{0}); + const auto input1 = test_utils::counting_iterator(key_type{0}); const auto input2 - = rocprim::counting_iterator(key_type{static_cast(size1)}); + = test_utils::counting_iterator(key_type{static_cast(size1)}); std::vector vec_input1(size1); std::vector vec_input2(size2); std::iota(vec_input1.begin(), vec_input1.end(), 0); diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_radix_sort.cpp.in b/projects/hipcub/test/hipcub/test_hipcub_device_radix_sort.cpp.in index 94daf870e630..2731de1cbc46 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_radix_sort.cpp.in +++ b/projects/hipcub/test/hipcub/test_hipcub_device_radix_sort.cpp.in @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2022-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2022-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -50,7 +50,7 @@ #endif #if HIPCUB_TEST_TYPE_SLICE == 0 -#if HIPCUB_IS_INT128_ENABLED +#if _CCCL_HAS_INT128() INSTANTIATE(params<__uint128_t, __uint128_t, true >) INSTANTIATE(params<__int128_t, __int128_t, true >) #endif diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_radix_sort.hpp b/projects/hipcub/test/hipcub/test_hipcub_device_radix_sort.hpp index 9ce9c64e0f1e..a8508b3087a5 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_radix_sort.hpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_radix_sort.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -82,9 +82,8 @@ class HipcubDeviceRadixSort : public ::testing::Test TYPED_TEST_SUITE_P(HipcubDeviceRadixSort); template -auto generate_key_input(size_t size, unsigned int seed_value) HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH - -> std::enable_if_t::CATEGORY == hipcub::FLOATING_POINT, - std::vector> HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP +auto generate_key_input(size_t size, unsigned int seed_value) + -> std::enable_if_t<_HIPCUB_STD::is_floating_point_v, std::vector> { auto result = test_utils::get_random_data(size, test_utils::numeric_limits::min(), @@ -95,9 +94,8 @@ auto generate_key_input(size_t size, unsigned int seed_value) HIPCUB_CLANG_SUPPR } template -auto generate_key_input(size_t size, unsigned int seed_value) HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH - -> std::enable_if_t::CATEGORY != hipcub::FLOATING_POINT, - std::vector> HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP +auto generate_key_input(size_t size, unsigned int seed_value) + -> std::enable_if_t, std::vector> { using inner_t = typename test_utils::inner_type::type; return test_utils::get_random_data(size, @@ -1229,12 +1227,18 @@ inline void sort_keys_over_4g() SCOPED_TRACE(testing::Message() << "with device_id= " << device_id); HIP_CHECK(hipSetDevice(device_id)); - using key_type = uint8_t; + using key_type = uint32_t; constexpr unsigned int start_bit = 0; constexpr unsigned int end_bit = 8ull * sizeof(key_type); constexpr hipStream_t stream = 0; - constexpr size_t size = (1ull << 32) + 32; - constexpr size_t number_of_possible_keys = 1ull << (8ull * sizeof(key_type)); + + constexpr size_t total_bytes = (1ull << 32) + 32; + static_assert(total_bytes > (1ull << 32), "must be over 4 GiB"); + static_assert(total_bytes % sizeof(key_type) == 0, + "total_bytes must be divisible by sizeof(key_type)"); + + constexpr size_t size = total_bytes / sizeof(key_type); + constexpr size_t number_of_possible_keys = 1ull << (8ull * sizeof(key_type)); assert(std::is_unsigned::value); hipDeviceProp_t dev_prop; HIP_CHECK(hipGetDeviceProperties(&dev_prop, device_id)); @@ -1265,8 +1269,8 @@ inline void sort_keys_over_4g() std::vector keys_input = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), seed_value); //generate histogram of the randomly generated values @@ -1320,7 +1324,7 @@ inline void sort_keys_over_4g() hipMemcpyDeviceToHost)); size_t counter = 0; - for(size_t i = 0; i <= std::numeric_limits::max(); ++i) + for(size_t i = 0; i <= _HIPCUB_STD::numeric_limits::max(); ++i) { for(size_t j = 0; j < histogram[i]; ++j) { @@ -1358,6 +1362,14 @@ inline void sort_keys_large_sizes() { SCOPED_TRACE(testing::Message() << "with size = " << size); + // Avoid sizes the CUB backend can't handle +#ifdef __HIP_PLATFORM_NVIDIA__ + if(size > static_cast(::cuda::std::numeric_limits::max())) + { + continue; + } +#endif // __HIP_PLATFORM_NVIDIA__ + // Generate data std::vector keys_input; try @@ -1372,6 +1384,10 @@ inline void sort_keys_large_sizes() key_type* d_keys; HIP_CHECK_MEMORY(test_common_utils::hipMallocHelper(&d_keys, size * sizeof(key_type))); + + key_type* d_keys_out; + HIP_CHECK_MEMORY(test_common_utils::hipMallocHelper(&d_keys_out, size * sizeof(key_type))); + HIP_CHECK( hipMemcpy(d_keys, keys_input.data(), size * sizeof(key_type), hipMemcpyHostToDevice)); @@ -1380,7 +1396,7 @@ inline void sort_keys_large_sizes() HIP_CHECK(invoke_sort_keys(d_temporary_storage, temporary_storage_bytes, d_keys, - d_keys, + d_keys_out, size, start_bit, end_bit, @@ -1394,7 +1410,7 @@ inline void sort_keys_large_sizes() HIP_CHECK(invoke_sort_keys(d_temporary_storage, temporary_storage_bytes, d_keys, - d_keys, + d_keys_out, size, start_bit, end_bit, @@ -1403,23 +1419,16 @@ inline void sort_keys_large_sizes() HIP_CHECK(hipFree(d_temporary_storage)); std::vector keys_output(size); - try - { - keys_output.resize(size); - } - catch(const std::bad_alloc& e) - { - HIP_CHECK(hipFree(d_keys)); - continue; - } - - HIP_CHECK( - hipMemcpy(keys_output.data(), d_keys, size * sizeof(key_type), hipMemcpyDeviceToHost)); + HIP_CHECK(hipMemcpy(keys_output.data(), + d_keys_out, + size * sizeof(key_type), + hipMemcpyDeviceToHost)); + HIP_CHECK(hipFree(d_keys_out)); HIP_CHECK(hipFree(d_keys)); // Check if output values are as expected - const size_t unique_keys = size_t(std::numeric_limits::max()) + 1; + const size_t unique_keys = size_t(_HIPCUB_STD::numeric_limits::max()) + 1; const size_t segment_length = test_utils::ceiling_div(size, unique_keys); const size_t full_segments = size % unique_keys == 0 ? unique_keys : size % unique_keys; for(size_t i = 0; i < size; i += 4321) diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_reduce.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_reduce.cpp index 2866cdd76d0a..41e3b64d51b6 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_reduce.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_reduce.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -29,7 +29,6 @@ // hipcub API #include -#include // Params for tests template @@ -62,17 +61,39 @@ using HipcubDeviceReduceTestsParams = ::testing::Types< DeviceReduceParams, DeviceReduceParams, DeviceReduceParams, - DeviceReduceParams + DeviceReduceParams, + DeviceReduceParams, #ifdef __HIP_PLATFORM_AMD__ - , - DeviceReduceParams, // Doesn't work on NVIDIA / CUB - DeviceReduceParams, // Doesn't work on NVIDIA / CUB - DeviceReduceParams, test_utils::custom_test_type>, - DeviceReduceParams, test_utils::custom_test_type> + DeviceReduceParams, #endif - >; + DeviceReduceParams, test_utils::custom_test_type>, + DeviceReduceParams, test_utils::custom_test_type>>; + +// Device numeric_limits for custom_test_type +_HIPCUB_STD_NAMESPACE_BEGIN + +template +struct numeric_limits> +{ + static constexpr bool is_specialized = true; + + static constexpr test_utils::custom_test_type min() noexcept + { + return {_HIPCUB_STD::numeric_limits::min(), _HIPCUB_STD::numeric_limits::min()}; + } + + static constexpr test_utils::custom_test_type lowest() noexcept + { + return {_HIPCUB_STD::numeric_limits::lowest(), _HIPCUB_STD::numeric_limits::lowest()}; + } + + static constexpr test_utils::custom_test_type max() noexcept + { + return {_HIPCUB_STD::numeric_limits::max(), _HIPCUB_STD::numeric_limits::max()}; + } +}; + +_HIPCUB_STD_NAMESPACE_END TYPED_TEST_SUITE(HipcubDeviceReduceTests, HipcubDeviceReduceTestsParams); @@ -123,11 +144,12 @@ TYPED_TEST(HipcubDeviceReduceTests, ReduceSum) HIP_CHECK(hipDeviceSynchronize()); // Calculate expected results on host using the same accumulator type than on device - using Sum = - typename AlgebraicSelector::type; // For custom_type_test tests + using Sum = typename AlgebraicSelector:: + type; // For custom_type_test tests using AccumT = hipcub::detail::accumulator_t; Sum sum_op; - AccumT tmp_result = AccumT(0.0f); // hipcub::Sum uses as initial type the output type + AccumT tmp_result + = AccumT(0.0f); // test_utils::plus uses as initial type the output type for(unsigned int i = 0; i < input.size(); i++) { tmp_result = sum_op(tmp_result, input[i]); @@ -138,15 +160,15 @@ TYPED_TEST(HipcubDeviceReduceTests, ReduceSum) size_t temp_storage_size_bytes; void* d_temp_storage = nullptr; // Get size of d_temp_storage - if constexpr(std::is_same::value - || std::is_same::value) + if constexpr(std::is_same_v + || std::is_same_v) { HIP_CHECK(hipcub::DeviceReduce::Reduce(d_temp_storage, temp_storage_size_bytes, d_input, d_output, input.size(), - ExtendedFloatBinOp(), + ExtendedFloatBinOp(), U(0.f), stream)); } @@ -172,15 +194,15 @@ TYPED_TEST(HipcubDeviceReduceTests, ReduceSum) gHelper.startStreamCapture(stream); // Run - if constexpr(std::is_same::value - || std::is_same::value) + if constexpr(std::is_same_v + || std::is_same_v) { HIP_CHECK(hipcub::DeviceReduce::Reduce(d_temp_storage, temp_storage_size_bytes, d_input, d_output, input.size(), - ExtendedFloatBinOp(), + ExtendedFloatBinOp(), U(0.f), stream)); } @@ -266,10 +288,10 @@ TYPED_TEST(HipcubDeviceReduceTests, ReduceMinimum) // Calculate expected results on host using the same accumulator type than on device using Min = typename MinSelector::type; // For custom_type_test tests - using AccumT = hipcub::detail::accumulator_t; + using AccumT = hipcub::detail::accumulator_t; Min min_op; AccumT tmp_result = test_utils::numeric_limits< - AccumT>::max(); // hipcub::Min uses as initial type the input type + AccumT>::max(); // test_utils::minimum uses as initial type the input type for(unsigned int i = 0; i < input.size(); i++) { tmp_result = min_op(tmp_result, input[i]); @@ -380,7 +402,7 @@ TYPED_TEST(HipcubDeviceReduceTests, ReduceMaximum) // Calculate expected results on host using the same accumulator type than on device using Max = typename MaxSelector::type; // For custom_type_test tests - using AccumT = hipcub::detail::accumulator_t; + using AccumT = hipcub::detail::accumulator_t; Max max_op; AccumT tmp_result = test_utils::numeric_limits::min(); for(unsigned int i = 0; i < input.size(); i++) @@ -686,8 +708,8 @@ void test_argminmax2(typename TestFixture::input_type empty_value) using T = typename TestFixture::input_type; using Iterator = typename hipcub::ArgIndexInputIterator; using argidx_type = typename Iterator::value_type; - using extremum_type = typename argidx_type::value_type; - using index_type = typename argidx_type::key_type; + using extremum_type = decltype(std::declval().value); + using index_type = decltype(std::declval().key); DispatchFunction function; @@ -886,13 +908,15 @@ void test_argminmax_allinf(TypeParam value, TypeParam empty_value) if(size > 0) { // all +/- infinity should produce +/- infinity - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output[0].key, 0)); + ASSERT_NO_FATAL_FAILURE( + test_utils::assert_eq(output[0].key, static_cast(0))); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output[0].value, value)); } else { // empty input should produce a special value - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output[0].key, 1)); + ASSERT_NO_FATAL_FAILURE( + test_utils::assert_eq(output[0].key, static_cast(1))); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output[0].value, empty_value)); } } @@ -952,13 +976,15 @@ void test_argminmax_extremum(TypeParam value, TypeParam empty_value) if(size > 0) { // all +/- infinity should produce +/- infinity - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output[0].key, 0)); + ASSERT_NO_FATAL_FAILURE( + test_utils::assert_eq(output[0].key, static_cast(0))); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output[0].value, value)); } else { // empty input should produce a special value - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output[0].key, 1)); + ASSERT_NO_FATAL_FAILURE( + test_utils::assert_eq(output[0].key, static_cast(1))); ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(output[0].value, empty_value)); } } @@ -1050,14 +1076,14 @@ TYPED_TEST(HipcubDeviceReduceTests, TransformReduce) hipMemcpy(d_input, input.data(), input.size() * sizeof(T), hipMemcpyHostToDevice)); // Calculate expected results on host using the same accumulator type than on device - using Sum = - typename AlgebraicSelector::type; // For custom_type_test tests + using Sum = typename AlgebraicSelector:: + type; // For custom_type_test tests using AccumT = hipcub::detail::accumulator_t; Sum reduction_op; TestTransformOp transform_op; const U init(10); - AccumT tmp_result = init; // hipcub::Sum uses as initial type the output type + AccumT tmp_result = init; // test_utils::plus uses as initial type the output type for(size_t i = 0; i < input.size(); ++i) { tmp_result = reduction_op(tmp_result, transform_op(input[i])); @@ -1160,7 +1186,7 @@ TYPED_TEST(HipcubDeviceReduceLargeIndicesTests, LargeIndices) using T = typename TestFixture::input_type; using U = typename TestFixture::output_type; - using IteratorType = rocprim::constant_iterator; + using IteratorType = test_utils::constant_iterator; const std::vector exponents = {30, 31, 32, 33, 34}; for(auto exponent : exponents) { @@ -1219,7 +1245,7 @@ TYPED_TEST(HipcubDeviceReduceLargeIndicesTests, LargeIndices) HIP_CHECK(hipDeviceSynchronize()); // Check if output values are as expected - const std::size_t result = output[0]; + const size_t result = output[0]; ASSERT_EQ(result, size); HIP_CHECK(hipFree(d_output)); diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_reduce_by_key.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_reduce_by_key.cpp index 33f86c435600..901d8f034b7d 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_reduce_by_key.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_reduce_by_key.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -52,23 +52,23 @@ class HipcubDeviceReduceByKey : public ::testing::Test { }; using Params = ::testing::Types< - params, - params, - params, - params, - params, - params, - params, - params, - params, - params, - params, - params, - params, + params, + params, + params, + params, + params, + params, + params, + params, + params, + params, + params, + params, + params, // Sum for half and bfloat will result in values too big due to limited range. - params, - params, - params>; + params, + params, + params>; TYPED_TEST_SUITE(HipcubDeviceReduceByKey, Params); @@ -89,7 +89,7 @@ TYPED_TEST(HipcubDeviceReduceByKey, ReduceByKey) std::uniform_int_distribution>>::type; reduce_op_type reduce_op; - hipcub::Equality key_compare_op; + test_utils::equal key_compare_op; hipStream_t stream = 0; // default if(TestFixture::params::use_graphs) @@ -136,7 +136,7 @@ TYPED_TEST(HipcubDeviceReduceByKey, ReduceByKey) const size_t key_count = key_count_dis(gen); current_key += key_delta_dis(gen); - const size_t end = std::min(size, offset + key_count); + const size_t end = _HIPCUB_STD::min(size, offset + key_count); for(size_t i = offset; i < end; i++) { keys_input[i] = test_utils::convert_to_device(current_key); diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_run_length_encode.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_run_length_encode.cpp index 797676b286c4..316a00943a16 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_run_length_encode.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_run_length_encode.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -123,7 +123,7 @@ TYPED_TEST(HipcubDeviceRunLengthEncode, Encode) size_t key_count = key_count_dis(gen); current_key += key_delta_dis(gen); - const size_t end = std::min(size, offset + key_count); + const size_t end = _HIPCUB_STD::min(size, offset + key_count); key_count = end - offset; for(size_t i = offset; i < end; i++) { @@ -296,7 +296,7 @@ TYPED_TEST(HipcubDeviceRunLengthEncode, NonTrivialRuns) } current_key += key_delta_dis(gen); - const size_t end = std::min(size, offset + key_count); + const size_t end = _HIPCUB_STD::min(size, offset + key_count); key_count = end - offset; for(size_t i = offset; i < end; i++) { @@ -321,12 +321,12 @@ TYPED_TEST(HipcubDeviceRunLengthEncode, NonTrivialRuns) offset_type* d_offsets_output; count_type* d_counts_output; count_type* d_runs_count_output; - HIP_CHECK(test_common_utils::hipMallocHelper(&d_offsets_output, - std::max(1, runs_count_expected) - * sizeof(offset_type))); - HIP_CHECK(test_common_utils::hipMallocHelper(&d_counts_output, - std::max(1, runs_count_expected) - * sizeof(count_type))); + HIP_CHECK(test_common_utils::hipMallocHelper( + &d_offsets_output, + _HIPCUB_STD::max(1, runs_count_expected) * sizeof(offset_type))); + HIP_CHECK(test_common_utils::hipMallocHelper( + &d_counts_output, + _HIPCUB_STD::max(1, runs_count_expected) * sizeof(count_type))); HIP_CHECK(test_common_utils::hipMallocHelper(&d_runs_count_output, sizeof(count_type))); size_t temporary_storage_bytes = 0; diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_scan.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_scan.cpp index ee585fd590db..92883f15c023 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_scan.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_scan.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -24,10 +24,6 @@ // hipcub API #include -#include -#include -#include -#include #include "single_index_iterator.hpp" #include "test_utils_bfloat16.hpp" @@ -36,7 +32,7 @@ // Params for tests template struct DeviceScanParams @@ -65,23 +61,22 @@ class HipcubDeviceScanTests : public ::testing::Test static constexpr bool use_graphs = Params::use_graphs; }; -using HipcubDeviceScanTestsParams - = ::testing::Types, - DeviceScanParams, - DeviceScanParams, - DeviceScanParams, - DeviceScanParams, - DeviceScanParams, - DeviceScanParams, - DeviceScanParams>; +using HipcubDeviceScanTestsParams = ::testing::Types< + DeviceScanParams, + DeviceScanParams, + DeviceScanParams, + DeviceScanParams, + DeviceScanParams, + DeviceScanParams, + DeviceScanParams, + DeviceScanParams>; // use float for accumulation of bfloat16 and half inputs if operator is plus template struct accum_type { - static constexpr bool is_low_precision - = std::is_same::value - || std::is_same::value; + static constexpr bool is_low_precision = std::is_same_v + || std::is_same_v; static constexpr bool is_add = test_utils::is_add_operator::value; using type = typename std::conditional_t; }; @@ -98,14 +93,15 @@ std::vector std::default_random_engine prng(seed_value); std::uniform_int_distribution segment_length_distribution(max_segment_length); - std::uniform_int_distribution key_distribution(std::numeric_limits::max()); + std::uniform_int_distribution key_distribution(_HIPCUB_STD::numeric_limits::max()); std::vector keys(size); size_t keys_start_index = 0; while(keys_start_index < size) { const size_t new_segment_length = segment_length_distribution(prng); - const size_t new_segment_end = std::min(size, keys_start_index + new_segment_length); + const size_t new_segment_end + = _HIPCUB_STD::min(size, keys_start_index + new_segment_length); const T key = key_distribution(prng); std::fill(std::next(keys.begin(), keys_start_index), std::next(keys.begin(), new_segment_end), @@ -121,7 +117,7 @@ TYPED_TEST(HipcubDeviceScanTests, AccumulatorTypeTest) using T = hipcub::detail::accumulator_t; using U = typename TestFixture::input_type; - static_assert(std::is_same::value, "accumulator type mismatch"); + static_assert(std::is_same_v, "accumulator type mismatch"); ASSERT_TRUE(true); } @@ -139,8 +135,8 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScan) // use float as device-side accumulator and double as host-side accumulator using is_add_op = test_utils::is_add_operator; using acc_type = typename accum_type::type; - using IteratorType = rocprim::transform_iterator, acc_type>; - constexpr bool inplace = std::is_same::value && std::is_same::value; + using IteratorType = test_utils::transform_iterator>; + constexpr bool inplace = std::is_same_v && std::is_same_v; // for non-associative operations in inclusive scan // intermediate results use the type of input iterator, then @@ -188,7 +184,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScan) T* d_input; U* d_output; HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - if HIPCUB_IF_CONSTEXPR(!inplace) + if constexpr(!inplace) { HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(U))); } @@ -211,9 +207,9 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScan) auto call = [&](void* d_temp_storage, size_t& temp_storage_size_bytes) { - if HIPCUB_IF_CONSTEXPR(std::is_same::value) + if constexpr(std::is_same_v) { - if HIPCUB_IF_CONSTEXPR(inplace) + if constexpr(inplace) { HIP_CHECK(hipcub::DeviceScan::InclusiveSum(d_temp_storage, temp_storage_size_bytes, @@ -233,7 +229,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScan) } else { - if HIPCUB_IF_CONSTEXPR(inplace) + if constexpr(inplace) { HIP_CHECK(hipcub::DeviceScan::InclusiveScan(d_temp_storage, temp_storage_size_bytes, @@ -281,7 +277,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScan) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - if HIPCUB_IF_CONSTEXPR(inplace) + if constexpr(inplace) { HIP_CHECK(hipMemcpy(output.data(), d_input, @@ -331,8 +327,8 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanInit) // use float as device-side accumulator and double as host-side accumulator using is_add_op = test_utils::is_add_operator; using acc_type = typename accum_type::type; - using IteratorType = rocprim::transform_iterator, acc_type>; - constexpr bool inplace = std::is_same::value && std::is_same::value; + using IteratorType = test_utils::transform_iterator>; + constexpr bool inplace = std::is_same_v && std::is_same_v; // for non-associative operations in inclusive scan // intermediate results use the type of input iterator, then @@ -385,7 +381,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanInit) T* d_input; U* d_output; HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - if HIPCUB_IF_CONSTEXPR(!inplace) + if constexpr(!inplace) { HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(U))); } @@ -412,7 +408,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanInit) auto call = [&](void* d_temp_storage, size_t& temp_storage_size_bytes) { - if HIPCUB_IF_CONSTEXPR(inplace) + if constexpr(inplace) { HIP_CHECK(hipcub::DeviceScan::InclusiveScanInit(d_temp_storage, temp_storage_size_bytes, @@ -462,7 +458,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanInit) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - if HIPCUB_IF_CONSTEXPR(inplace) + if constexpr(inplace) { HIP_CHECK(hipMemcpy(output.data(), d_input, @@ -515,7 +511,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanByKey) // use float as device-side accumulator and double as host-side accumulator using is_add_op = test_utils::is_add_operator; using acc_type = typename accum_type::type; - using IteratorType = rocprim::transform_iterator, acc_type>; + using IteratorType = test_utils::transform_iterator>; // for non-associative operations in inclusive scan // intermediate results use the type of input iterator, then // as all conversions in the tests are to more precise types, @@ -583,7 +579,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanByKey) keys.begin(), expected.begin(), scan_op, - hipcub::Equality()); + test_utils::equal()); // Scan operator: CastOp. hipcub::CastOp op{}; @@ -595,7 +591,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanByKey) size_t temp_storage_size_bytes{}; void* d_temp_storage = nullptr; // Get size of d_temp_storage - if(std::is_same::value) + if(std::is_same_v) { HIP_CHECK(hipcub::DeviceScan::InclusiveSumByKey(d_temp_storage, temp_storage_size_bytes, @@ -603,7 +599,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanByKey) input_iterator, d_output, static_cast(input.size()), - hipcub::Equality(), + test_utils::equal(), stream)); } else @@ -615,7 +611,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanByKey) d_output, scan_op, static_cast(input.size()), - hipcub::Equality(), + test_utils::equal(), stream)); } @@ -631,7 +627,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanByKey) gHelper.startStreamCapture(stream); // Run - if(std::is_same::value) + if(std::is_same_v) { HIP_CHECK(hipcub::DeviceScan::InclusiveSumByKey(d_temp_storage, temp_storage_size_bytes, @@ -639,7 +635,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanByKey) input_iterator, d_output, static_cast(input.size()), - hipcub::Equality(), + test_utils::equal(), stream)); } else @@ -651,7 +647,7 @@ TYPED_TEST(HipcubDeviceScanTests, InclusiveScanByKey) d_output, scan_op, static_cast(input.size()), - hipcub::Equality(), + test_utils::equal(), stream)); } @@ -700,8 +696,8 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScan) // use float as device-side accumulator and double as host-side accumulator using is_add_op = test_utils::is_add_operator; using acc_type = typename accum_type::type; - using IteratorType = rocprim::transform_iterator, acc_type>; - constexpr bool inplace = std::is_same::value && std::is_same::value; + using IteratorType = test_utils::transform_iterator>; + constexpr bool inplace = std::is_same_v && std::is_same_v; // for non-associative operations in inclusive scan // intermediate results use the type of input iterator, then @@ -749,7 +745,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScan) T* d_input; U* d_output; HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - if HIPCUB_IF_CONSTEXPR(!inplace) + if constexpr(!inplace) { HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, output.size() * sizeof(U))); } @@ -763,7 +759,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScan) // Calculate expected results on host std::vector expected(input.size()); const T initial_value - = std::is_same::value + = std::is_same_v ? test_utils::convert_to_device(0) : test_utils::get_random_value(test_utils::convert_to_device(1), test_utils::convert_to_device(100), @@ -782,9 +778,9 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScan) auto call = [&](void* d_temp_storage, size_t& temp_storage_size_bytes) { - if HIPCUB_IF_CONSTEXPR(std::is_same::value) + if constexpr(std::is_same_v) { - if HIPCUB_IF_CONSTEXPR(inplace) + if constexpr(inplace) { HIP_CHECK(hipcub::DeviceScan::ExclusiveSum(d_temp_storage, temp_storage_size_bytes, @@ -804,7 +800,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScan) } else { - if HIPCUB_IF_CONSTEXPR(inplace) + if constexpr(inplace) { HIP_CHECK(hipcub::DeviceScan::ExclusiveScan(d_temp_storage, temp_storage_size_bytes, @@ -854,7 +850,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScan) HIP_CHECK(hipDeviceSynchronize()); // Copy output to host - if HIPCUB_IF_CONSTEXPR(inplace) + if constexpr(inplace) { HIP_CHECK(hipMemcpy(output.data(), d_input, @@ -905,7 +901,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) // use float as device-side accumulator and double as host-side accumulator using is_add_op = test_utils::is_add_operator; using acc_type = typename accum_type::type; - using IteratorType = rocprim::transform_iterator, acc_type>; + using IteratorType = test_utils::transform_iterator>; // for non-associative operations in inclusive scan // intermediate results use the type of input iterator, then // as all conversions in the tests are to more precise types, @@ -957,7 +953,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) test_utils::convert_to_device(10), seed_value); T initial_value = initial_value_vector.front(); - if(std::is_same::value) + if(std::is_same_v) { initial_value = test_utils::convert_to_device(0); } @@ -985,7 +981,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) initial_value, expected.begin(), scan_op, - hipcub::Equality()); + test_utils::equal()); // Scan operator: CastOp. hipcub::CastOp op{}; @@ -997,7 +993,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) size_t temp_storage_size_bytes; void* d_temp_storage = nullptr; // Get size of d_temp_storage - if(std::is_same::value) + if(std::is_same_v) { HIP_CHECK(hipcub::DeviceScan::ExclusiveSumByKey(d_temp_storage, temp_storage_size_bytes, @@ -1005,7 +1001,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) input_iterator, d_output, static_cast(input.size()), - hipcub::Equality(), + test_utils::equal(), stream)); } else @@ -1018,7 +1014,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) scan_op, initial_value, static_cast(input.size()), - hipcub::Equality(), + test_utils::equal(), stream)); } @@ -1034,7 +1030,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) gHelper.startStreamCapture(stream); // Run - if(std::is_same::value) + if(std::is_same_v) { HIP_CHECK(hipcub::DeviceScan::ExclusiveSumByKey(d_temp_storage, temp_storage_size_bytes, @@ -1042,7 +1038,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) input_iterator, d_output, static_cast(input.size()), - hipcub::Equality(), + test_utils::equal(), stream)); } else @@ -1055,7 +1051,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) scan_op, initial_value, static_cast(input.size()), - hipcub::Equality(), + test_utils::equal(), stream)); } @@ -1093,7 +1089,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanByKey) TEST(HipcubDeviceScanTests, LargeIndicesInclusiveScan) { using T = unsigned int; - using InputIterator = rocprim::counting_iterator; + using InputIterator = test_utils::counting_iterator; using OutputIterator = test_utils::single_index_iterator; const size_t size = (1ul << 31) + 1ul; @@ -1103,7 +1099,7 @@ TEST(HipcubDeviceScanTests, LargeIndicesInclusiveScan) unsigned int seed_value = rand(); SCOPED_TRACE(testing::Message() << "with seed= " << seed_value); - // Create rocprim::counting_iterator with random starting point + // Create test_utils::counting_iterator with random starting point InputIterator input_begin(test_utils::get_random_value(0, 200, seed_value)); T* d_output; @@ -1120,7 +1116,7 @@ TEST(HipcubDeviceScanTests, LargeIndicesInclusiveScan) temp_storage_size_bytes, input_begin, output_it, - ::hipcub::Sum(), + test_utils::plus{}, size, stream)); @@ -1136,7 +1132,7 @@ TEST(HipcubDeviceScanTests, LargeIndicesInclusiveScan) temp_storage_size_bytes, input_begin, output_it, - ::hipcub::Sum(), + test_utils::plus{}, size, stream)); HIP_CHECK(hipGetLastError()); @@ -1163,7 +1159,7 @@ TEST(HipcubDeviceScanTests, LargeIndicesInclusiveScan) TEST(HipcubDeviceScanTests, LargeIndicesExclusiveScan) { using T = unsigned int; - using InputIterator = rocprim::counting_iterator; + using InputIterator = test_utils::counting_iterator; using OutputIterator = test_utils::single_index_iterator; const size_t size = (1ul << 31) + 1ul; @@ -1173,7 +1169,7 @@ TEST(HipcubDeviceScanTests, LargeIndicesExclusiveScan) unsigned int seed_value = rand(); SCOPED_TRACE(testing::Message() << "with seed= " << seed_value); - // Create rocprim::counting_iterator with random starting point + // Create test_utils::counting_iterator with random starting point InputIterator input_begin(test_utils::get_random_value(0, 200, seed_value)); T initial_value = test_utils::get_random_value(1, 10, seed_value); @@ -1191,7 +1187,7 @@ TEST(HipcubDeviceScanTests, LargeIndicesExclusiveScan) temp_storage_size_bytes, input_begin, output_it, - ::hipcub::Sum(), + test_utils::plus{}, initial_value, size, stream)); @@ -1208,7 +1204,7 @@ TEST(HipcubDeviceScanTests, LargeIndicesExclusiveScan) temp_storage_size_bytes, input_begin, output_it, - ::hipcub::Sum(), + test_utils::plus{}, initial_value, size, stream)); @@ -1258,7 +1254,7 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanFuture) // use float as device-side accumulator and double as host-side accumulator using is_add_op = test_utils::is_add_operator; using acc_type = typename accum_type::type; - using IteratorType = rocprim::transform_iterator, acc_type>; + using IteratorType = test_utils::transform_iterator>; // for non-associative operations in inclusive scan // intermediate results use the type of input iterator, then // as all conversions in the tests are to more precise types, @@ -1336,12 +1332,11 @@ TYPED_TEST(HipcubDeviceScanTests, ExclusiveScanFuture) const auto future_initial_value = hipcub::FutureValue{d_initial_value}; // Check the provided aliases to be correct at compile-time - static_assert( - std::is_same::value, - "The futures value type is expected to be U"); + static_assert(std::is_same_v, + "The futures value type is expected to be U"); static_assert( - std::is_same::value, + std::is_same_v, "The futures iterator type is expected to be U*"); // temp storage diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_segmented_radix_sort.hpp b/projects/hipcub/test/hipcub/test_hipcub_device_segmented_radix_sort.hpp index 470410994aed..c70e19a3aaeb 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_segmented_radix_sort.hpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_segmented_radix_sort.hpp @@ -104,11 +104,11 @@ inline void sort_keys() } else { - keys_input - = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input = test_utils::get_random_data( + size, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value + seed_value_addition); } std::vector offsets; @@ -255,11 +255,11 @@ inline void sort_keys_empty_data() } else { - keys_input - = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input = test_utils::get_random_data( + size, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value + seed_value_addition); } std::vector offsets(2); @@ -387,10 +387,11 @@ inline void sort_keys_large_segments() } else { - keys_input = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input = test_utils::get_random_data( + size, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value + seed_value_addition); } std::vector offsets(3); @@ -531,11 +532,11 @@ inline void sort_keys_unspecified_ranges() } else { - keys_input - = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input = test_utils::get_random_data( + size, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value + seed_value_addition); } std::vector begin_offsets; @@ -714,11 +715,11 @@ inline void sort_pairs() } else { - keys_input - = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input = test_utils::get_random_data( + size, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value + seed_value_addition); } std::vector offsets; @@ -911,11 +912,11 @@ inline void sort_pairs_unspecified_ranges() } else { - keys_input - = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input = test_utils::get_random_data( + size, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value + seed_value_addition); } std::vector values_input(size); @@ -1137,11 +1138,11 @@ inline void sort_keys_double_buffer() } else { - keys_input - = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input = test_utils::get_random_data( + size, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value + seed_value_addition); } std::vector offsets; @@ -1294,11 +1295,11 @@ inline void sort_pairs_double_buffer() } else { - keys_input - = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input = test_utils::get_random_data( + size, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value + seed_value_addition); } std::vector offsets; diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_segmented_reduce.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_segmented_reduce.cpp index 3d5c02575b84..e1e80ffcef38 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_segmented_reduce.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_segmented_reduce.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -28,11 +28,11 @@ // hipcub API #include -#include +#include template, - params1, - params1, - params1, - params1, - params1, - params1, - params1, - params1, - params1>; +using Params1 = ::testing::Types< + params1, + params1, + params1, + params1, + params1, + params1, + params1, + params1, + params1, + params1>; TYPED_TEST_SUITE(HipcubDeviceSegmentedReduceOp, Params1); @@ -125,7 +125,7 @@ TYPED_TEST(HipcubDeviceSegmentedReduceOp, Reduce) const size_t segment_length = segment_length_dis(gen); offsets.push_back(offset); - const size_t end = std::min(size, offset + segment_length); + const size_t end = _HIPCUB_STD::min(size, offset + segment_length); max_segment_length = std::max(max_segment_length, end - offset); result_type aggregate = init; @@ -278,7 +278,7 @@ TYPED_TEST(HipcubDeviceSegmentedReduce, Sum) using input_type = typename TestFixture::params::input_type; using output_type = typename TestFixture::params::output_type; - using reduce_op_type = typename hipcub::Sum; + using reduce_op_type = typename test_utils::plus; using result_type = output_type; using offset_type = unsigned int; @@ -324,7 +324,7 @@ TYPED_TEST(HipcubDeviceSegmentedReduce, Sum) const size_t segment_length = segment_length_dis(gen); offsets.push_back(offset); - const size_t end = std::min(size, offset + segment_length); + const size_t end = _HIPCUB_STD::min(size, offset + segment_length); max_segment_length = std::max(max_segment_length, end - offset); result_type aggregate = init; @@ -435,11 +435,11 @@ TYPED_TEST(HipcubDeviceSegmentedReduce, Min) using input_type = typename TestFixture::params::input_type; using output_type = typename TestFixture::params::output_type; - using reduce_op_type = typename hipcub::Min; + using reduce_op_type = typename test_utils::minimum; using result_type = output_type; using offset_type = unsigned int; - constexpr input_type init = std::numeric_limits::max(); + constexpr input_type init = _HIPCUB_STD::numeric_limits::max(); reduce_op_type reduce_op; std::random_device rd; @@ -481,7 +481,7 @@ TYPED_TEST(HipcubDeviceSegmentedReduce, Min) const size_t segment_length = segment_length_dis(gen); offsets.push_back(offset); - const size_t end = std::min(size, offset + segment_length); + const size_t end = _HIPCUB_STD::min(size, offset + segment_length); max_segment_length = std::max(max_segment_length, end - offset); result_type aggregate = init; @@ -592,11 +592,11 @@ TYPED_TEST(HipcubDeviceSegmentedReduce, Max) using input_type = typename TestFixture::params::input_type; using output_type = typename TestFixture::params::output_type; - using reduce_op_type = typename hipcub::Max; + using reduce_op_type = typename test_utils::maximum; using result_type = output_type; using offset_type = unsigned int; - constexpr input_type init = std::numeric_limits::lowest(); + constexpr input_type init = _HIPCUB_STD::numeric_limits::lowest(); reduce_op_type reduce_op; std::random_device rd; @@ -638,7 +638,7 @@ TYPED_TEST(HipcubDeviceSegmentedReduce, Max) const size_t segment_length = segment_length_dis(gen); offsets.push_back(offset); - const size_t end = std::min(size, offset + segment_length); + const size_t end = _HIPCUB_STD::min(size, offset + segment_length); max_segment_length = std::max(max_segment_length, end - offset); result_type aggregate = init; @@ -752,14 +752,14 @@ TYPED_TEST(HipcubDeviceSegmentedReduce, Max) struct ArgMinDispatch { template - auto operator()(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream) const + auto operator()(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream) const { return hipcub::DeviceSegmentedReduce::ArgMin(d_temp_storage, temp_storage_bytes, @@ -775,14 +775,14 @@ struct ArgMinDispatch struct ArgMaxDispatch { template - auto operator()(void* d_temp_storage, - size_t& temp_storage_bytes, - InputIteratorT d_in, - OutputIteratorT d_out, - int num_segments, - OffsetIteratorT d_begin_offsets, - OffsetIteratorT d_end_offsets, - hipStream_t stream) const + auto operator()(void* d_temp_storage, + size_t& temp_storage_bytes, + InputIteratorT d_in, + OutputIteratorT d_out, + _HIPCUB_STD::int64_t num_segments, + OffsetIteratorT d_begin_offsets, + OffsetIteratorT d_end_offsets, + hipStream_t stream) const { return hipcub::DeviceSegmentedReduce::ArgMax(d_temp_storage, temp_storage_bytes, @@ -848,7 +848,7 @@ void test_argminmax(typename TestFixture::params::input_type empty_value) offsets.push_back(offset); Iterator x(&values_input[offset]); - const size_t end = std::min(size, offset + segment_length); + const size_t end = _HIPCUB_STD::min(size, offset + segment_length); max_segment_length = std::max(max_segment_length, end - offset); if(offset < end) { @@ -1144,8 +1144,8 @@ TEST(HipcubDeviceSegmentedReduceLargeIndicesTests, LargeIndices) using T = size_t; using input_type = T; using output_type = T; - using IteratorType = rocprim::counting_iterator; - using reduce_op_type = typename hipcub::Sum; + using IteratorType = test_utils::counting_iterator; + using reduce_op_type = typename test_utils::plus; using offset_type = T; const input_type init = input_type(0); @@ -1186,7 +1186,7 @@ TEST(HipcubDeviceSegmentedReduceLargeIndicesTests, LargeIndices) const size_t segment_length = segment_length_dis(gen); offsets.push_back(offset); - const offset_type end = std::min(size, offset + segment_length); + const offset_type end = _HIPCUB_STD::min(size, offset + segment_length); output_type aggregate = init; aggregate = reduce_op(aggregate, gauss_sum(end) - gauss_sum(offset)); aggregates_expected.push_back(aggregate); diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_segmented_sort.hpp b/projects/hipcub/test/hipcub/test_hipcub_device_segmented_sort.hpp index b52f4290bdc6..4efa95241798 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_segmented_sort.hpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_segmented_sort.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -109,10 +109,11 @@ inline void generate_input_data(std::vector& keys_input, } else { - keys_input = test_utils::get_random_data(size, - std::numeric_limits::min(), - std::numeric_limits::max(), - seed_value + seed_value_addition); + keys_input + = test_utils::get_random_data(size, + _HIPCUB_STD::numeric_limits::min(), + _HIPCUB_STD::numeric_limits::max(), + seed_value + seed_value_addition); } offsets.clear(); diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_select.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_select.cpp index aca4048a79e1..e999514554da 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_device_select.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_device_select.cpp @@ -24,8 +24,6 @@ // hipcub API #include -#include -#include #include "single_index_iterator.hpp" #include "test_utils_bfloat16.hpp" @@ -75,7 +73,7 @@ TYPED_TEST(HipcubDeviceSelectTests, Flagged) using U = typename TestFixture::output_type; using F = typename TestFixture::flag_type; - constexpr bool inplace = std::is_same::value; + constexpr bool inplace = std::is_same_v; hipStream_t stream = 0; // default if(TestFixture::use_graphs) @@ -111,7 +109,7 @@ TYPED_TEST(HipcubDeviceSelectTests, Flagged) unsigned int* d_selected_count_output; HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); HIP_CHECK(test_common_utils::hipMallocHelper(&d_flags, flags.size() * sizeof(F))); - if HIPCUB_IF_CONSTEXPR(!inplace) + if constexpr(!inplace) { HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(U))); } @@ -135,7 +133,7 @@ TYPED_TEST(HipcubDeviceSelectTests, Flagged) auto call = [&](void* d_temp_storage, size_t& temp_storage_size_bytes) { - if HIPCUB_IF_CONSTEXPR(inplace) + if constexpr(inplace) { HIP_CHECK(hipcub::DeviceSelect::Flagged(d_temp_storage, temp_storage_size_bytes, @@ -192,7 +190,7 @@ TYPED_TEST(HipcubDeviceSelectTests, Flagged) // Check if output values are as expected std::vector output(input.size()); - if HIPCUB_IF_CONSTEXPR(inplace) + if constexpr(inplace) { HIP_CHECK(hipMemcpy(output.data(), d_input, @@ -244,8 +242,8 @@ TEST(HipcubDeviceSelectTests, FlagNormalization) for(size_t size : test_utils::get_sizes(seed_value)) { SCOPED_TRACE(testing::Message() << "with size= " << size); - rocprim::counting_iterator d_input(0); - rocprim::counting_iterator d_flags(1); + test_utils::counting_iterator d_input(0); + test_utils::counting_iterator d_flags(1); U* d_output; unsigned int* d_selected_count_output; @@ -337,7 +335,7 @@ TYPED_TEST(HipcubDeviceSelectTests, SelectOp) using T = typename TestFixture::input_type; using U = typename TestFixture::output_type; - constexpr bool inplace = std::is_same::value; + constexpr bool inplace = std::is_same_v; hipStream_t stream = 0; // default if(TestFixture::use_graphs) @@ -369,7 +367,7 @@ TYPED_TEST(HipcubDeviceSelectTests, SelectOp) U* d_output; unsigned int* d_selected_count_output; HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); - if HIPCUB_IF_CONSTEXPR(!inplace) + if constexpr(!inplace) { HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(U))); } @@ -391,7 +389,7 @@ TYPED_TEST(HipcubDeviceSelectTests, SelectOp) auto call = [&](void* d_temp_storage, size_t& temp_storage_size_bytes) { - if HIPCUB_IF_CONSTEXPR(inplace) + if constexpr(inplace) { HIP_CHECK(hipcub::DeviceSelect::If(d_temp_storage, temp_storage_size_bytes, @@ -448,7 +446,7 @@ TYPED_TEST(HipcubDeviceSelectTests, SelectOp) // Check if output values are as expected std::vector output(input.size()); - if HIPCUB_IF_CONSTEXPR(inplace) + if constexpr(inplace) { HIP_CHECK(hipMemcpy(output.data(), d_input, @@ -492,7 +490,7 @@ TYPED_TEST(HipcubDeviceSelectTests, FlaggedIf) using U = typename TestFixture::output_type; using F = typename TestFixture::flag_type; - constexpr bool inplace = std::is_same::value; + constexpr bool inplace = std::is_same_v; hipStream_t stream = 0; // default if(TestFixture::use_graphs) @@ -530,7 +528,7 @@ TYPED_TEST(HipcubDeviceSelectTests, FlaggedIf) unsigned int* d_selected_count_output; HIP_CHECK(test_common_utils::hipMallocHelper(&d_input, input.size() * sizeof(T))); HIP_CHECK(test_common_utils::hipMallocHelper(&d_flags, flags.size() * sizeof(F))); - if HIPCUB_IF_CONSTEXPR(!inplace) + if constexpr(!inplace) { HIP_CHECK(test_common_utils::hipMallocHelper(&d_output, input.size() * sizeof(U))); } @@ -554,7 +552,7 @@ TYPED_TEST(HipcubDeviceSelectTests, FlaggedIf) auto call = [&](void* d_temp_storage, size_t& temp_storage_size_bytes) { - if HIPCUB_IF_CONSTEXPR(inplace) + if constexpr(inplace) { HIP_CHECK(hipcub::DeviceSelect::FlaggedIf(d_temp_storage, temp_storage_size_bytes, @@ -613,7 +611,7 @@ TYPED_TEST(HipcubDeviceSelectTests, FlaggedIf) // Check if output values are as expected std::vector output(input.size()); - if HIPCUB_IF_CONSTEXPR(inplace) + if constexpr(inplace) { HIP_CHECK(hipMemcpy(output.data(), d_input, @@ -691,7 +689,7 @@ TYPED_TEST(HipcubDeviceSelectTests, Unique) test_utils::host_inclusive_scan(input01.begin(), input01.end(), input.begin(), - hipcub::Sum()); + test_utils::plus{}); } // Allocate and copy to device @@ -807,9 +805,11 @@ TEST(HipcubDeviceSelectTests, UniqueDiscardOutputIterator) for(size_t size : test_utils::get_sizes(seed_value)) { SCOPED_TRACE(testing::Message() << "with size= " << size); - rocprim::counting_iterator d_input(0); - rocprim::discard_iterator d_output; - size_t* d_selected_count_output; + test_utils::counting_iterator d_input(0); + + auto d_output = test_utils::make_discard_iterator(); + + size_t* d_selected_count_output; HIP_CHECK(test_common_utils::hipMallocHelper((&d_selected_count_output), sizeof(size_t))); @@ -902,7 +902,7 @@ TEST_P(HipcubDeviceSelectLargeIndicesTests, LargeIndicesSelectOp) #endif // Generate data - rocprim::counting_iterator d_input(0); + test_utils::counting_iterator d_input(0); U* d_output; selected_count_type* d_selected_count_output; selected_count_type expected_output_size = selected_size; @@ -974,7 +974,7 @@ TEST_P(HipcubDeviceSelectLargeIndicesTests, LargeIndicesSelectOp) template, DeviceUniqueByKeyParams, - DeviceUniqueByKeyParams, + DeviceUniqueByKeyParams, DeviceUniqueByKeyParams, DeviceUniqueByKeyParams, test_utils::custom_test_type>, - DeviceUniqueByKeyParams>; + DeviceUniqueByKeyParams>; TYPED_TEST_SUITE(HipcubDeviceUniqueByKeyTests, HipcubDeviceUniqueByKeyTestsParams); @@ -1069,7 +1069,7 @@ TYPED_TEST(HipcubDeviceUniqueByKeyTests, UniqueByKey) test_utils::host_inclusive_scan(input01.begin(), input01.end(), input_keys.begin(), - hipcub::Sum()); + test_utils::plus{}); } const auto input_values @@ -1244,8 +1244,8 @@ TEST(HipcubDeviceUniqueByKeyTests, LargeIndicesUniqueByKey) = (size + TestUniqueEqualityOp::segment - 1) / TestUniqueEqualityOp::segment; const size_t output_index = selected_count - 1; const size_t input_index = output_index * TestUniqueEqualityOp::segment; - rocprim::counting_iterator d_keys_input(0); - rocprim::counting_iterator d_values_input(123); + test_utils::counting_iterator d_keys_input(0); + test_utils::counting_iterator d_values_input(123); key_type* d_keys_output; value_type* d_values_output; HIP_CHECK(test_common_utils::hipMallocHelper(&d_keys_output, sizeof(*d_keys_output))); diff --git a/projects/hipcub/test/hipcub/test_hipcub_device_spmv.cpp b/projects/hipcub/test/hipcub/test_hipcub_device_spmv.cpp deleted file mode 100644 index 5b1459954202..000000000000 --- a/projects/hipcub/test/hipcub/test_hipcub_device_spmv.cpp +++ /dev/null @@ -1,292 +0,0 @@ -// MIT License -// -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -#include "experimental/sparse_matrix.hpp" - -#include -#include - -#include "common_test_header.hpp" -#include "test_utils_assertions.hpp" - -hipcub::CachingDeviceAllocator g_allocator; - -static constexpr float alpha_const = 1.0f; -static constexpr float beta_const = 0.0f; - -// Params for tests -template -struct DeviceSpmvParams -{ - using value_type = Type; - static constexpr int32_t grid_2d = Grid2D; - static constexpr int32_t grid_3d = Grid3D; - static constexpr int32_t wheel = Wheel; - static constexpr int32_t dense = Dense; - static constexpr bool use_graphs = UseGraphs; -}; - -// --------------------------------------------------------- -// Test for scan ops taking single input value -// --------------------------------------------------------- - -template -class HipcubDeviceSpmvTests : public ::testing::Test -{ -public: - using value_type = typename Params::value_type; - static constexpr int32_t grid_2d = Params::grid_2d; - static constexpr int32_t grid_3d = Params::grid_3d; - static constexpr int32_t wheel = Params::wheel; - static constexpr int32_t dense = Params::dense; - static constexpr bool use_graphs = Params::use_graphs; -}; - -using HipcubDeviceSpmvTestsParams = ::testing::Types, - DeviceSpmvParams>; - -template -static void generate_matrix(CooMatrix& coo_matrix, - int32_t grid2d, - int32_t grid3d, - int32_t wheel, - int32_t dense) -{ - if(grid2d > 0) - { - // Generate 2D lattice - coo_matrix.InitGrid2d(grid2d, false); - } - else if(grid3d > 0) - { - // Generate 3D lattice - coo_matrix.InitGrid3d(grid3d, false); - } - else if(wheel > 0) - { - // Generate wheel graph - coo_matrix.InitWheel(wheel); - } - else if(dense > 0) - { -#if 0 - // Generate dense graph - OffsetType size = 1 << 24; // 16M nnz - args.GetCmdLineArgument("size", size); - - OffsetType rows = size / dense; - printf("dense_%d_x_%d, ", rows, dense); fflush(stdout); - coo_matrix.InitDense(rows, dense); -#endif - } -} - -template -void SpmvGold(CsrMatrix& a, - const T* vector_x, - const T* vector_y_in, - T* vector_y_out, - T alpha, - T beta) -{ - for(OffsetType row = 0; row < a.num_rows; ++row) - { - T partial = beta * vector_y_in[row]; - for(OffsetType offset = a.row_offsets[row]; offset < a.row_offsets[row + 1]; ++offset) - { - partial += alpha * a.values[offset] * vector_x[a.column_indices[offset]]; - } - vector_y_out[row] = partial; - } -} - -TYPED_TEST_SUITE(HipcubDeviceSpmvTests, HipcubDeviceSpmvTestsParams); - -TYPED_TEST(HipcubDeviceSpmvTests, Spmv) -{ - int device_id = test_common_utils::obtain_device_from_ctest(); - SCOPED_TRACE(testing::Message() << "with device_id= " << device_id); - HIP_CHECK(hipSetDevice(device_id)); - - using T = typename TestFixture::value_type; - using OffsetType = int32_t; - constexpr int32_t grid_2d = TestFixture::grid_2d; - constexpr int32_t grid_3d = TestFixture::grid_3d; - constexpr int32_t wheel = TestFixture::wheel; - constexpr int32_t dense = TestFixture::dense; - - hipStream_t stream = 0; // default - if(TestFixture::use_graphs) - { - // Default stream does not support hipGraph stream capture, so create one - HIP_CHECK(hipStreamCreateWithFlags(&stream, hipStreamNonBlocking)); - } - - CooMatrix coo_matrix; - generate_matrix(coo_matrix, grid_2d, grid_3d, wheel, dense); - - // Convert to CSR - CsrMatrix csr_matrix; - csr_matrix.FromCoo(coo_matrix); - - // Allocate input and output vectors - T* vector_x = new T[csr_matrix.num_cols]; - T* vector_y_in = new T[csr_matrix.num_rows]; - T* vector_y_out = new T[csr_matrix.num_rows]; - - for(int col = 0; col < csr_matrix.num_cols; ++col) - vector_x[col] = 1.0; - - for(int row = 0; row < csr_matrix.num_rows; ++row) - vector_y_in[row] = 1.0; - - // Compute reference answer - SpmvGold(csr_matrix, vector_x, vector_y_in, vector_y_out, alpha_const, beta_const); - - HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH - // Allocate and initialize GPU problem - hipcub::DeviceSpmv::SpmvParams params{}; - HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP - - HIP_CHECK( - g_allocator.DeviceAllocate((void**)¶ms.d_values, sizeof(T) * csr_matrix.num_nonzeros)); - HIP_CHECK(g_allocator.DeviceAllocate((void**)¶ms.d_row_end_offsets, - sizeof(OffsetType) * (csr_matrix.num_rows + 1))); - HIP_CHECK(g_allocator.DeviceAllocate((void**)¶ms.d_column_indices, - sizeof(OffsetType) * csr_matrix.num_nonzeros)); - HIP_CHECK( - g_allocator.DeviceAllocate((void**)¶ms.d_vector_x, sizeof(T) * csr_matrix.num_cols)); - HIP_CHECK( - g_allocator.DeviceAllocate((void**)¶ms.d_vector_y, sizeof(T) * csr_matrix.num_rows)); - - params.num_rows = csr_matrix.num_rows; - params.num_cols = csr_matrix.num_cols; - params.num_nonzeros = csr_matrix.num_nonzeros; - params.alpha = alpha_const; - params.beta = beta_const; - - HIP_CHECK(hipMemcpy(params.d_values, - csr_matrix.values, - sizeof(T) * csr_matrix.num_nonzeros, - hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(params.d_row_end_offsets, - csr_matrix.row_offsets, - sizeof(OffsetType) * (csr_matrix.num_rows + 1), - hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(params.d_column_indices, - csr_matrix.column_indices, - sizeof(OffsetType) * csr_matrix.num_nonzeros, - hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(params.d_vector_x, - vector_x, - sizeof(T) * csr_matrix.num_cols, - hipMemcpyHostToDevice)); - HIP_CHECK(hipMemcpy(params.d_vector_y, - vector_y_in, - sizeof(T) * csr_matrix.num_rows, - hipMemcpyHostToDevice)); - - // Allocate temporary storage - size_t temp_storage_bytes = 0; - void* d_temp_storage = nullptr; - - // Get amount of temporary storage needed - HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH - HIP_CHECK(hipcub::DeviceSpmv::CsrMV(d_temp_storage, - temp_storage_bytes, - params.d_values, - params.d_row_end_offsets, - params.d_column_indices, - params.d_vector_x, - params.d_vector_y, - params.num_rows, - params.num_cols, - params.num_nonzeros, - stream)); - HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP - - // Allocate - //HIP_CHECK(hipMalloc(&d_temp_storage, temp_storage_bytes); - HIP_CHECK(g_allocator.DeviceAllocate(&d_temp_storage, temp_storage_bytes)); - HIP_CHECK(hipDeviceSynchronize()); - - test_utils::GraphHelper gHelper; - if(TestFixture::use_graphs) - gHelper.startStreamCapture(stream); - - HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH - HIP_CHECK(hipcub::DeviceSpmv::CsrMV(d_temp_storage, - temp_storage_bytes, - params.d_values, - params.d_row_end_offsets, - params.d_column_indices, - params.d_vector_x, - params.d_vector_y, - params.num_rows, - params.num_cols, - params.num_nonzeros, - stream)); - HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP - - if(TestFixture::use_graphs) - gHelper.createAndLaunchGraph(stream); - - HIP_CHECK(hipMemcpy(vector_y_in, - params.d_vector_y, - sizeof(T) * params.num_rows, - hipMemcpyDeviceToHost)); - - HIP_CHECK(hipPeekAtLastError()); - HIP_CHECK(hipDeviceSynchronize()); - - const auto max_row_len = csr_matrix.num_cols * csr_matrix.num_rows; - const float diff = max_row_len * test_utils::precision::value; - - for(int32_t i = 0; i < csr_matrix.num_rows; i++) - { - ASSERT_NO_FATAL_FAILURE(test_utils::assert_near(vector_y_in[i], vector_y_out[i], diff)) - << "where index = " << i; - } - - if(TestFixture::use_graphs) - { - gHelper.cleanupGraphHelper(); - HIP_CHECK(hipStreamDestroy(stream)); - } - - // De-allocate input and output vectors - delete[] vector_x; - delete[] vector_y_in; - delete[] vector_y_out; - - HIP_CHECK(g_allocator.DeviceFree(params.d_values)); - HIP_CHECK(g_allocator.DeviceFree(params.d_row_end_offsets)); - HIP_CHECK(g_allocator.DeviceFree(params.d_column_indices)); - HIP_CHECK(g_allocator.DeviceFree(params.d_vector_x)); - HIP_CHECK(g_allocator.DeviceFree(params.d_vector_y)); - HIP_CHECK(g_allocator.DeviceFree(d_temp_storage)); -} diff --git a/projects/hipcub/test/hipcub/test_hipcub_grid.cpp b/projects/hipcub/test/hipcub/test_hipcub_grid.cpp index b7ff8715c984..e9f38a585091 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_grid.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_grid.cpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2019-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2019-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -30,82 +30,10 @@ #include "common_test_header.hpp" #include -#include -#include #include #include -#if defined(__HIP_PLATFORM_NVIDIA__) -_CCCL_SUPPRESS_DEPRECATED_PUSH -#else -HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH -#endif -__global__ -void KernelGridBarrier(hipcub::GridBarrier global_barrier, int iterations) -#if defined(__HIP_PLATFORM_NVIDIA__) - _CCCL_SUPPRESS_DEPRECATED_POP -#else - HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP -#endif -{ - for (int i = 0; i < iterations; i++) - { - global_barrier.Sync(); - } -} - -TEST(HipcubGridTests, GridBarrier) -{ - int device_id = test_common_utils::obtain_device_from_ctest(); - SCOPED_TRACE(testing::Message() << "with device_id= " << device_id); - HIP_CHECK(hipSetDevice(device_id)); - - constexpr int32_t block_size = 256; - // NOTE increasing iterations will cause huge latency for tests - constexpr int32_t iterations = 3; - int32_t grid_size = -1; - - int32_t sm_count; - int32_t max_block_threads; - int32_t max_sm_occupancy; - - HIP_CHECK(hipDeviceGetAttribute(&sm_count, hipDeviceAttributeMultiprocessorCount, device_id)); - HIP_CHECK(hipDeviceGetAttribute(&max_block_threads, hipDeviceAttributeMaxThreadsPerBlock, device_id)); - - HIP_CHECK(hipOccupancyMaxActiveBlocksPerMultiprocessor( - &max_sm_occupancy, - KernelGridBarrier, - HIPCUB_HOST_WARP_THREADS, - 0)); - - int32_t occupancy = std::min((max_block_threads / block_size), max_sm_occupancy); - - if (grid_size == -1) - { - grid_size = occupancy * sm_count; - } - else - { - occupancy = grid_size / sm_count; - } -#if defined(__HIP_PLATFORM_NVIDIA__) - _CCCL_SUPPRESS_DEPRECATED_PUSH -#else - HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH -#endif - hipcub::GridBarrierLifetime global_barrier; -#if defined(__HIP_PLATFORM_NVIDIA__) - _CCCL_SUPPRESS_DEPRECATED_POP -#else - HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP -#endif - HIP_CHECK(global_barrier.Setup(grid_size)); - - KernelGridBarrier<<>>(global_barrier, iterations); - HIP_CHECK(hipGetLastError()); -} - template< int32_t BlockSize, class T, @@ -129,7 +57,7 @@ __global__ void KernelGridEvenShare( T value = device_output[index]; - value = breduce_t(temp_storage).Reduce(value, hipcub::Sum()); + value = breduce_t(temp_storage).Reduce(value, test_utils::plus{}); if(hipThreadIdx_x == 0) { device_output_reductions[hipBlockIdx_x] = value; @@ -250,7 +178,8 @@ __global__ void KernelGridQueue( int32_t index = block_tile_index * BlockSize + hipThreadIdx_x; T value = device_output[index]; - value = breduce_t(temp_storage).Reduce(value, hipcub::Sum()); + + value = breduce_t(temp_storage).Reduce(value, test_utils::plus{}); if(hipThreadIdx_x == 0) { diff --git a/projects/hipcub/test/hipcub/test_hipcub_iterators.cpp b/projects/hipcub/test/hipcub/test_hipcub_iterators.cpp index c3ce6e599575..21687e800694 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_iterators.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_iterators.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -27,10 +27,7 @@ #include #include #include -#include -#include #include -#include #include @@ -233,7 +230,7 @@ TYPED_TEST(HipcubIteratorTests, TestConstant) HIP_CHECK(hipSetDevice(device_id)); using T = typename TestFixture::input_type; - using IteratorType = rocprim::constant_iterator; + using IteratorType = test_utils::constant_iterator; constexpr uint32_t array_size = 8; std::vector h_reference(array_size); @@ -260,7 +257,7 @@ TYPED_TEST(HipcubIteratorTests, TestCounting) HIP_CHECK(hipSetDevice(device_id)); using T = typename TestFixture::input_type; - using IteratorType = rocprim::counting_iterator; + using IteratorType = test_utils::counting_iterator; constexpr uint32_t array_size = 8; std::vector h_reference(array_size); @@ -292,7 +289,7 @@ TYPED_TEST(HipcubIteratorTests, TestTransform) using T = typename TestFixture::input_type; using CastT = typename TestFixture::input_type; - using IteratorType = rocprim::transform_iterator, T>; + using IteratorType = test_utils::transform_iterator>; constexpr int TEST_VALUES = 11000; std::vector h_data(TEST_VALUES); @@ -546,11 +543,12 @@ TYPED_TEST(HipcubIteratorTests, TestTexTransform) HIP_CHECK(d_tex_itr.BindTexture(d_data, sizeof(T) * TEST_VALUES)); // Create transform iterator - rocprim::transform_iterator, T> xform_itr(d_tex_itr, + test_utils::transform_iterator> xform_itr(d_tex_itr, op); - iterator_test_function, T>, - T>(xform_itr, h_reference); + iterator_test_function>>( + xform_itr, + h_reference); HIP_CHECK(g_allocator.DeviceFree(d_data)); } } diff --git a/projects/hipcub/test/hipcub/test_hipcub_single_pass_scan_operators.cpp b/projects/hipcub/test/hipcub/test_hipcub_single_pass_scan_operators.cpp index f6f21f26c5c5..7bb09a7ba3b2 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_single_pass_scan_operators.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_single_pass_scan_operators.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -25,7 +25,6 @@ #include #include -#include #include @@ -34,7 +33,7 @@ #include #include -template +template struct custom_key_value_pair_op { using type = hipcub::KeyValuePair; @@ -135,7 +134,7 @@ static void PrefixKernel(TileState tile_state, T* d_input, T* d_output) template, int BlockSize = 64, - typename ScanOp = hipcub::Sum> + typename ScanOp = test_utils::plus> struct SinglePassScanRunner { void run(int num_items, T* d_input, T* d_output) @@ -166,7 +165,7 @@ struct custom_scan_tile_state : hipcub::ScanTileState template, - typename ScanOp = hipcub::Sum> + typename ScanOp = test_utils::plus> struct SinglePassScanParams { using type = T; @@ -272,7 +271,7 @@ static void RunningPrefixKernel(T* d_input, T* d_output) prefix_type prefix(T(), ScanOp{}); -#pragma unroll + _CCCL_PRAGMA_UNROLL_FULL() for(int i = 0; i < num_items; ++i) { T value = d_input[i]; @@ -281,7 +280,7 @@ static void RunningPrefixKernel(T* d_input, T* d_output) } } -template +template struct RunningPrefixRunner { void run(T* d_input, T* d_output) diff --git a/projects/hipcub/test/hipcub/test_hipcub_thread.cpp b/projects/hipcub/test/hipcub/test_hipcub_thread.cpp index d47f85746e6f..0cc347d7aef0 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_thread.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_thread.cpp @@ -1,7 +1,7 @@ /****************************************************************************** * Copyright (c) 2011, Duane Merrill. All rights reserved. * Copyright (c) 2011-2018, NVIDIA CORPORATION. All rights reserved. - * Modifications Copyright (c) 2017-2025, Advanced Micro Devices, Inc. All rights reserved. + * Modifications Copyright (c) 2017-2026, Advanced Micro Devices, Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: @@ -484,8 +484,16 @@ void thread_reduce_kernel(Type* const device_input, Type* device_output) { size_t input_index = (hipBlockIdx_x * hipBlockDim_x + hipThreadIdx_x) * Length; size_t output_index = (hipBlockIdx_x * hipBlockDim_x + hipThreadIdx_x) * Length; - device_output[output_index] - = hipcub::ThreadReduce(&device_input[input_index], sum_op()); + + // Load into a local array + Type values[Length]; +#pragma unroll + for(int i = 0; i < Length; i++) + { + values[i] = device_input[input_index + i]; + } + + device_output[output_index] = hipcub::ThreadReduce(values, sum_op()); } TYPED_TEST(HipcubThreadOperationTests, Reduction) diff --git a/projects/hipcub/test/hipcub/test_hipcub_thread_operators.cpp b/projects/hipcub/test/hipcub/test_hipcub_thread_operators.cpp index b024c5267112..db3b16407e37 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_thread_operators.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_thread_operators.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -24,18 +24,21 @@ #include "test_utils_assertions.hpp" #include "test_utils_data_generation.hpp" +#include "test_utils_functional.hpp" #include "test_utils_thread_operators.hpp" #include #include #include -#include #include #include +#include #include #include #include +#include _HIPCUB_STD_INCLUDE(functional) + template struct ThreadOperatorsParams { @@ -78,217 +81,6 @@ TYPED_TEST_SUITE(HipcubThreadOperatorsTests, ThreadOperatorsParameters); // Commutative operators tests. -/// \brief Shared code for equality/inequality operators. -template -void equality_op_test(ScanOpT op, bool equality) -{ - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) - { - // Generate random input value. - unsigned int seed_value - = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; - SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); - const InputT input_val - = test_utils::get_random_data(1, 1.0f, 100.0f, seed_value)[0]; - - OutputT output_val{}; - - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(op(input_val, input_val), equality)); - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(op(output_val, output_val), equality)); - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(op(output_val, input_val), !equality)); - - output_val = OutputT(input_val); - - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(op(output_val, input_val), equality)); - } -} - -TYPED_TEST(HipcubThreadOperatorsTests, Equality) -{ - using input_type = typename TestFixture::input_type; - using output_type = typename TestFixture::output_type; - - using Equality = typename EqualitySelector::type; - Equality op{}; - - equality_op_test(op, true); -} - -TYPED_TEST(HipcubThreadOperatorsTests, Inequality) -{ - using input_type = typename TestFixture::input_type; - using output_type = typename TestFixture::output_type; - - using Inequality = typename EqualitySelector::type; - Inequality op{}; - - equality_op_test(op, false); -} - -TYPED_TEST(HipcubThreadOperatorsTests, InequalityWrapper) -{ - using input_type = typename TestFixture::input_type; - using output_type = typename TestFixture::output_type; - - using Equality = typename EqualitySelector::type; - Equality wrapped_op{}; - hipcub::InequalityWrapper op{wrapped_op}; - - equality_op_test(op, false); -} - -/// \brief Shared code for algebraic operators. -template -void algebraic_op_test(const InputT input_val, OutputT init_val) -{ - using accum_type = hipcub::detail::accumulator_t; - - ScanOpT op{}; - - accum_type output_val = init_val; - - // Check result. - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(op(init_val, input_val), output_val)); - - // Check return type. - ASSERT_NO_FATAL_FAILURE(test_utils::assert_type(op(init_val, input_val), output_val)); -} - -TYPED_TEST(HipcubThreadOperatorsTests, Sum) -{ - using input_type = typename TestFixture::input_type; - using output_type = typename TestFixture::output_type; - using Sum = typename AlgebraicSelector::type; - - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) - { - // Generate random initial value. - unsigned int seed_value - = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; - SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); - output_type init_val - = test_utils::get_random_data(1, 1.0f, 100.0f, seed_value)[0]; - - algebraic_op_test(input_type{}, init_val); - } -} - -TYPED_TEST(HipcubThreadOperatorsTests, Difference) -{ - using input_type = typename TestFixture::input_type; - using output_type = typename TestFixture::output_type; - using Difference = - typename AlgebraicSelector::type; - - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) - { - // Generate random initial value. - unsigned int seed_value - = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; - SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); - output_type init_val - = test_utils::get_random_data(1, 1.0f, 100.0f, seed_value)[0]; - - algebraic_op_test(input_type{}, init_val); - } -} - -// Division operator is not defined for custom_test_type. -template -class HipcubDivisionOperatorTests : public ::testing::Test -{ -public: - using input_type = typename Params::input_type; - using output_type = typename Params::output_type; -}; - -using DivisionOperatorParameters = ::testing::Types< - ThreadOperatorsParams, - ThreadOperatorsParams, - ThreadOperatorsParams, - ThreadOperatorsParams, - ThreadOperatorsParams, - ThreadOperatorsParams, - ThreadOperatorsParams, - ThreadOperatorsParams, - ThreadOperatorsParams, - ThreadOperatorsParams, - ThreadOperatorsParams, - ThreadOperatorsParams, - ThreadOperatorsParams -#ifdef __HIP_PLATFORM_AMD__ - , - ThreadOperatorsParams, // Doesn't work on NVIDIA / CUB - ThreadOperatorsParams // Doesn't work on NVIDIA / CUB -#endif - >; -TYPED_TEST_SUITE(HipcubDivisionOperatorTests, DivisionOperatorParameters); - -TYPED_TEST(HipcubDivisionOperatorTests, Division) -{ - using input_type = typename TestFixture::input_type; - using output_type = typename TestFixture::output_type; - using Division = typename AlgebraicSelector::type; - - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) - { - // Generate random input value. - unsigned int seed_value - = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; - SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); - input_type input_val - = test_utils::get_random_data(1, 1.0f, 100.0f, seed_value)[0]; - - algebraic_op_test(input_val, output_type{}); - } -} - -/// \brief Shared code for min/max operators. -template -void minmax_op_test(bool is_max) -{ - ScanOpT op{}; - - for(size_t seed_index = 0; seed_index < random_seeds_count + seed_size; seed_index++) - { - // Generate random initial and input values. - unsigned int seed_value - = seed_index < random_seeds_count ? rand() : seeds[seed_index - random_seeds_count]; - SCOPED_TRACE(testing::Message() << "with seed = " << seed_value); - OutputT init_val = test_utils::get_random_data(1, 1.0f, 100.0f, seed_value)[0]; - InputT input_val = test_utils::get_random_data(1, 1.0f, 100.0f, seed_value)[0]; - - AccumT output_val - = is_max ? test_utils::max(init_val, input_val) : test_utils::min(init_val, input_val); - - // Check result. - ASSERT_NO_FATAL_FAILURE(test_utils::assert_eq(op(init_val, input_val), output_val)); - - // Check return type. - ASSERT_NO_FATAL_FAILURE(test_utils::assert_type(op(init_val, input_val), output_val)); - } -} - -TYPED_TEST(HipcubThreadOperatorsTests, Max) -{ - using input_type = typename TestFixture::input_type; - using output_type = typename TestFixture::output_type; - using accum_type = typename std::common_type::type; - using Max = typename MaxSelector::type; - - minmax_op_test(true); -} - -TYPED_TEST(HipcubThreadOperatorsTests, Min) -{ - using input_type = typename TestFixture::input_type; - using output_type = typename TestFixture::output_type; - using accum_type = typename std::common_type::type; - using Min = typename MinSelector::type; - - minmax_op_test(false); -} - /// \brief Shared code for ArgMin/ArgMax operators. template void arg_op_test(bool is_max) @@ -458,8 +250,8 @@ TYPED_TEST(HipcubNCThreadOperatorsTests, SwizzleScanOp) std::iota(h_input.begin(), h_input.end(), static_cast(1)); // Scan function: SwizzleScanOp. - hipcub::Sum sum_op{}; - hipcub::SwizzleScanOp scan_op(sum_op); + test_utils::plus sum_op{}; + hipcub::SwizzleScanOp scan_op(sum_op); // Calculate expected results on host. std::vector h_expected(input_size); @@ -503,15 +295,15 @@ TYPED_TEST(HipcubNCThreadOperatorsTests, ReduceBySegmentOp) } // Reduce and scan operators. - hipcub::Sum sum_op{}; - hipcub::ReduceBySegmentOp op(sum_op); + test_utils::plus sum_op{}; + hipcub::ReduceBySegmentOp op(sum_op); // Calculate expected results on host. std::vector expected{}; pair_type init(0, 0); for(size_t offset = 0; offset < input_size; offset += segment_size) { - const size_t end = std::min(input_size, offset + segment_size); + const size_t end = _HIPCUB_STD::min(input_size, offset + segment_size); pair_type aggregate = init; for(size_t i = offset; i < end; ++i) { @@ -528,7 +320,7 @@ TYPED_TEST(HipcubNCThreadOperatorsTests, ReduceBySegmentOp) std::vector output{}; for(size_t offset = 0; offset < input_size; offset += segment_size) { - const size_t end = std::min(input_size, offset + segment_size); + const size_t end = _HIPCUB_STD::min(input_size, offset + segment_size); pair_type aggregate = init; for(size_t i = offset; i < end; ++i) { @@ -584,8 +376,8 @@ TYPED_TEST(HipcubNCThreadOperatorsTests, ReduceByKeyOp) } // Reduce operators. - hipcub::Sum sum_op; - hipcub::ReduceByKeyOp op{}; + test_utils::plus sum_op; + hipcub::ReduceByKeyOp op{}; // Calculate output on host. std::vector h_output(h_unique_keys); @@ -696,44 +488,6 @@ TYPED_TEST(HipcubNCThreadOperatorsTests, ReduceByKeyOp) } } -TYPED_TEST(HipcubNCThreadOperatorsTests, BinaryFlip) -{ - using input_type = typename TestFixture::input_type; - using output_type = typename TestFixture::output_type; - - const std::vector sizes = get_sizes(); - for(auto input_size : sizes) - { - // Generate data. - std::vector h_input(input_size); - std::iota(h_input.begin(), h_input.end(), static_cast(1)); - - // Scan function: BinaryFlip. - hipcub::Sum sum_op{}; - hipcub::BinaryFlip scan_op(sum_op); - - // Calculate expected results on host. - std::vector h_expected{}; - - // BinaryFlip's () operator is a device function, so cannot be called from the host function - // test_utils::host_inclusive_scan. We do the scan "manually". - output_type accum = h_input[0]; - h_expected.push_back(accum); - for(size_t i = 1; i < input_size; ++i) - { - // The host_inclusive_cast would do: - // - // accum = scan_op(accum, static_cast(h_input[i])); - // - // But for the BinaryFlip this is equivalent to: - accum = sum_op(static_cast(h_input[i]), accum); - h_expected.push_back(accum); - } - - scan_op_test(h_input, h_expected, scan_op, input_size); - } -} - // Unary operators tests. TYPED_TEST(HipcubNCThreadOperatorsTests, CastOp) @@ -741,7 +495,8 @@ TYPED_TEST(HipcubNCThreadOperatorsTests, CastOp) using input_type = typename TestFixture::input_type; using output_type = typename TestFixture::output_type; using IteratorType - = rocprim::transform_iterator, output_type>; + = test_utils::transform_iterator, output_type>; + const std::vector sizes = get_sizes(); for(auto input_size : sizes) { diff --git a/projects/hipcub/test/hipcub/test_hipcub_util_device.cpp b/projects/hipcub/test/hipcub/test_hipcub_util_device.cpp deleted file mode 100644 index 34277a5ecfcb..000000000000 --- a/projects/hipcub/test/hipcub/test_hipcub_util_device.cpp +++ /dev/null @@ -1,133 +0,0 @@ -// MIT License -// -// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in all -// copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -// SOFTWARE. - -#include "common_test_header.hpp" - -// hipcub API -#include - -template -__global__ -void alias_temporaries_kernel(T* data, size_t* temp_storage_bytes) -{ - T* allocations[10]; - size_t allocation_sizes[10] = {1, 2, 3, 5, 8, 13, 21, 34, 55, 89}; - (void) - hipcub::detail::AliasTemporaries(data, *temp_storage_bytes, allocations, allocation_sizes); -} - -TEST(HipcubUtilDevice, AliasTemporariesDevice) -{ - int device_id = test_common_utils::obtain_device_from_ctest(); - SCOPED_TRACE(testing::Message() << "with device_id= " << device_id); - HIP_CHECK(hipSetDevice(device_id)); - - void* data = nullptr; - size_t temp_storage_bytes_host = 0; // Temporary storage on the host - size_t* device_temp_storage_bytes; - HIP_CHECK(test_common_utils::hipMallocHelper(&device_temp_storage_bytes, sizeof(size_t))); - - // First kernel call to determine required temp storage size - alias_temporaries_kernel<<<1, 1, 0, 0>>>(data, device_temp_storage_bytes); - HIP_CHECK(hipDeviceSynchronize()); - HIP_CHECK(hipGetLastError()); - - // Copy the device storage size to host - HIP_CHECK(hipMemcpy(&temp_storage_bytes_host, - device_temp_storage_bytes, - sizeof(size_t), - hipMemcpyDeviceToHost)); - ASSERT_GT(temp_storage_bytes_host, 0U); - - // Allocate the actual data buffer on the device - HIP_CHECK(test_common_utils::hipMallocHelper(&data, temp_storage_bytes_host)); - - // Second kernel call with allocated buffer - alias_temporaries_kernel<<<1, 1, 0, 0>>>(data, device_temp_storage_bytes); - HIP_CHECK(hipDeviceSynchronize()); - HIP_CHECK(hipGetLastError()); - - // Free device memory - HIP_CHECK(hipFree(device_temp_storage_bytes)); - HIP_CHECK(hipFree(data)); -} - -TEST(HipcubUtilDevice, AliasTemporariesHost) -{ - int device_id = test_common_utils::obtain_device_from_ctest(); - SCOPED_TRACE(testing::Message() << "with device_id= " << device_id); - HIP_CHECK(hipSetDevice(device_id)); - - void* data = nullptr; - size_t temp_storage_bytes = 0; - void* allocations[10]; - size_t allocation_sizes[10] = {1, 789, 3, 5, 8, 13, 21, 257, 256, 890}; - - size_t min_size = 0; - for(unsigned int i = 0; i < 10; i++) - { - min_size += allocation_sizes[i]; - } - - // Determine storage size - HIP_CHECK( - hipcub::detail::AliasTemporaries(data, temp_storage_bytes, allocations, allocation_sizes)); - - // Should be larger or equal to the sum of all sizes. - ASSERT_GT(temp_storage_bytes, min_size - 1); - - // Allocate the actual data buffer on the device - HIP_CHECK(test_common_utils::hipMallocHelper(&data, temp_storage_bytes)); - - size_t zero_size = 0; - // Check for error if it does not fit - hipError_t error - = hipcub::detail::AliasTemporaries(data, zero_size, allocations, allocation_sizes); - test_utils::assert_eq(error, hipErrorInvalidValue); - - HIP_CHECK( - hipcub::detail::AliasTemporaries(data, temp_storage_bytes, allocations, allocation_sizes)); - - test_utils::assert_eq(data, allocations[0]); - - for(unsigned int i = 1; i < 10; i++) - { - // The allocations should be in increasing order. - ASSERT_GT(allocations[i], allocations[i - 1]); - size_t current_pointer = (size_t)allocations[i]; - size_t before_pointer = (size_t)allocations[i - 1]; - size_t distance = current_pointer - before_pointer; - - // Check if all pointer have enough space - ASSERT_GT(distance + 1, allocation_sizes[i - 1]); - } - - size_t last_pointer = (size_t)allocations[9]; - size_t start_pointer = (size_t)data; - size_t max_size = start_pointer + temp_storage_bytes; - size_t last_size = max_size - last_pointer; - - // Last size should be equal or larger then the last value in allocation_sizes - ASSERT_GT(last_size + 1, allocation_sizes[9]); - - HIP_CHECK(hipFree(data)); -} diff --git a/projects/hipcub/test/hipcub/test_hipcub_util_ptx.cpp b/projects/hipcub/test/hipcub/test_hipcub_util_ptx.cpp index c583c0b9e39c..26ce3faa6a76 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_util_ptx.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_util_ptx.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -156,9 +156,9 @@ TYPED_TEST(HipcubUtilPtxTests, ShuffleUp) std::vector output(input.size()); auto src_offsets = test_utils::get_random_data( - std::max(1, logical_warp_size / 2), + _HIPCUB_STD::max(1, logical_warp_size / 2), 1U, - std::max(1, logical_warp_size - 1), + _HIPCUB_STD::max(1, logical_warp_size - 1), seed_value + seed_value_addition); T* device_data; @@ -267,9 +267,9 @@ TYPED_TEST(HipcubUtilPtxTests, ShuffleDown) std::vector output(input.size()); auto src_offsets = test_utils::get_random_data( - std::max(1, logical_warp_size / 2), + _HIPCUB_STD::max(1, logical_warp_size / 2), 1U, - std::max(1, logical_warp_size - 1), + _HIPCUB_STD::max(1, logical_warp_size - 1), seed_value + seed_value_addition); T* device_data; @@ -379,10 +379,11 @@ TYPED_TEST(HipcubUtilPtxTests, ShuffleIndex) seed_value); std::vector output(input.size()); - auto src_offsets = test_utils::get_random_data(hardware_warp_size / logical_warp_size, - 0, - std::max(1, logical_warp_size - 1), - seed_value + seed_value_addition); + auto src_offsets + = test_utils::get_random_data(hardware_warp_size / logical_warp_size, + 0, + _HIPCUB_STD::max(1, logical_warp_size - 1), + seed_value + seed_value_addition); // Calculate expected results on host std::vector expected(size, test_utils::convert_to_device(0)); @@ -481,9 +482,9 @@ TEST(HipcubUtilPtxTests, ShuffleUpCustomStruct) } auto src_offsets = test_utils::get_random_data( - std::max(1, logical_warp_size / 2), + _HIPCUB_STD::max(1, logical_warp_size / 2), 1U, - std::max(1, logical_warp_size - 1), + _HIPCUB_STD::max(1, logical_warp_size - 1), seed_value + seed_value_addition); T* device_data; @@ -592,9 +593,9 @@ TEST(HipcubUtilPtxTests, ShuffleUpCustomAlignedStruct) } auto src_offsets = test_utils::get_random_data( - std::max(1, logical_warp_size / 2), + _HIPCUB_STD::max(1, logical_warp_size / 2), 1U, - std::max(1, logical_warp_size - 1), + _HIPCUB_STD::max(1, logical_warp_size - 1), seed_value + seed_value_addition); T* device_data; @@ -670,7 +671,11 @@ __global__ void warp_id_kernel(unsigned int* output) { const unsigned int index = (hipBlockIdx_x * hipBlockDim_x) + hipThreadIdx_x; - output[index] = ::rocprim::warp_id(); +#ifdef __HIP_PLATFORM_NVIDIA__ + output[index] = hipThreadIdx_x / warpSize; +#else + output[index] = ::rocprim::warp_id(); +#endif } TEST(HipcubUtilPtxTests, WarpId) @@ -754,7 +759,11 @@ template HIPCUB_DEVICE std::enable_if_t<(HIPCUB_DEVICE_WARP_THREADS >= LogicalWarpSize), TestStatus> test_warp_mask_pow_two() { +#ifdef __HIP_PLATFORM_NVIDIA__ + const unsigned int logical_warp_id = (hipThreadIdx_x % warpSize) / LogicalWarpSize; +#else const unsigned int logical_warp_id = ::rocprim::lane_id() / LogicalWarpSize; +#endif const uint64_t mask = hipcub::WarpMask(logical_warp_id); const unsigned int warp_start = logical_warp_id * LogicalWarpSize; @@ -795,7 +804,11 @@ template HIPCUB_DEVICE std::enable_if_t<(HIPCUB_DEVICE_WARP_THREADS >= LogicalWarpSize), TestStatus> test_warp_mask_non_pow_two() { +#ifdef __HIP_PLATFORM_NVIDIA__ + const unsigned int logical_warp_id = (hipThreadIdx_x % warpSize) / LogicalWarpSize; +#else const unsigned int logical_warp_id = ::rocprim::lane_id() / LogicalWarpSize; +#endif const uint64_t mask = hipcub::WarpMask(logical_warp_id); for(unsigned int lane = 0; lane < LogicalWarpSize; ++lane) diff --git a/projects/hipcub/test/hipcub/test_hipcub_warp_exchange.cpp b/projects/hipcub/test/hipcub/test_hipcub_warp_exchange.cpp index 30adcdbb5733..25bd35263d5f 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_warp_exchange.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_warp_exchange.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -133,11 +133,7 @@ __device__ auto warp_exchange_test(T* d_input, T* d_output) thread_data[i] = d_input[threadIdx.x * ItemsPerThread + i]; } - using WarpExchangeT = ::hipcub::WarpExchange; + using WarpExchangeT = ::hipcub::WarpExchange; constexpr unsigned warps_in_block = BlockSize / LogicalWarpSize; __shared__ typename WarpExchangeT::TempStorage temp_storage[warps_in_block]; const unsigned warp_id = threadIdx.x / LogicalWarpSize; @@ -222,7 +218,7 @@ std::enable_if_t> run_warp_exch input[i] = test_utils::convert_to_device(i); } std::vector expected; - if(std::is_same::value) + if(std::is_same_v) { expected = input; input = stripe_vector(input, warp_size, items_per_thread); diff --git a/projects/hipcub/test/hipcub/test_hipcub_warp_load.cpp b/projects/hipcub/test/hipcub/test_hipcub_warp_load.cpp index 71cead9b29a8..f35adb743c9c 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_warp_load.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_warp_load.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -232,14 +232,14 @@ TYPED_TEST(HipcubWarpLoadTest, WarpLoadGuarded) SCOPED_TRACE(testing::Message() << "with device_id= " << device_id); HIP_CHECK(hipSetDevice(device_id)); - using T = typename TestFixture::params::type; - constexpr unsigned warp_size = TestFixture::params::warp_size; - constexpr ::hipcub::WarpLoadAlgorithm algorithm = TestFixture::params::algorithm; - constexpr unsigned items_per_thread = 4; - constexpr unsigned block_size = 1024; - constexpr unsigned items_count = items_per_thread * block_size; - constexpr int valid_items = warp_size / 4; - constexpr T oob_default = std::numeric_limits::max(); + using T = typename TestFixture::params::type; + constexpr unsigned warp_size = TestFixture::params::warp_size; + constexpr ::hipcub::WarpLoadAlgorithm algorithm = TestFixture::params::algorithm; + constexpr unsigned items_per_thread = 4; + constexpr unsigned block_size = 1024; + constexpr unsigned items_count = items_per_thread * block_size; + constexpr int valid_items = warp_size / 4; + constexpr T oob_default = _HIPCUB_STD::numeric_limits::max(); SKIP_IF_UNSUPPORTED_WARP_SIZE(warp_size); diff --git a/projects/hipcub/test/hipcub/test_hipcub_warp_merge_sort.cpp b/projects/hipcub/test/hipcub/test_hipcub_warp_merge_sort.cpp index 9055e377558c..c5aab18cf110 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_warp_merge_sort.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_warp_merge_sort.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -28,7 +28,6 @@ #include #include -#include #include #include @@ -99,7 +98,7 @@ __device__ auto sort_keys_full_test(Key* keys, Compare compare_op) __shared__ typename warp_merge_sort::TempStorage storage[warps_per_block]; warp_merge_sort wsort{storage[warp_id]}; - if HIPCUB_IF_CONSTEXPR(Stable) + if constexpr(Stable) { wsort.StableSort(thread_keys, compare_op); } else @@ -157,7 +156,7 @@ __device__ auto sort_keys_values_full_test(Key* keys, Value* values, Compare com __shared__ typename warp_merge_sort::TempStorage storage[warps_per_block]; warp_merge_sort wsort{storage[warp_id]}; - if HIPCUB_IF_CONSTEXPR(Stable) + if constexpr(Stable) { wsort.StableSort(thread_keys, thread_values, compare_op); } else @@ -202,12 +201,26 @@ struct sort_last; template struct sort_last { - static constexpr T value = std::numeric_limits::max(); + static constexpr T value = _HIPCUB_STD::numeric_limits::max(); }; template struct sort_last { - static constexpr T value = std::numeric_limits::lowest(); + static constexpr T value = _HIPCUB_STD::numeric_limits::lowest(); +}; + +template +struct sort_last> +{ + static constexpr test_utils::custom_test_type value = test_utils::custom_test_type( + _HIPCUB_STD::numeric_limits::max(), _HIPCUB_STD::numeric_limits::max()); +}; + +template +struct sort_last> +{ + static constexpr test_utils::custom_test_type value = test_utils::custom_test_type( + _HIPCUB_STD::numeric_limits::lowest(), _HIPCUB_STD::numeric_limits::lowest()); }; template::value; - if HIPCUB_IF_CONSTEXPR(Stable) + if constexpr(Stable) { wsort.StableSort(thread_keys, compare, segment_size, oob_default); } else @@ -311,7 +324,7 @@ __device__ auto sort_keys_values_segmented_test(Key* keys, hipcub::LoadDirectBlocked(flat_tid, values + warp_offset, thread_values, segment_size); const Key oob_default = sort_last::value; - if HIPCUB_IF_CONSTEXPR(Stable) + if constexpr(Stable) { wsort.StableSort(thread_keys, thread_values, compare, segment_size, oob_default); } else @@ -400,15 +413,15 @@ TYPED_TEST(HipcubWarpMergeSort, SortKeysSegmented) auto keys = test_utils::is_floating_point::value ? test_utils::get_random_data( - size, - test_utils::convert_to_device(-1000), - test_utils::convert_to_device(1000), - seed_value) + size, + test_utils::convert_to_device(-1000), + test_utils::convert_to_device(1000), + seed_value) : test_utils::get_random_data( - size, - std::numeric_limits::lowest(), - std::numeric_limits::max(), - seed_value); + size, + _HIPCUB_STD::numeric_limits::lowest(), + _HIPCUB_STD::numeric_limits::max(), + seed_value); const auto segment_sizes = test_utils::get_random_data( num_warps, 0u, max_segment_size, ~seed_value); @@ -513,29 +526,29 @@ TYPED_TEST(HipcubWarpMergeSort, SortKeysValuesSegmented) auto keys = test_utils::is_floating_point::value ? test_utils::get_random_data( - size, - test_utils::convert_to_device(-1000), - test_utils::convert_to_device(1000), - seed_value) + size, + test_utils::convert_to_device(-1000), + test_utils::convert_to_device(1000), + seed_value) : test_utils::get_random_data( - size, - std::numeric_limits::lowest(), - std::numeric_limits::max(), - seed_value); + size, + _HIPCUB_STD::numeric_limits::lowest(), + _HIPCUB_STD::numeric_limits::max(), + seed_value); using value_wrapped_type = typename test_utils::inner_type::type; auto values = test_utils::is_floating_point::value ? test_utils::get_random_data( - size, - test_utils::convert_to_device(-1000), - test_utils::convert_to_device(1000), - seed_value) + size, + test_utils::convert_to_device(-1000), + test_utils::convert_to_device(1000), + seed_value) : test_utils::get_random_data( - size, - std::numeric_limits::lowest(), - std::numeric_limits::max(), - seed_value ^ (seed_value >> 1ul)); + size, + _HIPCUB_STD::numeric_limits::lowest(), + _HIPCUB_STD::numeric_limits::max(), + seed_value ^ (seed_value >> 1ul)); const auto segment_sizes = test_utils::get_random_data( num_warps, 0u, max_segment_size, ~seed_value); @@ -661,15 +674,15 @@ TYPED_TEST(HipcubWarpMergeSort, SortKeys) auto keys = test_utils::is_floating_point::value ? test_utils::get_random_data( - size, - test_utils::convert_to_device(-1000), - test_utils::convert_to_device(1000), - seed_value) + size, + test_utils::convert_to_device(-1000), + test_utils::convert_to_device(1000), + seed_value) : test_utils::get_random_data( - size, - std::numeric_limits::lowest(), - std::numeric_limits::max(), - seed_value); + size, + _HIPCUB_STD::numeric_limits::lowest(), + _HIPCUB_STD::numeric_limits::max(), + seed_value); const auto compare = typename params::compare_function{}; @@ -760,29 +773,29 @@ TYPED_TEST(HipcubWarpMergeSort, SortKeysValues) auto keys = test_utils::is_floating_point::value ? test_utils::get_random_data( - size, - test_utils::convert_to_device(-1000), - test_utils::convert_to_device(1000), - seed_value) + size, + test_utils::convert_to_device(-1000), + test_utils::convert_to_device(1000), + seed_value) : test_utils::get_random_data( - size, - std::numeric_limits::lowest(), - std::numeric_limits::max(), - seed_value); + size, + _HIPCUB_STD::numeric_limits::lowest(), + _HIPCUB_STD::numeric_limits::max(), + seed_value); using value_wrapped_type = typename test_utils::inner_type::type; auto values = test_utils::is_floating_point::value ? test_utils::get_random_data( - size, - test_utils::convert_to_device(-1000), - test_utils::convert_to_device(1000), - seed_value) + size, + test_utils::convert_to_device(-1000), + test_utils::convert_to_device(1000), + seed_value) : test_utils::get_random_data( - size, - std::numeric_limits::lowest(), - std::numeric_limits::max(), - seed_value ^ (seed_value >> 1ul)); + size, + _HIPCUB_STD::numeric_limits::lowest(), + _HIPCUB_STD::numeric_limits::max(), + seed_value ^ (seed_value >> 1ul)); const auto compare = typename params::compare_function{}; diff --git a/projects/hipcub/test/hipcub/test_hipcub_warp_reduce.cpp b/projects/hipcub/test/hipcub/test_hipcub_warp_reduce.cpp index d60cb0333cc7..0f36220cc98f 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_warp_reduce.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_warp_reduce.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -122,7 +122,9 @@ auto warp_reduce_kernel(T* device_input, T* device_output) -> using wreduce_t = hipcub::WarpReduce; __shared__ typename wreduce_t::TempStorage storage[warps_no]; - auto reduce_op = hipcub::Sum(); + + auto reduce_op = test_utils::plus{}; + value = wreduce_t(storage[warp_id]).Reduce(value, reduce_op); if (hipThreadIdx_x % LogicalWarpSize == 0) @@ -280,7 +282,9 @@ auto warp_reduce_valid_kernel(T* device_input, T* device_output, const int valid using wreduce_t = hipcub::WarpReduce; __shared__ typename wreduce_t::TempStorage storage[warps_no]; - auto reduce_op = hipcub::Sum(); + + auto reduce_op = test_utils::plus{}; + value = wreduce_t(storage[warp_id]).Reduce(value, reduce_op, valid); if (hipThreadIdx_x % LogicalWarpSize == 0) @@ -649,7 +653,9 @@ auto tail_segmented_warp_reduce_kernel(T* input, Flag* flags, T* output) -> using wreduce_t = hipcub::WarpReduce; __shared__ typename wreduce_t::TempStorage storage[warps_no]; - auto reduce_op = hipcub::Sum(); + + auto reduce_op = test_utils::plus{}; + value = wreduce_t(storage[warp_id]).TailSegmentedReduce(value, flag, reduce_op); output[index] = value; diff --git a/projects/hipcub/test/hipcub/test_hipcub_warp_scan.cpp b/projects/hipcub/test/hipcub/test_hipcub_warp_scan.cpp index 9a91412211ff..4d801be1bf69 100644 --- a/projects/hipcub/test/hipcub/test_hipcub_warp_scan.cpp +++ b/projects/hipcub/test/hipcub/test_hipcub_warp_scan.cpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -121,7 +121,8 @@ auto warp_inclusive_scan_kernel(T* device_input, T* device_output) using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage[warps_no]; - auto scan_op = hipcub::Sum(); + + auto scan_op = test_utils::plus{}; wscan_t(storage[warp_id]).InclusiveScan(value, value, scan_op); device_output[index] = value; @@ -272,7 +273,7 @@ auto warp_inclusive_scan_initial_value_kernel(T* device_input, T* device_output, using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage[warps_no]; - auto scan_op = hipcub::Sum(); + auto scan_op = test_utils::plus{}; wscan_t(storage[warp_id]).InclusiveScan(value, value, initial_value, scan_op); device_output[index] = value; @@ -445,7 +446,7 @@ auto warp_inclusive_scan_reduce_kernel(T* device_input, __shared__ typename wscan_t::TempStorage storage[warps_no]; if(hipBlockIdx_x%2 == 0) { - auto scan_op = hipcub::Sum(); + auto scan_op = test_utils::plus{}; wscan_t(storage[warp_id]).InclusiveScan(value, value, scan_op, reduction); } else @@ -630,7 +631,8 @@ auto warp_inclusive_scan_reduce_initial_value_kernel(T* device_input, using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage[warps_no]; - wscan_t(storage[warp_id]).InclusiveScan(value, value, initial_value, hipcub::Sum(), reduction); + wscan_t(storage[warp_id]) + .InclusiveScan(value, value, initial_value, test_utils::plus{}, reduction); device_output[index] = value; if((hipThreadIdx_x % LogicalWarpSize) == 0) @@ -817,7 +819,8 @@ auto warp_exclusive_scan_kernel(T* device_input, T* device_output, T init) using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage[warps_no]; - auto scan_op = hipcub::Sum(); + + auto scan_op = test_utils::plus{}; wscan_t(storage[warp_id]).ExclusiveScan(value, value, init, scan_op); device_output[index] = value; @@ -975,7 +978,8 @@ auto warp_exclusive_scan_reduce_kernel(T* device_input, using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage[warps_no]; - auto scan_op = hipcub::Sum(); + + auto scan_op = test_utils::plus{}; wscan_t(storage[warp_id]).ExclusiveScan(value, value, init, scan_op, reduction); device_output[index] = value; @@ -1169,7 +1173,8 @@ auto warp_scan_kernel(T* device_input, using wscan_t = hipcub::WarpScan; __shared__ typename wscan_t::TempStorage storage[warps_no]; - auto scan_op = hipcub::Sum(); + + auto scan_op = test_utils::plus{}; wscan_t(storage[warp_id]).Scan(input, inclusive_output, exclusive_output, init, scan_op); device_inclusive_output[index] = inclusive_output; diff --git a/projects/hipcub/test/hipcub/test_utils.hpp b/projects/hipcub/test/hipcub/test_utils.hpp index 88eac81521a4..fff732ed7647 100644 --- a/projects/hipcub/test/hipcub/test_utils.hpp +++ b/projects/hipcub/test/hipcub/test_utils.hpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -21,18 +21,29 @@ #ifndef HIPCUB_TEST_TEST_UTILS_HPP_ #define HIPCUB_TEST_TEST_UTILS_HPP_ -#ifndef TEST_UTILS_INCLUDE_GAURD +#ifndef TEST_UTILS_INCLUDE_GUARD #error test_utils.hpp must ONLY be included by common_test_header.hpp. Please include common_test_header.hpp instead. #endif // hipCUB API #ifdef __HIP_PLATFORM_AMD__ - #include + #include + #include + #include + #include + #include #elif defined(__HIP_PLATFORM_NVIDIA__) + #include + #include + #include #include - #include + #include + #include + #include #endif +#include + #include "test_utils_assertions.hpp" #include "test_utils_bfloat16.hpp" #include "test_utils_custom_test_types.hpp" @@ -45,6 +56,8 @@ // Seed values #include "test_seed.hpp" +#include +#include #include namespace test_utils @@ -167,7 +180,7 @@ OutputIt host_inclusive_scan_impl( template OutputIt host_inclusive_scan(InputIt first, InputIt last, OutputIt d_first, BinaryOperation op) { - using acc_type = typename std::iterator_traits::value_type; + using acc_type = ::hipcub::detail::it_value_t; return host_inclusive_scan_impl(first, last, d_first, op, acc_type{}); } @@ -178,35 +191,31 @@ OutputIt host_inclusive_scan_init( return host_inclusive_scan_impl(first, last, d_first, op, init_value); } -template::value_type, - test_utils::bfloat16>::value - || std::is_same::value_type, - test_utils::half>::value - || std::is_same::value_type, float>::value, - bool> - = true> +template< + class InputIt, + class OutputIt, + class T, + std::enable_if_t, test_utils::bfloat16> + || std::is_same_v, test_utils::half> + || std::is_same_v, float>, + bool> + = true> OutputIt host_inclusive_scan(InputIt first, InputIt last, OutputIt d_first, test_utils::plus) { using acc_type = double; return host_inclusive_scan_impl(first, last, d_first, test_utils::plus(), acc_type{}); } -template::value_type, - test_utils::bfloat16>::value - || std::is_same::value_type, - test_utils::half>::value - || std::is_same::value_type, float>::value, - bool> - = true> +template< + class InputIt, + class OutputIt, + class InitType, + class T, + std::enable_if_t, test_utils::bfloat16> + || std::is_same_v<::hipcub::detail::it_value_t, test_utils::half> + || std::is_same_v<::hipcub::detail::it_value_t, float>, + bool> + = true> OutputIt host_inclusive_scan_init( InputIt first, InputIt last, OutputIt d_first, InitType init_value, test_utils::plus) { @@ -242,22 +251,20 @@ template OutputIt host_exclusive_scan( InputIt first, InputIt last, T initial_value, OutputIt d_first, BinaryOperation op) { - using acc_type = typename std::iterator_traits::value_type; + using acc_type = ::hipcub::detail::it_value_t; return host_exclusive_scan_impl(first, last, initial_value, d_first, op, acc_type{}); } -template::value_type, - test_utils::bfloat16>::value - || std::is_same::value_type, - test_utils::half>::value - || std::is_same::value_type, float>::value, - bool> - = true> +template< + class InputIt, + class T, + class OutputIt, + class U, + std::enable_if_t, test_utils::bfloat16> + || std::is_same_v<::hipcub::detail::it_value_t, test_utils::half> + || std::is_same_v<::hipcub::detail::it_value_t, float>, + bool> + = true> OutputIt host_exclusive_scan( InputIt first, InputIt last, T initial_value, OutputIt d_first, test_utils::plus) { @@ -321,7 +328,7 @@ OutputIt host_exclusive_scan_by_key(InputIt first, BinaryOperation op, KeyCompare key_compare_op) { - using acc_type = typename std::iterator_traits::value_type; + using acc_type = ::hipcub::detail::it_value_t; return host_exclusive_scan_by_key_impl(first, last, k_first, @@ -332,20 +339,18 @@ OutputIt host_exclusive_scan_by_key(InputIt first, acc_type{}); } -template::value_type, - test_utils::bfloat16>::value - || std::is_same::value_type, - test_utils::half>::value - || std::is_same::value_type, float>::value, - bool> - = true> +template< + class InputIt, + class KeyIt, + class T, + class OutputIt, + class U, + class KeyCompare, + std::enable_if_t, test_utils::bfloat16> + || std::is_same_v<::hipcub::detail::it_value_t, test_utils::half> + || std::is_same_v<::hipcub::detail::it_value_t, float>, + bool> + = true> OutputIt host_exclusive_scan_by_key(InputIt first, InputIt last, KeyIt k_first, @@ -407,7 +412,7 @@ OutputIt host_inclusive_scan_by_key(InputIt first, BinaryOperation op, KeyCompare key_compare_op) { - using acc_type = typename std::iterator_traits::value_type; + using acc_type = ::hipcub::detail::it_value_t; return host_inclusive_scan_by_key_impl(first, last, k_first, @@ -417,19 +422,17 @@ OutputIt host_inclusive_scan_by_key(InputIt first, acc_type{}); } -template::value_type, - test_utils::bfloat16>::value - || std::is_same::value_type, - test_utils::half>::value - || std::is_same::value_type, float>::value, - bool> - = true> +template< + class InputIt, + class KeyIt, + class OutputIt, + class U, + class KeyCompare, + std::enable_if_t, test_utils::bfloat16> + || std::is_same_v<::hipcub::detail::it_value_t, test_utils::half> + || std::is_same_v<::hipcub::detail::it_value_t, float>, + bool> + = true> OutputIt host_inclusive_scan_by_key(InputIt first, InputIt last, KeyIt k_first, @@ -619,8 +622,8 @@ constexpr T get_min_warp_size(const T block_size, const T max_warp_size) } template -__device__ constexpr bool device_test_enabled_for_warp_size_v - = HIPCUB_DEVICE_WARP_THREADS >= LogicalWarpSize; +constexpr bool device_test_enabled_for_warp_size_v + = (HIPCUB_DEVICE_WARP_THREADS >= LogicalWarpSize); template 0 ? 1 : 0); } +#if defined(__HIP_PLATFORM_AMD__) + +template +using extents = ::hipcub::extents; + +template +struct extents_size; + +template +struct extents_size> +{ + static constexpr std::size_t value = (Dims * ... * 1); +}; + +template +using constant_iterator = ::rocprim::constant_iterator; + +template +using counting_iterator = ::rocprim::counting_iterator; + +template> +using transform_iterator = ::rocprim::transform_iterator; + +struct discard_iterator : public ::rocprim::discard_iterator +{ + using base_type = ::rocprim::discard_iterator; + using value_type = void; + using difference_type = std::ptrdiff_t; + using iterator_category = std::random_access_iterator_tag; + + using base_type::base_type; + + discard_iterator(const ::rocprim::discard_iterator& other) : base_type(other) {} +}; + +using discard_output_iterator = discard_iterator; + +inline auto make_discard_iterator() -> discard_iterator +{ + return discard_iterator(::rocprim::make_discard_iterator()); +} + +#elif defined(__HIP_PLATFORM_NVIDIA__) + +template +using extents = ::cuda::std::extents; + +template +struct extents_size; + +template +struct extents_size> +{ + static constexpr std::size_t value = (Dims * ... * 1); +}; + +template +using constant_iterator = ::cub::ConstantInputIterator; + +template +using counting_iterator = ::cub::CountingInputIterator; + +template> +using transform_iterator = ::cub::TransformInputIterator; + +template +using discard_iterator = ::cub::DiscardOutputIterator; + +template +using discard_output_iterator = ::cub::DiscardOutputIterator; + +template +inline auto make_discard_iterator() -> ::cub::DiscardOutputIterator +{ + return ::cub::DiscardOutputIterator(); +} + +#endif + } // namespace test_utils // Need for hipcub::DeviceReduce::Min/Max etc. @@ -644,17 +726,17 @@ namespace std static constexpr inline T max() { - return std::numeric_limits::max(); + return _HIPCUB_STD::numeric_limits::max(); } static constexpr inline T min() { - return std::numeric_limits::min(); + return _HIPCUB_STD::numeric_limits::min(); } static constexpr inline T lowest() { - return std::numeric_limits::lowest(); + return _HIPCUB_STD::numeric_limits::lowest(); } }; @@ -667,17 +749,17 @@ namespace std static constexpr inline T max() { - return std::numeric_limits::max(); + return _HIPCUB_STD::numeric_limits::max(); } static constexpr inline T min() { - return std::numeric_limits::min(); + return _HIPCUB_STD::numeric_limits::min(); } static constexpr inline T lowest() { - return std::numeric_limits::lowest(); + return _HIPCUB_STD::numeric_limits::lowest(); } }; } diff --git a/projects/hipcub/test/hipcub/test_utils_assertions.hpp b/projects/hipcub/test/hipcub/test_utils_assertions.hpp index 08903f7ce5c7..ffed4ccfdd88 100644 --- a/projects/hipcub/test/hipcub/test_utils_assertions.hpp +++ b/projects/hipcub/test/hipcub/test_utils_assertions.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2023 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -48,7 +48,7 @@ template inline void assert_eq(const std::vector& result, const std::vector& expected, const size_t max_length = SIZE_MAX) { if(max_length == SIZE_MAX || max_length > expected.size()) ASSERT_EQ(result.size(), expected.size()); - for(size_t i = 0; i < std::min(result.size(), max_length); i++) + for(size_t i = 0; i < _HIPCUB_STD::min(result.size(), max_length); i++) { if(bit_equal(result[i], expected[i])) continue; // Check to also regard equality of NaN's, -NaN, +inf, -inf as correct. @@ -74,7 +74,7 @@ inline void assert_eq(const std::vector& result, const std::vector& expect inline void assert_eq(const std::vector& result, const std::vector& expected, const size_t max_length = SIZE_MAX) { if(max_length == SIZE_MAX || max_length > expected.size()) ASSERT_EQ(result.size(), expected.size()); - for(size_t i = 0; i < std::min(result.size(), max_length); i++) + for(size_t i = 0; i < _HIPCUB_STD::min(result.size(), max_length); i++) { if(bit_equal(result[i], expected[i])) continue; // Check to also regard equality of NaN's, -NaN, +inf, -inf as correct. ASSERT_EQ(test_utils::native_half(result[i]), test_utils::native_half(expected[i])) << "where index = " << i; @@ -84,7 +84,7 @@ inline void assert_eq(const std::vector& result, const std::ve inline void assert_eq(const std::vector& result, const std::vector& expected, const size_t max_length = SIZE_MAX) { if(max_length == SIZE_MAX || max_length > expected.size()) ASSERT_EQ(result.size(), expected.size()); - for(size_t i = 0; i < std::min(result.size(), max_length); i++) + for(size_t i = 0; i < _HIPCUB_STD::min(result.size(), max_length); i++) { if(bit_equal(result[i], expected[i])) continue; // Check to also regard equality of NaN's, -NaN, +inf, -inf as correct. ASSERT_EQ(test_utils::native_bfloat16(result[i]), test_utils::native_bfloat16(expected[i])) << "where index = " << i; @@ -137,9 +137,13 @@ inline auto assert_near(const std::vector& result, const std::vector& expe } } -template::value || - std::is_same::value, bool> = true> -inline void assert_near(const std::vector& result, const std::vector& expected, const float percent) +template< + class T, + std::enable_if_t || std::is_same_v, + bool> + = true> +inline void + assert_near(const std::vector& result, const std::vector& expected, const float percent) { ASSERT_EQ(result.size(), expected.size()); for(size_t i = 0; i < result.size(); i++) @@ -176,9 +180,14 @@ inline auto assert_near(const std::vector>& result, const st } } -template::value || - std::is_same::value, bool> = true> -inline void assert_near(const std::vector>& result, const std::vector>& expected, const float percent) +template< + class T, + std::enable_if_t || std::is_same_v, + bool> + = true> +inline void assert_near(const std::vector>& result, + const std::vector>& expected, + const float percent) { ASSERT_EQ(result.size(), expected.size()); for(size_t i = 0; i < result.size(); i++) @@ -209,8 +218,11 @@ inline auto assert_near(const T& result, const T& expected, const float) ASSERT_EQ(result, expected); } -template::value || - std::is_same::value, bool> = true> +template< + class T, + std::enable_if_t || std::is_same_v, + bool> + = true> inline void assert_near(const T& result, const T& expected, const float percent) { if(bit_equal(result, expected)) return; // Check to also regard equality of NaN's, -NaN, +inf, -inf as correct. @@ -224,8 +236,10 @@ inline auto assert_near(const custom_test_type& result, const custom_test_typ { auto diff1 = std::abs(percent * expected.x); auto diff2 = std::abs(percent * expected.y); - if(!bit_equal(result.x, expected.x)) ASSERT_NEAR(result.x, expected.x, diff1); - if(!bit_equal(result.x, expected.x)) ASSERT_NEAR(result.y, expected.y, diff2); + if(!bit_equal(result.x, expected.x)) + ASSERT_NEAR(result.x, expected.x, diff1); + if(!bit_equal(result.y, expected.y)) + ASSERT_NEAR(result.y, expected.y, diff2); } template @@ -254,7 +268,7 @@ inline void assert_bit_eq(const std::vector& result, const std::vector& ex } } -#if HIPCUB_IS_INT128_ENABLED +#if _CCCL_HAS_INT128() inline void assert_bit_eq(const std::vector<__int128_t>& result, const std::vector<__int128_t>& expected) { @@ -328,7 +342,7 @@ inline void assert_bit_eq(const std::vector<__uint128_t>& result, } } } -#endif //HIPCUB_IS_INT128_ENABLED +#endif //_CCCL_HAS_INT128() /// Compile-time assertion for type equality of two objects. template @@ -336,5 +350,5 @@ inline void assert_type(ExpectedT /*obj1*/, ActualT /*obj2*/) { testing::StaticAssertTypeEq(); } -} +} // namespace test_utils #endif // HIPCUB_TEST_HIPCUB_TEST_UTILS_ASSERTIONS_HPP_ diff --git a/projects/hipcub/test/hipcub/test_utils_data_generation.hpp b/projects/hipcub/test/hipcub/test_utils_data_generation.hpp index 75f0010fe184..f73a593e8251 100644 --- a/projects/hipcub/test/hipcub/test_utils_data_generation.hpp +++ b/projects/hipcub/test/hipcub/test_utils_data_generation.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2021-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -48,14 +48,15 @@ T set_half_bits(uint16_t value) // Numeric limits which also supports custom_test_type classes template -struct numeric_limits : std::numeric_limits +struct numeric_limits : _HIPCUB_STD::numeric_limits {}; template<> -struct numeric_limits : public std::numeric_limits +struct numeric_limits : public _HIPCUB_STD::numeric_limits { public: using T = test_utils::half; + static constexpr bool is_specialized = true; static inline T min() { return T(0.00006104f); @@ -74,11 +75,11 @@ struct numeric_limits : public std::numeric_limits::quiet_NaN()); + return T(_HIPCUB_STD::numeric_limits::quiet_NaN()); }; static inline T signaling_NaN() { - return T(std::numeric_limits::signaling_NaN()); + return T(_HIPCUB_STD::numeric_limits::signaling_NaN()); }; static inline T infinity_neg() { @@ -87,17 +88,19 @@ struct numeric_limits : public std::numeric_limits -class numeric_limits : public std::numeric_limits +class numeric_limits + : public _HIPCUB_STD::numeric_limits { public: using T = test_utils::bfloat16; + static constexpr bool is_specialized = true; static inline T max() { return set_half_bits(0x7f7f); }; static inline T min() { - return T(std::numeric_limits::min()); + return T(_HIPCUB_STD::numeric_limits::min()); }; static inline T lowest() { @@ -109,11 +112,11 @@ class numeric_limits : public std::numeric_limits::quiet_NaN()); + return T(_HIPCUB_STD::numeric_limits::quiet_NaN()); }; static inline T signaling_NaN() { - return T(std::numeric_limits::signaling_NaN()); + return T(_HIPCUB_STD::numeric_limits::signaling_NaN()); }; static inline T infinity_neg() { @@ -122,40 +125,41 @@ class numeric_limits : public std::numeric_limits -class numeric_limits : public std::numeric_limits +class numeric_limits : public _HIPCUB_STD::numeric_limits { public: static inline float infinity_neg() { - return -std::numeric_limits::infinity(); + return -_HIPCUB_STD::numeric_limits::infinity(); }; }; // End of extended numeric_limits -#if HIPCUB_IS_INT128_ENABLED +#if _CCCL_HAS_INT128() template -using is_int128 = std::is_same<__int128_t, typename std::remove_cv::type>; +using is_int128 = _HIPCUB_STD::is_same<__int128_t, typename std::remove_cv::type>; template -using is_uint128 = std::is_same<__uint128_t, typename std::remove_cv::type>; +using is_uint128 = _HIPCUB_STD::is_same<__uint128_t, typename std::remove_cv::type>; #else template using is_int128 = std::false_type; template using is_uint128 = std::false_type; -#endif // HIPCUB_IS_INT128_ENABLED +#endif // _CCCL_HAS_INT128() template -using is_half = std::is_same::type>; +using is_half = _HIPCUB_STD::is_same::type>; template -using is_bfloat16 = std::is_same::type>; +using is_bfloat16 = _HIPCUB_STD::is_same::type>; template -using is_native_half = std::is_same::type>; +using is_native_half + = _HIPCUB_STD::is_same::type>; template using is_native_bfloat16 - = std::is_same::type>; + = _HIPCUB_STD::is_same::type>; template struct convert_to_native_t_impl @@ -333,12 +337,12 @@ void add_special_values(std::vector& source, int seed_value) // Actually causes problems with signed/unsigned char on Windows using clang. template struct is_valid_for_int_distribution - : std::integral_constant< - bool, - std::is_same::value || std::is_same::value - || std::is_same::value || std::is_same::value - || std::is_same::value || std::is_same::value - || std::is_same::value || std::is_same::value> + : std::integral_constant || std::is_same_v + || std::is_same_v || std::is_same_v + || std::is_same_v || std::is_same_v + || std::is_same_v + || std::is_same_v> {}; template @@ -360,8 +364,8 @@ inline auto get_random_data(size_t size, T min, T max, int seed_value) -> template inline auto get_random_data(size_t size, S min, U max, int seed_value) -> typename std::enable_if::value && !is_custom_test_type::value - && !std::is_same::value - && !std::is_same::value, + && !std::is_same_v + && !std::is_same_v, std::vector>::type { std::default_random_engine gen(seed_value); @@ -376,7 +380,7 @@ inline auto get_random_data(size_t size, S min, U max, int seed_value) -> template inline auto get_random_data(size_t size, S min, U max, int seed_value) -> - typename std::enable_if::value, std::vector>::type + typename std::enable_if, std::vector>::type { std::default_random_engine gen(seed_value); std::uniform_int_distribution distribution(static_cast(min), @@ -390,7 +394,7 @@ inline auto get_random_data(size_t size, S min, U max, int seed_value) -> template inline auto get_random_data(size_t size, S min, U max, int seed_value) -> - typename std::enable_if::value, std::vector>::type + typename std::enable_if, std::vector>::type { std::default_random_engine gen(seed_value); std::uniform_int_distribution distribution(static_cast(min), @@ -460,16 +464,21 @@ inline std::vector get_random_data01(size_t size, float p, int seed_value) std::bernoulli_distribution distribution(p); std::vector data(size); std::generate(data.begin(), - data.begin() + std::min(size, max_random_size), + data.begin() + _HIPCUB_STD::min(size, max_random_size), [&]() { return convert_to_device(distribution(gen)); }); for(size_t i = max_random_size; i < size; i += max_random_size) { - std::copy_n(data.begin(), std::min(size - i, max_random_size), data.begin() + i); + std::copy_n(data.begin(), _HIPCUB_STD::min(size - i, max_random_size), data.begin() + i); } return data; } +// Windows HIP cannot support allocations >= 4 GiB. +#ifdef _WIN32 +template +#else template +#endif inline std::vector get_large_sizes(int seed_value) { // clang-format off diff --git a/projects/hipcub/test/hipcub/test_utils_functional.hpp b/projects/hipcub/test/hipcub/test_utils_functional.hpp index 404ad12b11c0..d74214479ddf 100644 --- a/projects/hipcub/test/hipcub/test_utils_functional.hpp +++ b/projects/hipcub/test/hipcub/test_utils_functional.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2024-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -35,7 +35,8 @@ namespace test_utils struct less { template - HIPCUB_HOST_DEVICE constexpr bool operator()(const T& a, const T& b) const + HIPCUB_HOST_DEVICE + constexpr bool operator()(const T& a, const T& b) const { return a < b; } @@ -44,7 +45,8 @@ struct less struct less_equal { template - HIPCUB_HOST_DEVICE constexpr bool operator()(const T& a, const T& b) const + HIPCUB_HOST_DEVICE + constexpr bool operator()(const T& a, const T& b) const { return a <= b; } @@ -53,7 +55,8 @@ struct less_equal struct greater { template - HIPCUB_HOST_DEVICE constexpr bool operator()(const T& a, const T& b) const + HIPCUB_HOST_DEVICE + constexpr bool operator()(const T& a, const T& b) const { return a > b; } @@ -62,16 +65,38 @@ struct greater struct greater_equal { template - HIPCUB_HOST_DEVICE constexpr bool operator()(const T& a, const T& b) const + HIPCUB_HOST_DEVICE + constexpr bool operator()(const T& a, const T& b) const { return a >= b; } }; +struct equal +{ + template + HIPCUB_HOST_DEVICE + inline constexpr auto operator()(const T& a, const U& b) const + { + return a == b; + } +}; + +struct not_equal +{ + template + HIPCUB_HOST_DEVICE + inline constexpr auto operator()(const T& a, const U& b) const + { + return a != b; + } +}; + struct plus { - template - HIPCUB_HOST_DEVICE inline constexpr T operator()(const T& a, const T& b) const + template + HIPCUB_HOST_DEVICE + inline constexpr auto operator()(const T& a, const U& b) const -> decltype(a + b) { return a + b; } @@ -79,8 +104,9 @@ struct plus struct minus { - template - HIPCUB_HOST_DEVICE inline constexpr T operator()(const T& a, const T& b) const + template + HIPCUB_HOST_DEVICE + inline constexpr auto operator()(const T& a, const U& b) const -> decltype(a - b) { return a - b; } @@ -88,13 +114,48 @@ struct minus struct multiplies { - template - HIPCUB_HOST_DEVICE inline constexpr T operator()(const T& a, const T& b) const + template + HIPCUB_HOST_DEVICE + inline constexpr auto operator()(const T& a, const U& b) const -> decltype(a * b) { return a * b; } }; +struct divides +{ + template + HIPCUB_HOST_DEVICE + inline constexpr auto operator()(const T& a, const U& b) const -> decltype(a / b) + { + return a / b; + } +}; + +struct maximum +{ + template + HIPCUB_HOST_DEVICE + auto operator()(const T& a, const U& b) const + { + using result_type = ::std::common_type_t; + result_type ra = a, rb = b; + return ra < rb ? rb : ra; + } +}; + +struct minimum +{ + template + HIPCUB_HOST_DEVICE + auto operator()(const T& a, const U& b) const + { + using result_type = ::std::common_type_t; + result_type ra = a, rb = b; + return ra < rb ? ra : rb; + } +}; + // HALF template<> HIPCUB_HOST_DEVICE inline bool diff --git a/projects/hipcub/test/hipcub/test_utils_sort_comparator.hpp b/projects/hipcub/test/hipcub/test_utils_sort_comparator.hpp index 27fc65c40769..00fe6bb8d86a 100644 --- a/projects/hipcub/test/hipcub/test_utils_sort_comparator.hpp +++ b/projects/hipcub/test/hipcub/test_utils_sort_comparator.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2017-2024 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -33,6 +33,7 @@ #include "test_utils_custom_test_types.hpp" #include "test_utils_half.hpp" +#include #include #include @@ -42,40 +43,62 @@ namespace test_utils namespace detail { +template +constexpr bool is_extended_int + = std::is_same_v || std::is_same_v; template::CATEGORY == hipcub::SIGNED_INTEGER - || hipcub::NumericTraits::CATEGORY == hipcub::UNSIGNED_INTEGER, - int> HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP + std::enable_if_t< + // Catch integral types and extended integral types. + // The std::is_same_v<...> clauses can be removed once + // libhipcxx is a hard depedency and test types half_t + // and bfloat_t are removed. + _HIPCUB_STD::is_integral_v || is_extended_int, + int> = 0> Key to_bits(const Key key) { - static constexpr Key radix_mask_upper - = EndBit == 8 * sizeof(Key) ? ~Key(0) : static_cast((Key(1) << EndBit) - 1); + using Bits = typename hipcub::Traits::UnsignedBits; + static constexpr Key radix_mask_upper = EndBit == 8 * sizeof(Key) + ? static_cast(~Bits(0)) + : static_cast((Bits(1) << EndBit) - 1); static constexpr Key radix_mask_bottom = static_cast((Key(1) << StartBit) - 1); static constexpr Key radix_mask = radix_mask_upper ^ radix_mask_bottom; return key & radix_mask; } +template +constexpr bool is_extended_fp + = std::is_same_v || std::is_same_v + || std::is_same_v || std::is_same_v + || std::is_same_v; + template::CATEGORY == hipcub::FLOATING_POINT, int> + std::enable_if_t< + // Catch floating types and extended floating types. + // The std::is_same_v<...> clauses can be removed once + // libhipcxx is a hard depedency and test types half_t + // and bfloat_t are removed. + _HIPCUB_STD::is_floating_point_v || is_extended_fp, + int> = 0> -HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP auto to_bits(const Key key) +auto to_bits(const Key key) { using unsigned_bits_type = typename hipcub::NumericTraits::UnsignedBits; + static_assert(sizeof(unsigned_bits_type) == sizeof(Key)); unsigned_bits_type bit_key; - memcpy(&bit_key, &key, sizeof(Key)); + std::memcpy(&bit_key, &key, sizeof(unsigned_bits_type)); // Remove signed zero, this case is supposed to be treated the same as // unsigned zero in hipcub sorting algorithms. - constexpr unsigned_bits_type minus_zero = unsigned_bits_type{1} << (8 * sizeof(Key) - 1); + constexpr unsigned_bits_type minus_zero = unsigned_bits_type{1} + << (8 * sizeof(unsigned_bits_type) - 1); + // Positive and negative zero should compare the same. if(bit_key == minus_zero) { @@ -96,7 +119,7 @@ HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP auto to_bits(const Key key) template::value, int> = 0> + std::enable_if_t>::value, int> = 0> auto to_bits(const Key& key) { using inner_t = typename inner_type::type; @@ -108,19 +131,16 @@ auto to_bits(const Key& key) uint32_t, std::conditional_t>>; - auto bit_key_upper = static_cast(to_bits<0, sizeof(key.x) * 8>(key.x)); - auto bit_key_lower = static_cast(to_bits<0, sizeof(key.y) * 8>(key.y)); + auto bit_key_upper = static_cast(to_bits<0, sizeof(inner_t) * 8>(key.x)); + auto bit_key_lower = static_cast(to_bits<0, sizeof(inner_t) * 8>(key.y)); // Flip sign bit to properly order signed types - HIPCUB_CLANG_SUPPRESS_DEPRECATED_PUSH - if(::hipcub::NumericTraits::CATEGORY == hipcub::SIGNED_INTEGER) - HIPCUB_CLANG_SUPPRESS_DEPRECATED_POP - { - constexpr auto sign_bit = static_cast(1) - << (sizeof(inner_t) * 8 - 1); - bit_key_upper ^= sign_bit; - bit_key_lower ^= sign_bit; - } + if(std::is_signed::value) + { + constexpr auto sign_bit = static_cast(1) << (sizeof(inner_t) * 8 - 1); + bit_key_upper ^= sign_bit; + bit_key_lower ^= sign_bit; + } // Create the result containing both parts const auto bit_key diff --git a/projects/hipcub/test/hipcub/test_utils_thread_operators.hpp b/projects/hipcub/test/hipcub/test_utils_thread_operators.hpp index 8a85a5f4dd11..dce0155b29cb 100644 --- a/projects/hipcub/test/hipcub/test_utils_thread_operators.hpp +++ b/projects/hipcub/test/hipcub/test_utils_thread_operators.hpp @@ -1,6 +1,6 @@ // MIT License // -// Copyright (c) 2023-2025 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2023-2026 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -73,7 +73,7 @@ struct ExtendedFloatBoolOp }; /** - * \brief ExtendedFloatBinOp general functor - Because hipcub::Sum(), Difference(), Division(), + * \brief ExtendedFloatBinOp general functor - Because test_utils::plus{}, Difference(), Division(), * Max() and Min() don't work with input types , * and * and . @@ -155,13 +155,13 @@ struct ArgMax { template::value - || std::is_same::value, + std::enable_if_t + || std::is_same_v, bool> = true> - HIPCUB_HOST_DEVICE __forceinline__ hipcub::KeyValuePair - operator()(const hipcub::KeyValuePair& a, - const hipcub::KeyValuePair& b) const + HIPCUB_HOST_DEVICE __forceinline__ + hipcub::KeyValuePair operator()(const hipcub::KeyValuePair& a, + const hipcub::KeyValuePair& b) const { const hipcub::KeyValuePair native_a(a.key, a.value); const hipcub::KeyValuePair native_b(b.key, b.value); @@ -179,13 +179,13 @@ struct ArgMin { template::value - || std::is_same::value, + std::enable_if_t + || std::is_same_v, bool> = true> - HIPCUB_HOST_DEVICE __forceinline__ hipcub::KeyValuePair - operator()(const hipcub::KeyValuePair& a, - const hipcub::KeyValuePair& b) const + HIPCUB_HOST_DEVICE __forceinline__ + hipcub::KeyValuePair operator()(const hipcub::KeyValuePair& a, + const hipcub::KeyValuePair& b) const { const hipcub::KeyValuePair native_a(a.key, a.value); const hipcub::KeyValuePair native_b(b.key, b.value); @@ -279,50 +279,50 @@ struct AlgebraicSelector template struct MaxSelector { - using type = hipcub::Max; + using type = test_utils::maximum; }; template struct MaxSelector, test_utils::custom_test_type> { - using type = CustomTestOp; + using type = CustomTestOp; }; template struct MaxSelector { - using type = ExtendedFloatBinOp; + using type = ExtendedFloatBinOp; }; template struct MaxSelector { - using type = ExtendedFloatBinOp; + using type = ExtendedFloatBinOp; }; // Min functor selector. template struct MinSelector { - using type = hipcub::Min; + using type = test_utils::minimum; }; template struct MinSelector, test_utils::custom_test_type> { - using type = CustomTestOp; + using type = CustomTestOp; }; template struct MinSelector { - using type = ExtendedFloatBinOp; + using type = ExtendedFloatBinOp; }; template struct MinSelector { - using type = ExtendedFloatBinOp; + using type = ExtendedFloatBinOp; }; // ArgMax functor selector diff --git a/projects/hipcub/toolchain-windows.cmake b/projects/hipcub/toolchain-windows.cmake index 6b688314c935..313ab047ba7b 100644 --- a/projects/hipcub/toolchain-windows.cmake +++ b/projects/hipcub/toolchain-windows.cmake @@ -31,4 +31,11 @@ if (DEFINED ENV{VCPKG_PATH}) else() set(VCPKG_PATH "C:/github/vcpkg") endif() + +# Force static libraries on Windows +set(VCPKG_TARGET_TRIPLET "x64-windows-static") + +# Force static MSVC runtime (/MT) to match vcpkg static triplet +set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") + include("${VCPKG_PATH}/scripts/buildsystems/vcpkg.cmake") diff --git a/projects/rocprim/CHANGELOG.md b/projects/rocprim/CHANGELOG.md index 09c33f6bf894..60fdc0b69917 100644 --- a/projects/rocprim/CHANGELOG.md +++ b/projects/rocprim/CHANGELOG.md @@ -4,6 +4,18 @@ Full documentation for rocPRIM is available at [https://rocm.docs.amd.com/projec ## rocPRIM 5.0.0 for ROCm 10.0 +### Added + +* Added C++ 17 style type_traits utilities + * is_floating_point_v + * is_integral_v + * is_arithmetic_v + * is_fundamental_v + * is_unsigned_v + * is_signed_v + * is_scalar_v + * is_compound_v + ### Changed * Combined and simplified separate assertion templates using `std::is_floating_point`, `rocprim::half`, and `rocprim::bfloat16` to use `rocprim::is_floating_point` diff --git a/projects/rocprim/rmake.py b/projects/rocprim/rmake.py index ae6274876787..c1f25a4c2ba8 100644 --- a/projects/rocprim/rmake.py +++ b/projects/rocprim/rmake.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 -""" Copyright (c) 2021-2025 Advanced Micro Devices, Inc. All rights reserved. +""" Copyright (c) 2021-2026 Advanced Micro Devices, Inc. All rights reserved. Manage build and installation""" import re diff --git a/projects/rocprim/rocprim/include/rocprim/type_traits.hpp b/projects/rocprim/rocprim/include/rocprim/type_traits.hpp index 10e602c869a2..f8e5cffb2b8e 100644 --- a/projects/rocprim/rocprim/include/rocprim/type_traits.hpp +++ b/projects/rocprim/rocprim/include/rocprim/type_traits.hpp @@ -1425,6 +1425,34 @@ template struct is_compound : std::integral_constant().is_compound()> {}; +#ifndef DOXYGEN_DOCUMENTATION_BUILD + +template +constexpr bool is_floating_point_v = is_floating_point::value; + +template +constexpr bool is_integral_v = is_integral::value; + +template +constexpr bool is_arithmetic_v = is_arithmetic::value; + +template +constexpr bool is_fundamental_v = is_fundamental::value; + +template +constexpr bool is_unsigned_v = is_unsigned::value; + +template +constexpr bool is_signed_v = is_unsigned::value; + +template +constexpr bool is_scalar_v = is_scalar::value; + +template +constexpr bool is_compound_v = is_compound::value; + +#endif + static_assert(::rocprim::traits::radix_key_codec::radix_key_fundamental::value, "'int' should be fundamental"); static_assert(!::rocprim::traits::radix_key_codec::radix_key_fundamental::value, @@ -1436,6 +1464,13 @@ static_assert(::rocprim::traits::radix_key_codec::radix_key_fundamental::value, "'rocprim::int128_t*' should not be fundamental"); +static_assert(::rocprim::is_floating_point_v<__half>, "__half should be a floating point type"); +static_assert(::rocprim::is_floating_point_v, "bfloat16 should be a floating point type"); +static_assert(::rocprim::is_integral_v<::rocprim::int128_t>, + "::rocprim::int128_t should be a integral type"); +static_assert(::rocprim::is_integral_v<::rocprim::uint128_t>, + "::rocprim::uint128_t should be a integral type"); + END_ROCPRIM_NAMESPACE #endif